scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-01 02:02:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

Merge branch 'litellm_internal_staging' into cursor/fix-non-admin-key-edit-allowed-routes-27005-b201

Browse source
This commit is contained in:
shivam 2026-05-07 17:52:31 -07:00
commit 3ea5c17813
No known key found for this signature in database
863 changed files with 31637 additions and 7937 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

355
.circleci/config.yml
View file

@ -350,7 +350,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"langfuse\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -395,7 +403,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
# Store test results
@ -471,7 +487,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -495,7 +519,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"assistants\""
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -528,14 +560,19 @@ jobs:
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
IGNORE_DIRS=(
"tests/llm_translation/realtime"
)
IGNORE_ARGS=""
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
mkdir -p test-results
# Glob excludes the realtime/ subdirectory since it has its own job
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread \
--retries 2 --retry-delay 5 \
--max-worker-restart=5"
no_output_timeout: 15m
# Store test results
@ -560,7 +597,17 @@ jobs:
command: |
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -593,7 +640,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -626,7 +681,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -660,7 +726,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -702,7 +777,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 8"
no_output_timeout: 15m
# Store test results
@ -725,7 +808,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -758,7 +850,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -793,7 +894,15 @@ jobs:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -815,7 +924,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -848,7 +966,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -882,7 +1009,16 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -916,7 +1052,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
@ -939,7 +1083,18 @@ jobs:
- run:
name: Run tests
command: |
LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
mkdir -p test-results
export LITELLM_LOG=WARNING
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=litellm --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
--timeout=120 --timeout_method=thread"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -972,7 +1127,15 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
@ -1012,14 +1175,19 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv \
mkdir -p test-results
TEST_FILES=$(printf "%s\n" \
tests/local_testing/test_dual_cache.py \
tests/local_testing/test_redis_batch_optimizations.py \
tests/local_testing/test_router_utils.py \
--cov=litellm --cov-report=xml \
-x -s -v --junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1
tests/local_testing/test_router_utils.py)
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
no_output_timeout: 20m
- run:
name: Rename the coverage files
@ -1260,8 +1428,17 @@ jobs:
- run:
name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
build_and_test:
machine:
@ -1331,7 +1508,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
mkdir -p test-results
# Original used `tests/*.py` (top-level only); the `--ignore=...`
# flags were vestigial since shell globbing did not descend into
# subdirectories. Replicate by globbing only top-level test files.
TEST_FILES=$(circleci tests glob "tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -v -x \
--junitxml=test-results/junit.xml \
-n 4 \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1406,7 +1594,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -vv \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1475,7 +1670,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- run:
@ -1518,7 +1720,14 @@ jobs:
- run:
name: Run second round of tests
command: |
uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1587,8 +1796,17 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
when: always
@ -1676,7 +1894,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
# Store test results
@ -1732,7 +1957,14 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Stop and remove containers
@ -1805,9 +2037,18 @@ jobs:
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
no_output_timeout: 15m
# Clean up first container
- store_test_results:
path: test-results
- run:
name: Stop and remove first container
command: |
@ -1935,12 +2176,19 @@ jobs:
name: Run Vertex AI, Google AI Studio Node.js tests
command: |
cd tests/pass_through_tests
npx jest . --verbose
NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
no_output_timeout: 30m
- run:
name: Run tests
command: |
uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -1997,9 +2245,16 @@ jobs:
- run:
name: Run Claude Agent SDK E2E Tests
command: |
mkdir -p test-results
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
# Store test results
@ -2138,17 +2393,23 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
# The cimg/python:3.12-browsers image already ships the Chromium system
# libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
# `--with-deps` triggers a redundant apt-get update + install that adds
# 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
# so we install just the browser binary.
command: |
cd ui/litellm-dashboard
npm ci
npx playwright install chromium --with-deps
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- ~/.cache/ms-playwright
- run:
name: Build UI from source
# Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.

1
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -141,6 +141,7 @@ jobs:
tests/proxy_unit_tests/test_server_root_path.py
tests/proxy_unit_tests/test_proxy_pass_user_config.py
tests/proxy_unit_tests/test_proxy_token_counter.py
tests/proxy_unit_tests/test_request_size_limit_middleware.py
workers: 4
dist: loadscope
timeout: 15

3
.gitignore vendored
View file

@ -100,4 +100,5 @@ STABILIZATION_TODO.md
**/playwright-report
**/*.storageState.json
**/coverage
test-config
test-config
.vscode

20
Dockerfile
View file

@ -1,8 +1,8 @@
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
GLOBAL="$(npm root -g)" && \
for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
name="${pkg##*/}"; \
@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
COPY --from=builder /app /app
# Prisma binaries live in $HOME/.cache (default prisma-python location),
# which is /root/.cache here. Copy them from the builder so they survive
# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
COPY --from=builder /root/.cache /root/.cache
# which is /root/.cache here. Copy only the Prisma subdirs — copying the
# whole /root/.cache drags in the uv build cache (~660 MB, includes a
# setuptools wheel that surfaces as a CVE finding even though it's not
# on the runtime sys.path).
COPY --from=builder /root/.cache/prisma /root/.cache/prisma
COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

7
codecov.yaml
View file

@ -1,3 +1,8 @@
codecov:
require_ci_to_pass: false # post coverage status even if CI has unrelated failures
notify:
wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
component_management:
individual_components:
- component_id: "Router"
@ -28,7 +33,7 @@ coverage:
project:
default:
target: auto
threshold: 1% # at maximum allow project coverage to drop by 1%
threshold: 0% # do not allow project coverage to drop
patch:
default:
target: auto

2
cookbook/litellm-ollama-docker-image/requirements.txt
View file

@ -1 +1 @@
litellm==1.83.5
litellm==1.83.14

34
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -116,21 +116,32 @@ spec:
name: {{ include "redis.secretName" .Subcharts.redis }}
key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
{{- end }}
{{- /*
Inject LITELLM_LOG only when envVars does not already define it.
*/}}
{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
- name: LITELLM_LOG
value: {{ .Values.logLevel | quote }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- if .Values.separateHealthApp }}
- name: SEPARATE_HEALTH_APP
value: "1"
- name: SEPARATE_HEALTH_PORT
value: {{ .Values.separateHealthPort | default "8081" | quote }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.migrationJob.enabled }}
# Schema updates are owned by the dedicated migrations Job; skip
# the proxy's startup `prisma db push` so N replicas don't race
# one DB on every rollout. Placed last (after envVars and
# extraEnvVars) so this override can't be silently shadowed by a
# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
# semantics — same pattern the migrations Job uses.
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
@ -158,15 +169,10 @@ spec:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- if .Values.separateHealthApp }}
- name: health
containerPort: {{ .Values.separateHealthPort | default 8081 }}
protocol: TCP
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@ -175,7 +181,7 @@ spec:
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@ -184,7 +190,7 @@ spec:
startupProbe:
httpGet:
path: {{ .Values.startupProbe.path | quote }}
port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
port: "http"
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

10
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -257,16 +257,16 @@ tests:
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 10
value: 15
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 3
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
@ -278,7 +278,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
@ -296,7 +296,7 @@ tests:
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 1
value: 5
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1

29
deploy/charts/litellm-helm/values.yaml
View file

@ -88,26 +88,20 @@ service:
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
# Separate health app configuration
# When enabled, health checks will use a separate port and the application
# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
separateHealthApp: false
separateHealthPort: 8081
# Probe tuning for proxy container
# Probes for LiteLLM gateway container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
failureThreshold: 5
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
@ -115,7 +109,7 @@ startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 1
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 30
@ -329,6 +323,17 @@ migrationJob:
helm:
enabled: false
# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
# any `envFrom:` source. If you currently source LITELLM_LOG from an
# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
# disable injection — otherwise this value silently overrides your secret /
# configmap entry.
#
# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
# this injection entirely when envVars already defines LITELLM_LOG.
logLevel: INFO
# Additional environment variables to be added to the deployment as a map of key-value pairs
envVars: {}

4
docker/Dockerfile.database
View file

@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
EXPOSE 4000/tcp
COPY docker/supervisord.conf /etc/supervisord.conf
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

5
docker/Dockerfile.non_root
View file

@ -32,7 +32,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
PATH="/app/.venv/bin:${PATH}" \
LITELLM_NON_ROOT=true \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
XDG_CACHE_HOME=/app/.cache
# Copy dependency metadata first for layer caching
@ -104,17 +103,15 @@ RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
done
COPY --from=builder /app /app
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
HOME=/app \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache \

8
docker/prod_entrypoint.sh
View file

@ -1,14 +1,8 @@
#!/bin/sh
if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
export LITELLM_ARGS="$@"
export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
exec supervisord -c /etc/supervisord.conf
fi
if [ "$USE_DDTRACE" = "true" ]; then
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
else
exec litellm "$@"
fi
fi

46
docker/supervisord.conf
View file

@ -1,46 +0,0 @@
[supervisord]
nodaemon=true
loglevel=info
logfile=/tmp/supervisord.log
pidfile=/tmp/supervisord.pid
[group:litellm]
programs=main,health
[program:main]
command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
autostart=true
autorestart=true
startretries=3
priority=1
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[program:health]
command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
autostart=true
autorestart=true
startretries=3
priority=2
exitcodes=0
stopasgroup=true
killasgroup=true
stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
stdout_logfile=/dev/stdout
stderr_logfile=/dev/stderr
stdout_logfile_maxbytes = 0
stderr_logfile_maxbytes = 0
environment=PYTHONUNBUFFERED=true
[eventlistener:process_monitor]
command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
autostart=true
autorestart=true

BIN
docs/images/local-testing/hosted-vllm-custom-tool-local-test.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 82 KiB

26
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -11,6 +11,10 @@ from typing import Literal
import litellm
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import (
is_text_content_call_type,
iter_message_text,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from fastapi import HTTPException
@ -73,10 +77,9 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
- check if user id part of blocked list
"""
self.print_verbose("Inside Banned Keyword List Pre-Call Hook")
if call_type == "completion" and "messages" in data:
for m in data["messages"]:
if "content" in m and isinstance(m["content"], str):
self.test_violation(test_str=m["content"])
if is_text_content_call_type(call_type):
for text in iter_message_text(data):
self.test_violation(test_str=text)
except HTTPException as e:
raise e
@ -93,11 +96,16 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
response,
):
if isinstance(response, litellm.ModelResponse) and isinstance(
response.choices[0], litellm.utils.Choices
):
for word in self.banned_keywords_list:
self.test_violation(test_str=response.choices[0].message.content or "")
if not isinstance(response, litellm.ModelResponse):
return
for choice in response.choices:
if not isinstance(choice, litellm.utils.Choices):
continue
message = getattr(choice, "message", None)
content = getattr(message, "content", None)
if isinstance(content, str):
self.test_violation(test_str=content)
async def async_post_call_streaming_hook(
self,

9
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@ -94,11 +95,9 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
- Calls Google's Text Moderation API
- Rejects request if it fails safety check
"""
if "messages" in data and isinstance(data["messages"], list):
text = ""
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
if text:
document = self.language_document(content=text, type_=self.document_type)
request = self.moderate_text_request(

8
enterprise/enterprise_hooks/openai_moderation.py
View file

@ -19,6 +19,7 @@ import litellm
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
@ -37,11 +38,8 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
user_api_key_dict: UserAPIKeyAuth,
call_type: CallTypesLiteral,
):
text = ""
if "messages" in data and isinstance(data["messages"], list):
for m in data["messages"]: # assume messages is a list
if "content" in m and isinstance(m["content"], str):
text += m["content"]
# Covers multimodal list content + Responses-API input.
text = "".join(iter_message_text(data))
from litellm.proxy.proxy_server import llm_router

64
enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py
View file

@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import walk_user_text
GUARDRAIL_NAME = "hide_secrets"
@ -473,23 +474,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
if await self.should_run_check(user_api_key_dict) is False:
return
if "messages" in data and isinstance(data["messages"], list):
for message in data["messages"]:
if "content" in message and isinstance(message["content"], str):
detected_secrets = self.scan_message_for_secrets(message["content"])
# Covers multimodal list content + Responses-API input.
def _redact_message_text(text: str) -> str:
detected_secrets = self.scan_message_for_secrets(text)
for secret in detected_secrets:
text = text.replace(secret["value"], "[REDACTED]")
if detected_secrets:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
return text
for secret in detected_secrets:
message["content"] = message["content"].replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in message: {secret_types}"
)
else:
verbose_proxy_logger.debug("No secrets detected on input.")
walk_user_text(data, _redact_message_text)
if "prompt" in data:
if isinstance(data["prompt"], str):
@ -504,11 +501,15 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
elif isinstance(data["prompt"], list):
for item in data["prompt"]:
# Index back into the list — assigning to ``item`` would only
# rebind the loop variable and leave ``data["prompt"]``
# carrying the unredacted secret.
for idx, item in enumerate(data["prompt"]):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
item = item.replace(secret["value"], "[REDACTED]")
data["prompt"][idx] = item
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
@ -517,31 +518,6 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
f"Detected and redacted secrets in prompt: {secret_types}"
)
if "input" in data:
if isinstance(data["input"], str):
detected_secrets = self.scan_message_for_secrets(data["input"])
for secret in detected_secrets:
data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
if len(detected_secrets) > 0:
secret_types = [secret["type"] for secret in detected_secrets]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
elif isinstance(data["input"], list):
_input_in_request = data["input"]
for idx, item in enumerate(_input_in_request):
if isinstance(item, str):
detected_secrets = self.scan_message_for_secrets(item)
for secret in detected_secrets:
_input_in_request[idx] = item.replace(
secret["value"], "[REDACTED]"
)
if len(detected_secrets) > 0:
secret_types = [
secret["type"] for secret in detected_secrets
]
verbose_proxy_logger.warning(
f"Detected and redacted secrets in input: {secret_types}"
)
verbose_proxy_logger.debug("Data after redacting input %s", data)
# ``data["input"]`` (Responses API and embeddings/moderation) is
# already covered by ``walk_user_text`` above.
return

318
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -15,6 +15,11 @@ from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
can_access_resource,
)
from litellm.proxy._types import (
CallTypes,
LiteLLM_ManagedFileTable,
@ -99,6 +104,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
created_by=user_api_key_dict.user_id,
team_id=user_api_key_dict.team_id,
updated_by=user_api_key_dict.user_id,
)
await self.internal_usage_cache.async_set_cache(
@ -114,6 +120,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
@ -125,7 +132,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
db_data["storage_backend"] = hidden_params["storage_backend"]
if "storage_url" in hidden_params:
db_data["storage_url"] = hidden_params["storage_url"]
verbose_logger.debug(
f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
f"storage_url={db_data.get('storage_url')}"
@ -171,6 +178,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
"status": file_object.status,
},
@ -229,15 +237,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def can_user_call_unified_file_id(
self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified file id
user_id = user_api_key_dict.user_id
managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": unified_file_id}
)
if managed_file:
return managed_file.created_by == user_id
return can_access_resource(
user_api_key_dict=user_api_key_dict,
created_by=managed_file.created_by,
resource_team_id=managed_file.team_id,
)
raise HTTPException(
status_code=404,
detail=f"File not found: {unified_file_id}",
@ -246,8 +255,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def can_user_call_unified_object_id(
self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
) -> bool:
## check if the user has access to the unified object id
user_id = user_api_key_dict.user_id
managed_object = (
await self.prisma_client.db.litellm_managedobjecttable.find_first(
where={"unified_object_id": unified_object_id}
@ -255,7 +262,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if managed_object:
return managed_object.created_by == user_id
return can_access_resource(
user_api_key_dict=user_api_key_dict,
created_by=managed_object.created_by,
resource_team_id=managed_object.team_id,
)
raise HTTPException(
status_code=404,
detail=f"Object not found: {unified_object_id}",
@ -285,28 +296,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
raise Exception(
"Filtering by 'target_model_names' is not supported when using managed batches."
)
where_clause: Dict[str, Any] = {"file_purpose": "batch"}
# Filter by user who created the batch
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return build_list_page([])
where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
if after:
where_clause["id"] = {"gt": after}
# Fetch more than needed to allow for post-fetch filtering
fetch_limit = limit or 20
if target_model_names:
# Fetch extra to account for filtering
# Oversample so post-fetch model-name filtering still has enough rows.
fetch_limit = max(fetch_limit * 3, 100)
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where=where_clause,
take=fetch_limit,
order={"created_at": "desc"},
)
batch_objects: List[LiteLLMBatch] = []
for batch in batches:
try:
@ -314,7 +324,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if len(batch_objects) >= (limit or 20):
break
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_data = (
json.loads(batch.file_object)
if isinstance(batch.file_object, str)
else batch.file_object
)
batch_obj = LiteLLMBatch(**batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
@ -324,27 +338,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Failed to parse batch object {batch.unified_object_id}: {e}"
)
continue
return {
"object": "list",
"data": batch_objects,
"first_id": batch_objects[0].id if batch_objects else None,
"last_id": batch_objects[-1].id if batch_objects else None,
"has_more": len(batch_objects) == (limit or 20),
}
return build_list_page(
batch_objects, has_more=len(batch_objects) == (limit or 20)
)
async def get_user_created_file_ids(
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
) -> List[OpenAIFileObject]:
"""
Get all file ids created by the user for a list of model object ids
Get all file ids the caller is allowed to see for a list of model
object ids. Service-account keys (no user_id) are scoped to their
team via ``team_id``; admins see all matches.
Returns:
- List of OpenAIFileObject's
"""
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return []
file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
where={
"created_by": user_api_key_dict.user_id,
**owner_filter,
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
@ -377,11 +393,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"""
Check if the user has access to a list of file IDs.
Only checks managed (unified) file IDs.
Args:
file_ids: List of file IDs to check access for
user_api_key_dict: User API key authentication details
Raises:
HTTPException: If user doesn't have access to any of the files
"""
@ -419,10 +435,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
### HANDLE TRANSFORMATIONS ###
# Check both completion and acompletion call types
is_completion_call = (
call_type == CallTypes.completion.value
call_type == CallTypes.completion.value
or call_type == CallTypes.acompletion.value
)
if is_completion_call:
messages = data.get("messages")
model = data.get("model", "")
@ -431,22 +447,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
# Check if any files are stored in storage backends and need base64 conversion
# This is needed for Vertex AI/Gemini which requires base64 content
is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower())
is_vertex_ai = model and (
"vertex_ai" in model or "gemini" in model.lower()
)
if is_vertex_ai:
await self._convert_storage_files_to_base64(
messages=messages,
file_ids=file_ids,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
)
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
elif (
call_type == CallTypes.aresponses.value
or call_type == CallTypes.responses.value
):
# Handle managed files in responses API input and tools
file_ids = []
@ -611,7 +632,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if model_id is None:
model_id = cast(
Optional[str],
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
kwargs.get("litellm_metadata", {})
.get("model_info", {})
.get("id", None),
)
mapped_file_id: Optional[str] = None
if input_file_id and model_file_id_mapping and model_id:
@ -648,7 +671,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> List[str]:
"""
Gets file ids from responses API input.
The input can be:
- A string (no files)
- A list of input items, where each item can have:
@ -656,32 +679,35 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
- content: a list that can contain items with type: "input_file" and file_id
"""
file_ids: List[str] = []
if isinstance(input, str):
return file_ids
if not isinstance(input, list):
return file_ids
for item in input:
if not isinstance(item, dict):
continue
# Check for direct input_file type
if item.get("type") == "input_file":
file_id = item.get("file_id")
if file_id:
file_ids.append(file_id)
# Check for input_file in content array
content = item.get("content")
if isinstance(content, list):
for content_item in content:
if isinstance(content_item, dict) and content_item.get("type") == "input_file":
if (
isinstance(content_item, dict)
and content_item.get("type") == "input_file"
):
file_id = content_item.get("file_id")
if file_id:
file_ids.append(file_id)
return file_ids
def get_file_ids_from_responses_tools(
@ -689,7 +715,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> List[str]:
"""
Gets file ids from responses API tools parameter.
The tools can contain code_interpreter with container.file_ids:
[
{
@ -699,14 +725,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
]
"""
file_ids: List[str] = []
if not isinstance(tools, list):
return file_ids
for tool in tools:
if not isinstance(tool, dict):
continue
# Check for code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
@ -716,7 +742,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in container_file_ids:
if isinstance(file_id, str):
file_ids.append(file_id)
return file_ids
def get_vector_store_ids_from_file_search_tools(
@ -916,10 +942,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Emit Prometheus metrics for managed file creation
prom_logger = self._get_prometheus_logger()
if prom_logger:
first_model = target_model_names_list[0] if target_model_names_list else None
first_model = (
target_model_names_list[0] if target_model_names_list else None
)
first_provider = ""
if responses:
first_provider = getattr(responses[0], "_hidden_params", {}).get("custom_llm_provider") or ""
first_provider = (
getattr(responses[0], "_hidden_params", {}).get(
"custom_llm_provider"
)
or ""
)
prom_logger.record_managed_file_created(
model=first_model or "",
api_provider=first_provider,
@ -1073,16 +1106,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_name=resolved_model_name,
)
setattr(response, file_attr, unified_file_id)
# Use llm_router credentials when available. Without credentials,
# Azure and other auth-required providers return 500/401.
file_object = None
try:
# Import module and use getattr for better testability with mocks
import litellm.proxy.proxy_server as proxy_server_module
_llm_router = getattr(proxy_server_module, 'llm_router', None)
_llm_router = getattr(
proxy_server_module, "llm_router", None
)
if _llm_router is not None and model_id:
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
_creds = (
_llm_router.get_deployment_credentials_with_provider(
model_id
)
or {}
)
file_object = await litellm.afile_retrieve(
file_id=original_file_id,
**_creds,
@ -1099,7 +1140,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
verbose_logger.warning(
f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
)
await self.store_unified_file_id(
file_id=unified_file_id,
file_object=file_object,
@ -1128,6 +1169,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
)
_, batch_provider, _, _ = get_llm_provider(model=model_name)
except Exception:
if "/" in model_name:
@ -1199,7 +1241,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Case 1 : This is not a managed file
if not stored_file_object:
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
# Case 2: Managed file and the file object exists in the database
# The stored file_object has the raw provider ID. Replace with the unified ID
# so callers see a consistent ID (matching Case 3 which does response.id = file_id).
@ -1217,13 +1259,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
try:
model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
credentials = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(file_id=model_file_id, **credentials)
model_id, model_file_id = next(
iter(stored_file_object.model_mappings.items())
)
credentials = (
llm_router.get_deployment_credentials_with_provider(model_id) or {}
)
response = await litellm.afile_retrieve(
file_id=model_file_id, **credentials
)
response.id = file_id # Replace with unified ID
return response
except Exception as e:
raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e
raise Exception(
f"Failed to retrieve file {file_id} from provider: {str(e)}"
) from e
async def afile_list(
self,
@ -1245,19 +1295,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
import litellm.proxy.proxy_server as proxy_server_module
# Check if the scheduler has the batch cost checking job registered
scheduler = getattr(proxy_server_module, 'scheduler', None)
scheduler = getattr(proxy_server_module, "scheduler", None)
if scheduler is None:
return False
# Check if the check_batch_cost_job exists in the scheduler
try:
job = scheduler.get_job('check_batch_cost_job')
job = scheduler.get_job("check_batch_cost_job")
if job is not None:
return True
except Exception:
# Job not found or scheduler doesn't support get_job
pass
return False
except Exception as e:
verbose_logger.warning(
@ -1265,28 +1315,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return False
async def _get_batches_referencing_file(
self, file_id: str
) -> List[Dict[str, Any]]:
async def _get_batches_referencing_file(self, file_id: str) -> List[Dict[str, Any]]:
"""
Find batches that reference this file and still need cost tracking.
Find batches that are in non-terminal state and have not yet been processed by CheckBatchCost.
Args:
file_id: The unified file ID to check
Returns:
List of batch objects referencing this file in non-terminal state
(max 10 for error message display)
"""
# Prepare list of file IDs to check (both unified and provider IDs)
file_ids_to_check = [file_id]
# Get model-specific file IDs for this unified file ID if it's a managed file
try:
model_file_id_mapping = await self.get_model_file_id_mapping(
[file_id], litellm_parent_otel_span=None
)
if model_file_id_mapping and file_id in model_file_id_mapping:
# Add all provider file IDs for this unified file
provider_file_ids = list(model_file_id_mapping[file_id].values())
@ -1296,59 +1344,67 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Could not get model file ID mapping for {file_id}: {e}. "
f"Will only check unified file ID."
)
MAX_MATCHES_TO_RETURN = 10
MAX_MATCHES_TO_RETURN = 10
batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"not_in": ["failed", "expired", "cancelled"]}
"status": {"not_in": ["failed", "expired", "cancelled"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
referencing_batches = []
for batch in batches:
try:
# Parse the batch file_object to check for file references
batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
batch_data = (
json.loads(batch.file_object)
if isinstance(batch.file_object, str)
else batch.file_object
)
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")
referenced_file_ids = [fid for fid in [input_file_id, output_file_id, error_file_id] if fid]
referenced_file_ids = [
fid for fid in [input_file_id, output_file_id, error_file_id] if fid
]
# Check if any referenced file ID matches the file we're trying to delete
if any(ref_id in file_ids_to_check for ref_id in referenced_file_ids):
referencing_batches.append({
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
})
referencing_batches.append(
{
"batch_id": batch.unified_object_id,
"status": batch.status,
"created_at": batch.created_at,
}
)
except Exception as e:
verbose_logger.warning(
f"Error parsing batch object {batch.unified_object_id}: {e}"
)
continue
return referencing_batches
async def _check_file_deletion_allowed(self, file_id: str) -> None:
"""
Check if file deletion should be blocked due to batch references.
Blocks deletion if:
1. File is referenced by any batch in non-terminal state, AND
2. Batch polling is configured (user wants cost tracking)
Args:
file_id: The unified file ID to check
Raises:
HTTPException: If file deletion should be blocked
"""
@ -1356,39 +1412,45 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if not self._is_batch_polling_enabled():
# Batch polling not configured, allow deletion
return
# Check if file is referenced by any non-terminal batches
referencing_batches = await self._get_batches_referencing_file(file_id)
if referencing_batches:
# File is referenced by non-terminal batches and polling is enabled
MAX_BATCHES_IN_ERROR = 5 # Limit batches shown in error message for readability
MAX_BATCHES_IN_ERROR = (
5 # Limit batches shown in error message for readability
)
# Show up to MAX_BATCHES_IN_ERROR in the error message
batches_to_show = referencing_batches[:MAX_BATCHES_IN_ERROR]
batch_statuses = [f"{b['batch_id']}: {b['status']}" for b in batches_to_show]
batch_statuses = [
f"{b['batch_id']}: {b['status']}" for b in batches_to_show
]
# Determine the count message
count_message = f"{len(referencing_batches)}"
if len(referencing_batches) >= 10: # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
if (
len(referencing_batches) >= 10
): # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
count_message = "10+"
error_message = (
f"Cannot delete file {file_id}. "
f"The file is referenced by {count_message} batch(es) in non-terminal state"
)
# Add specific batch details if not too many
if len(referencing_batches) <= MAX_BATCHES_IN_ERROR:
error_message += f": {', '.join(batch_statuses)}. "
else:
error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
error_message += (
f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
f"Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)."
)
# Record blocked deletion metric
prom_logger = self._get_prometheus_logger()
if prom_logger:
@ -1419,7 +1481,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
if specific_model_file_id_mapping:
# Remove conflicting keys from data to avoid duplicate keyword arguments
filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
filtered_data = {
k: v for k, v in data.items() if k not in ("model", "file_id")
}
for model_id, model_file_id in specific_model_file_id_mapping.items():
delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data) # type: ignore
@ -1480,7 +1544,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> None:
"""
Convert files stored in storage backends to base64 format for Vertex AI/Gemini.
This method checks if any managed files are stored in storage backends,
downloads them, and converts them to base64 format in the messages.
"""
@ -1488,29 +1552,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
for file_id in file_ids:
# Check if this is a base64 encoded unified file ID
decoded_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if not decoded_unified_file_id:
continue
# Check database for storage backend info
# IMPORTANT: The database stores the base64 encoded unified_file_id (not the decoded version)
# So we query with the original file_id (which is base64 encoded)
db_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
where={"unified_file_id": file_id}
)
if not db_file or not db_file.storage_backend or not db_file.storage_url:
continue
# File is stored in a storage backend, download and convert to base64
try:
from litellm.llms.base_llm.files.storage_backend_factory import (
get_storage_backend,
)
storage_backend_name = db_file.storage_backend
storage_url = db_file.storage_url
# Get storage backend (uses same env vars as callback)
try:
storage_backend = get_storage_backend(storage_backend_name)
@ -1519,18 +1583,22 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}"
)
continue
file_content = await storage_backend.download_file(storage_url)
# Determine content type from file object
content_type = self._get_content_type_from_file_object(db_file.file_object)
content_type = self._get_content_type_from_file_object(
db_file.file_object
)
# Convert to base64
base64_data = base64.b64encode(file_content).decode("utf-8")
base64_data_uri = f"data:{content_type};base64,{base64_data}"
# Update messages to use base64 instead of file_id
self._update_messages_with_base64_data(messages, file_id, base64_data_uri, content_type)
self._update_messages_with_base64_data(
messages, file_id, base64_data_uri, content_type
)
except Exception as e:
verbose_logger.exception(
f"Error converting file {file_id} from storage backend to base64: {str(e)}"
@ -1541,21 +1609,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
"""
Determine content type from file object.
Uses the MIME type utility for consistent detection and normalization.
Args:
file_object: The file object from the database (can be dict, JSON string, or None)
Returns:
str: MIME type (defaults to "application/octet-stream" if cannot be determined)
"""
# Use utility function for detection
content_type = get_content_type_from_file_object(file_object)
# Normalize for Gemini/Vertex AI (requires image/jpeg, not image/jpg)
content_type = normalize_mime_type_for_provider(content_type, provider="gemini")
return content_type
def _update_messages_with_base64_data(
@ -1567,7 +1635,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
) -> None:
"""
Update messages to replace file_id with base64 data URI.
Args:
messages: List of messages to update
file_id: The file ID to replace
@ -1582,7 +1650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if element.get("type") == "file":
file_element = cast(ChatCompletionFileObject, element)
file_element_file = file_element.get("file", {})
if file_element_file.get("file_id") == file_id:
# Replace file_id with base64 data
file_element_file["file_data"] = base64_data_uri
@ -1590,7 +1658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_element_file["format"] = content_type
# Remove file_id to ensure only file_data is used
file_element_file.pop("file_id", None)
verbose_logger.debug(
f"Converted file {file_id} from storage backend to base64 with format {content_type}"
)

6
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.39"
version = "0.1.40"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
Documentation = "https://docs.litellm.ai"
[build-system]
requires = ["uv_build==0.10.7"]
requires = ["uv_build==0.11.8"]
build-backend = "uv_build"
[tool.uv]
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.39"
version = "0.1.40"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

2054
litellm-js/proxy/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

6
litellm-js/proxy/package.json
View file

@ -4,11 +4,11 @@
"deploy": "wrangler deploy --minify src/index.ts"
},
"dependencies": {
"hono": "4.12.12",
"hono": "4.12.16",
"openai": "4.29.2"
},
"devDependencies": {
"@cloudflare/workers-types": "4.20240208.0",
"wrangler": "3.32.0"
"@cloudflare/workers-types": "4.20260501.1",
"wrangler": "4.87.0"
}
}

14
litellm-js/spend-logs/package-lock.json generated
View file

@ -6,7 +6,7 @@
"": {
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.12"
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",
@ -535,9 +535,9 @@
}
},
"node_modules/get-tsconfig": {
"version": "4.13.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
"integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
"version": "4.14.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
"integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -548,9 +548,9 @@
}
},
"node_modules/hono": {
"version": "4.12.12",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz",
"integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==",
"version": "4.12.16",
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
"license": "MIT",
"engines": {
"node": ">=16.9.0"

2
litellm-js/spend-logs/package.json
View file

@ -4,7 +4,7 @@
},
"dependencies": {
"@hono/node-server": "1.19.13",
"hono": "4.12.12"
"hono": "4.12.16"
},
"devDependencies": {
"@types/node": "20.19.25",

20
litellm-proxy-extras/litellm_proxy_extras/migrations/20260501195714_managed_resource_team_owner/migration.sql Normal file
View file

@ -0,0 +1,20 @@
-- Adds `team_id` to managed-resource tables so service-account API
-- keys (no `user_id`) can be scoped by team instead of bypassing the
-- `created_by` filter entirely. Existing rows keep `team_id = NULL`
-- and become invisible to team-only callers — that is the intended isolation
-- outcome; backfill manually if legacy rows must remain visible.
--
-- The composite indexes match the listing query: filter by team owner, sort by
-- created_at DESC. Tables are typically small (resources per tenant, not per
-- request); a future operator with a large table can switch to
-- CREATE INDEX CONCURRENTLY in a follow-up migration.
ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
ALTER TABLE "LiteLLM_ManagedVectorStoreTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
-- Index names follow Prisma's auto-generated convention so `prisma migrate diff`
-- against the schema is clean.
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_team_id_created_at_idx" ON "LiteLLM_ManagedFileTable" ("team_id", "created_at" DESC);
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedObjectTable_team_id_created_at_idx" ON "LiteLLM_ManagedObjectTable" ("team_id", "created_at" DESC);
CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable_team_id_created_at_idx" ON "LiteLLM_ManagedVectorStoreTable" ("team_id", "created_at" DESC);

16
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -884,28 +884,32 @@ model LiteLLM_ManagedFileTable {
storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
storage_url String? // The actual storage URL where the file is stored
created_at DateTime @default(now())
created_by String?
created_by String?
team_id String? // Team that owns the resource; populated for service-account keys without a user_id so listings can isolate by team.
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_file_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
id String @id @default(uuid())
unified_object_id String @unique // The base64 encoded unified file ID
model_object_id String @unique // the id returned by the backend API provider
model_object_id String @unique // the id returned by the backend API provider
file_object Json // Stores the OpenAIFileObject
file_purpose String // either 'batch' or 'fine-tune'
status String? // check if batch cost has been tracked
status String? // check if batch cost has been tracked
batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
created_at DateTime @default(now())
created_by String?
team_id String?
updated_at DateTime @updatedAt
updated_by String?
updated_by String?
@@index([unified_object_id])
@@index([model_object_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedVectorStoreTable {
@ -918,10 +922,12 @@ model LiteLLM_ManagedVectorStoreTable {
storage_url String? // Storage URL (if applicable)
created_at DateTime @default(now())
created_by String?
team_id String?
updated_at DateTime @updatedAt
updated_by String?
@@index([unified_resource_id])
@@index([team_id, created_at(sort: Desc)])
}
model LiteLLM_ManagedVectorStoresTable {

6
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.70"
version = "0.4.71"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
Documentation = "https://docs.litellm.ai"
[build-system]
requires = ["uv_build==0.10.7"]
requires = ["uv_build==0.11.8"]
build-backend = "uv_build"
[tool.uv]
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.70"
version = "0.4.71"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

12
litellm/__init__.py
View file

@ -388,6 +388,7 @@ anthropic_beta_headers_url: str = os.getenv(
suppress_debug_info = False
dynamodb_table_name: Optional[str] = None
s3_callback_params: Optional[Dict] = None
s3_audit_callback_params: Optional[Dict] = None
datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
aws_sqs_callback_params: Optional[Dict] = None
@ -414,6 +415,9 @@ custom_prometheus_metadata_labels: List[str] = []
custom_prometheus_tags: List[str] = []
prometheus_metrics_config: Optional[List] = None
prometheus_emit_stream_label: bool = False
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
@ -586,6 +590,7 @@ anyscale_models: Set = set()
cerebras_models: Set = set()
galadriel_models: Set = set()
nvidia_nim_models: Set = set()
nvidia_riva_models: Set = set()
sambanova_models: Set = set()
sambanova_embedding_models: Set = set()
novita_models: Set = set()
@ -812,6 +817,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
galadriel_models.add(key)
elif value.get("litellm_provider") == "nvidia_nim":
nvidia_nim_models.add(key)
elif value.get("litellm_provider") == "nvidia_riva":
nvidia_riva_models.add(key)
elif value.get("litellm_provider") == "sambanova":
sambanova_models.add(key)
elif value.get("litellm_provider") == "sambanova-embedding-models":
@ -971,6 +978,7 @@ model_list = list(
| cerebras_models
| galadriel_models
| nvidia_nim_models
| nvidia_riva_models
| sambanova_models
| azure_text_models
| novita_models
@ -1067,6 +1075,7 @@ models_by_provider: dict = {
"cerebras": cerebras_models,
"galadriel": galadriel_models,
"nvidia_nim": nvidia_nim_models,
"nvidia_riva": nvidia_riva_models,
"sambanova": sambanova_models | sambanova_embedding_models,
"novita": novita_models,
"nebius": nebius_models | nebius_embedding_models,
@ -1618,6 +1627,9 @@ if TYPE_CHECKING:
from .llms.deepgram.audio_transcription.transformation import (
DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
)
from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
)
from .llms.topaz.image_variations.transformation import (
TopazImageVariationConfig as TopazImageVariationConfig,
)

11
litellm/constants.py
View file

@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
| (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
)
# MCP OAuth2 Token Exchange (OBO) Defaults
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
@ -1457,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
)
SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
)
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(

2
litellm/experimental_mcp_client/client.py
View file

@ -366,6 +366,8 @@ class MCPClient:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.token:
headers["Authorization"] = f"token {self._mcp_auth_value}"
elif self.auth_type == MCPAuth.oauth2_token_exchange:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
elif isinstance(self._mcp_auth_value, dict):
headers.update(self._mcp_auth_value)
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

11
litellm/integrations/arize/arize_phoenix_prompt_manager.py
View file

@ -5,7 +5,8 @@ Fetches prompt versions from Arize Phoenix and provides workspace-based access c
from typing import Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import (
@ -74,7 +75,13 @@ class ArizePhoenixTemplateManager:
api_key=self.api_key, api_base=self.api_base
)
self.jinja_env = Environment(
# Templates fetched from Arize Phoenix come from external workspace
# users; in a plain `Environment()` a malicious template could reach
# `__class__.__init__.__globals__` and execute arbitrary code on the
# proxy host. The sandbox blocks that attribute traversal while
# leaving normal `{{ var }}` substitution intact. Matches the
# dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
# Use Mustache/Handlebars-style delimiters

139
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
import asyncio
import os
import time
import traceback
from typing import List, Optional
from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.types.utils import StandardLoggingPayload
from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
class AzureSentinelLogger(CustomBatchLogger):
@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
tenant_id: Optional[str] = None,
client_id: Optional[str] = None,
client_secret: Optional[str] = None,
audit_stream_name: Optional[str] = None,
**kwargs,
):
"""
@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
audit_stream_name (str, optional): Stream name from DCR for audit logs.
If not provided, audit logs use the standard stream name.
"""
self.async_httpx_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.dcr_immutable_id = dcr_immutable_id or os.getenv(
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID"
)
self.stream_name = stream_name or os.getenv(
"AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
resolved_stream_name = (
stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
)
self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
self.tenant_id = (
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
resolved_tenant_id = (
tenant_id
or os.getenv("AZURE_SENTINEL_TENANT_ID")
or os.getenv("AZURE_TENANT_ID")
)
self.client_id = (
resolved_client_id = (
client_id
or os.getenv("AZURE_SENTINEL_CLIENT_ID")
or os.getenv("AZURE_CLIENT_ID")
)
self.client_secret = (
resolved_client_secret = (
client_secret
or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
or os.getenv("AZURE_CLIENT_SECRET")
)
if not self.dcr_immutable_id:
if not resolved_dcr_immutable_id:
raise ValueError(
"AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
)
if not self.endpoint:
if not resolved_endpoint:
raise ValueError(
"AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
)
if not self.tenant_id:
if not resolved_tenant_id:
raise ValueError(
"AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
)
if not self.client_id:
if not resolved_client_id:
raise ValueError(
"AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
)
if not self.client_secret:
if not resolved_client_secret:
raise ValueError(
"AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
)
self.dcr_immutable_id = resolved_dcr_immutable_id
self.stream_name = resolved_stream_name
self.audit_stream_name = resolved_audit_stream_name
self.endpoint = resolved_endpoint
self.tenant_id = resolved_tenant_id
self.client_id = resolved_client_id
self.client_secret = resolved_client_secret
# Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
self.api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_stream_name,
)
self.audit_api_endpoint = self._build_api_endpoint(
endpoint=resolved_endpoint,
dcr_immutable_id=resolved_dcr_immutable_id,
stream_name=resolved_audit_stream_name,
)
# OAuth2 scope for Azure Monitor
self.oauth_scope = "https://monitor.azure.com/.default"
@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
super().__init__(**kwargs, flush_lock=self.flush_lock)
asyncio.create_task(self.periodic_flush())
self.log_queue: List[StandardLoggingPayload] = []
self.audit_log_queue: List[StandardAuditLogPayload] = []
@staticmethod
def _build_api_endpoint(
endpoint: str, dcr_immutable_id: str, stream_name: str
) -> str:
return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
async def _get_oauth_token(self) -> str:
"""
@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
Returns:
Bearer token string
"""
# Check if we have a valid cached token
import time
if (
self.oauth_token
and self.oauth_token_expires_at
@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
if not self.oauth_token:
raise Exception("OAuth2 token response did not contain access_token")
# Cache token expiry time
import time
self.oauth_token_expires_at = time.time() + expires_in
return self.oauth_token
@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
)
pass
async def async_log_audit_log_event(
self, audit_log: StandardAuditLogPayload
) -> None:
"""
Async log LiteLLM audit log events to Azure Sentinel.
Audit logs are queued separately from standard LLM logs so mixed callback
usage never sends schema-mismatched records in the same ingestion batch.
"""
try:
verbose_logger.debug(
"Azure Sentinel: Logging audit event id=%s action=%s table=%s",
audit_log.get("id"),
audit_log.get("action"),
audit_log.get("table_name"),
)
self.audit_log_queue.append(audit_log)
if len(self.audit_log_queue) >= self.batch_size:
await self.async_send_audit_batch()
except Exception as e:
verbose_logger.exception(
f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
)
pass
async def async_send_batch(self):
"""
Sends the batch of logs to Azure Monitor Logs Ingestion API
@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
Raises:
Raises a NON Blocking verbose_logger.exception if an error occurs
"""
await self._async_send_batch_to_api(
log_queue=self.log_queue,
api_endpoint=self.api_endpoint,
log_type="logs",
)
async def async_send_audit_batch(self):
"""
Sends the batch of audit logs to Azure Monitor Logs Ingestion API
"""
await self._async_send_batch_to_api(
log_queue=self.audit_log_queue,
api_endpoint=self.audit_api_endpoint,
log_type="audit logs",
)
async def _async_send_batch_to_api(
self,
log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
api_endpoint: str,
log_type: str,
) -> None:
try:
if not self.log_queue:
if not log_queue:
return
verbose_logger.debug(
"Azure Sentinel - about to flush %s events", len(self.log_queue)
"Azure Sentinel - about to flush %s %s", len(log_queue), log_type
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
# Get OAuth2 token
bearer_token = await self._get_oauth_token()
# Convert log queue to JSON array format expected by Logs Ingestion API
# Each log entry should be a JSON object in the array
body = safe_dumps(self.log_queue)
body = safe_dumps(log_queue)
# Set headers for Logs Ingestion API
headers = {
@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
# Send the request
response = await self.async_httpx_client.post(
url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
url=api_endpoint, data=body.encode("utf-8"), headers=headers
)
if response.status_code not in [200, 204]:
@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
)
finally:
self.log_queue.clear()
log_queue.clear()
async def flush_queue(self):
if self.flush_lock is None:
return
async with self.flush_lock:
if self.log_queue:
await self.async_send_batch()
if self.audit_log_queue:
await self.async_send_audit_batch()
self.last_flush_time = time.time()

11
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -5,7 +5,8 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -74,7 +75,13 @@ class BitBucketTemplateManager:
self.prompts: Dict[str, BitBucketPromptTemplate] = {}
self.bitbucket_client = BitBucketClient(bitbucket_config)
self.jinja_env = Environment(
# Templates fetched from a BitBucket repo are not trustworthy:
# anyone with repo write access can ship Jinja syntax that, in a
# plain `Environment()`, would reach `__class__.__init__.__globals__`
# and pivot into RCE on the proxy host. The sandbox blocks that
# attribute traversal while leaving normal `{{ var }}` substitution
# intact. Matches the dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
# Use Handlebars-style delimiters to match Dotprompt spec

11
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -4,7 +4,8 @@ GitLab prompt manager with configurable prompts folder.
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from jinja2 import DictLoader, Environment, select_autoescape
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
@ -90,7 +91,13 @@ class GitLabTemplateManager:
or ""
).strip("/")
self.jinja_env = Environment(
# Templates fetched from a GitLab repo are not trustworthy:
# anyone with repo write access can ship Jinja syntax that, in a
# plain `Environment()`, would reach `__class__.__init__.__globals__`
# and pivot into RCE on the proxy host. The sandbox blocks that
# attribute traversal while leaving normal `{{ var }}` substitution
# intact. Matches the dotprompt manager's hardening.
self.jinja_env = ImmutableSandboxedEnvironment(
loader=DictLoader({}),
autoescape=select_autoescape(["html", "xml"]),
variable_start_string="{{",

190
litellm/integrations/prometheus.py
View file

@ -25,6 +25,9 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
BoundedPrometheusSeriesTracker,
)
from litellm.integrations.prometheus_helpers import (
PrometheusLabelFactoryContext,
_get_cached_end_user_id_for_cost_tracking,
@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger):
if _custom_buckets is not None
else LATENCY_BUCKETS
)
self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
# Create metric factory functions
self._counter_factory = self._create_metric_factory(Counter)
@ -984,6 +988,40 @@ class PrometheusLogger(CustomLogger):
return filtered_labels
def _track_end_user_metric_series(
self,
metric: Any,
metric_name: DEFINED_PROMETHEUS_METRICS,
labels: Dict[str, Optional[str]],
) -> None:
"""
Cap the cardinality of metrics that include the ``end_user`` label.
Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
recorded in prometheus-client's child map before any eviction runs.
Series that get evicted before the next scrape lose updates accrued
since the last scrape — this is inherent to any cardinality cap.
"""
labelnames = self.get_labels_for_metric(metric_name)
if UserAPIKeyLabelNames.END_USER.value not in labelnames:
return
if labels.get(UserAPIKeyLabelNames.END_USER.value) is None:
return
max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
if max_series is None and ttl_seconds is None:
return
self._bounded_prometheus_series_tracker.track_series(
metric=metric,
metric_name=metric_name,
label_values=tuple(labels.get(label) for label in labelnames),
max_series=max_series,
ttl_seconds=ttl_seconds,
cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds,
)
def _inc_labeled_counter(
self,
counter: Any,
@ -998,6 +1036,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
counter.labels(**_labels).inc(amount)
self._track_end_user_metric_series(counter, metric_name, _labels)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
# Define prometheus client
@ -1047,21 +1086,9 @@ class PrometheusLogger(CustomLogger):
output_tokens = standard_logging_payload["completion_tokens"]
tokens_used = standard_logging_payload["total_tokens"]
response_cost = standard_logging_payload["response_cost"]
_requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"requester_metadata"
combined_metadata = _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
"metadata"
].get("user_api_key_auth_metadata")
spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
"spend_logs_metadata"
)
combined_metadata: Dict[str, Any] = {
**(_requester_metadata if _requester_metadata else {}),
**(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
**(spend_logs_metadata if spend_logs_metadata else {}),
}
if standard_logging_payload is not None and isinstance(
standard_logging_payload, dict
):
@ -1416,26 +1443,46 @@ class PrometheusLogger(CustomLogger):
)
remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
remaining_requests = (
metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
remaining_requests = metadata.get(remaining_requests_variable_name)
if remaining_requests is None:
remaining_requests = sys.maxsize
remaining_tokens = metadata.get(remaining_tokens_variable_name)
if remaining_tokens is None:
remaining_tokens = sys.maxsize
enum_values = UserAPIKeyLabelValues(
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model_group,
model_id=model_id,
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=kwargs.get("standard_logging_object")
)
),
)
remaining_tokens = (
metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
label_context = PrometheusLabelFactoryContext(enum_values)
requests_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_requests_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_requests_for_model.labels(**requests_labels).set(
remaining_requests
)
self.litellm_remaining_api_key_requests_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_requests)
self.litellm_remaining_api_key_tokens_for_model.labels(
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model_group),
_sanitize_prometheus_label_value(model_id),
).set(remaining_tokens)
tokens_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
"litellm_remaining_api_key_tokens_for_model"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set(
remaining_tokens
)
def _set_latency_metrics(
self,
@ -1471,6 +1518,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_time_to_first_token_metric.labels(
**_ttft_labels
).observe(time_to_first_token_seconds)
self._track_end_user_metric_series(
self.litellm_llm_api_time_to_first_token_metric,
"litellm_llm_api_time_to_first_token_metric",
_ttft_labels,
)
else:
verbose_logger.debug(
"Time to first token metric not emitted, stream option in model_parameters is not True"
@ -1491,6 +1543,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_llm_api_latency_metric.labels(**_labels).observe(
api_call_total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_llm_api_latency_metric,
"litellm_llm_api_latency_metric",
_labels,
)
# total request latency
total_time_seconds = self._safe_duration_seconds(
@ -1508,6 +1565,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_total_latency_metric.labels(**_labels).observe(
total_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params = kwargs.get("litellm_params", {}) or {}
@ -1525,6 +1587,11 @@ class PrometheusLogger(CustomLogger):
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
self._track_end_user_metric_series(
self.litellm_request_queue_time_metric,
"litellm_request_queue_time_seconds",
_labels,
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
verbose_logger.debug(
@ -1561,18 +1628,27 @@ class PrometheusLogger(CustomLogger):
)
try:
self.litellm_llm_api_failed_requests_metric.labels(
_sanitize_prometheus_label_value(end_user_id),
_sanitize_prometheus_label_value(user_api_key),
_sanitize_prometheus_label_value(user_api_key_alias),
_sanitize_prometheus_label_value(model),
_sanitize_prometheus_label_value(user_api_team),
_sanitize_prometheus_label_value(user_api_team_alias),
_sanitize_prometheus_label_value(user_id),
_sanitize_prometheus_label_value(
standard_logging_payload.get("model_id", "")
enum_values = UserAPIKeyLabelValues(
end_user=end_user_id,
hashed_api_key=user_api_key,
api_key_alias=user_api_key_alias,
model=model,
team=user_api_team,
team_alias=user_api_team_alias,
user=user_id,
model_id=standard_logging_payload.get("model_id", ""),
custom_metadata_labels=get_custom_labels_from_metadata(
metadata=_get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload=standard_logging_payload
)
),
).inc()
)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_llm_api_failed_requests_metric,
"litellm_llm_api_failed_requests_metric",
enum_values,
)
self.set_llm_deployment_failure_metrics(kwargs)
await self._set_org_budget_metrics_after_api_request(
org_id=user_api_key_org_id,
@ -3622,6 +3698,36 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
return result
def _get_combined_custom_metadata_from_standard_logging_payload(
standard_logging_payload: Optional[dict],
) -> Dict[str, Any]:
"""
Combine the metadata sources that can supply custom Prometheus labels.
"""
if not isinstance(standard_logging_payload, dict):
return {}
standard_logging_metadata = standard_logging_payload.get("metadata") or {}
if not isinstance(standard_logging_metadata, dict):
return {}
requester_metadata = standard_logging_metadata.get("requester_metadata")
user_api_key_auth_metadata = standard_logging_metadata.get(
"user_api_key_auth_metadata"
)
spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata")
return {
**(requester_metadata if isinstance(requester_metadata, dict) else {}),
**(
user_api_key_auth_metadata
if isinstance(user_api_key_auth_metadata, dict)
else {}
),
**(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
}
def _tag_matches_wildcard_configured_pattern(
tags: Sequence[str], configured_tag: str
) -> bool:

107
litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py Normal file
View file

@ -0,0 +1,107 @@
from __future__ import annotations
import time
from collections import OrderedDict
from threading import RLock
from typing import Any, Dict, Optional
class BoundedPrometheusSeriesTracker:
"""
Tracks Prometheus child series and removes stale/excess labelsets.
The tracker is label-agnostic: callers decide which series should be tracked
and pass the full label tuple used by the Prometheus metric.
"""
def __init__(self) -> None:
self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
self._last_ttl_cleanup: Dict[str, float] = {}
self.lock = RLock()
def track_series(
self,
metric: Any,
metric_name: str,
label_values: tuple[Optional[str], ...],
max_series: Optional[int],
ttl_seconds: Optional[float],
cleanup_interval_seconds: Optional[float],
) -> None:
if max_series is None and ttl_seconds is None:
return
now = time.monotonic()
with self.lock:
series = self._series.setdefault(metric_name, OrderedDict())
series[label_values] = now
series.move_to_end(label_values)
if ttl_seconds is not None and self._should_run_ttl_cleanup(
metric_name=metric_name,
now=now,
cleanup_interval_seconds=cleanup_interval_seconds,
):
expired_label_values = [
tracked_label_values
for tracked_label_values, last_seen in series.items()
if now - last_seen > ttl_seconds
]
for tracked_label_values in expired_label_values:
self._remove_metric_series(metric, series, tracked_label_values)
# max_series <= 0 is treated as "unlimited" so a misconfigured zero
# value cannot silently drop every emission for this metric.
if max_series is not None and max_series > 0:
while len(series) > max_series:
tracked_label_values = next(iter(series))
if not self._remove_metric_child(metric, tracked_label_values):
break
del series[tracked_label_values]
def _should_run_ttl_cleanup(
self,
metric_name: str,
now: float,
cleanup_interval_seconds: Optional[float],
) -> bool:
if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0:
self._last_ttl_cleanup[metric_name] = now
return True
last_cleanup = self._last_ttl_cleanup.get(metric_name)
if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds:
self._last_ttl_cleanup[metric_name] = now
return True
return False
def _remove_metric_series(
self,
metric: Any,
series: OrderedDict[tuple[Optional[str], ...], float],
label_values: tuple[Optional[str], ...],
) -> None:
if self._remove_metric_child(metric, label_values):
series.pop(label_values, None)
@staticmethod
def _remove_metric_child(
metric: Any, label_values: tuple[Optional[str], ...]
) -> bool:
"""
Remove the Prometheus child for ``label_values`` and report whether the
tracker should commit the matching state change.
Returns ``True`` when the child is no longer present in Prometheus
(either it was just removed or it was already gone), and ``False`` when
``metric.remove()`` raised an unexpected error and the child likely
still exists.
"""
try:
metric.remove(*label_values)
return True
except KeyError:
return True
except (AttributeError, ValueError):
return False

100
litellm/integrations/s3_v2.py
View file

@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
from litellm.integrations.s3 import get_s3_object_key
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
s3_callback_params_override: Optional[dict] = None,
**kwargs,
):
try:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
)
_masker = SensitiveDataMasker()
if s3_callback_params_override is not None:
verbose_logger.debug(
f"in init s3 logger (audit override) - "
f"{_masker.mask_dict(dict(s3_callback_params_override))}"
)
else:
verbose_logger.debug(
f"in init s3 logger - s3_callback_params "
f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
)
# Initialize S3 params first to get the correct s3_verify value
self._init_s3_params(
params_source=s3_callback_params_override,
s3_bucket_name=s3_bucket_name,
s3_region_name=s3_region_name,
s3_api_version=s3_api_version,
@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
s3_strip_base64_files: bool = False,
s3_use_key_prefix: bool = False,
s3_use_virtual_hosted_style: bool = False,
params_source: Optional[dict] = None,
):
"""
Initialize the s3 params for this logging callback
Initialize the s3 params for this logging callback. Reads from
`params_source` if given (e.g. `s3_audit_callback_params` for the
audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
Resolves `os.environ/X` markers into a local dict; never mutates the source.
"""
litellm.s3_callback_params = litellm.s3_callback_params or {}
# read in .env variables - example os.environ/AWS_BUCKET_NAME
for key, value in litellm.s3_callback_params.items():
if isinstance(value, str) and value.startswith("os.environ/"):
litellm.s3_callback_params[key] = litellm.get_secret(value)
if params_source is None:
params_source = litellm.s3_callback_params or {}
params: dict = {
key: (
litellm.get_secret(value)
if isinstance(value, str) and value.startswith("os.environ/")
else value
)
for key, value in params_source.items()
}
self.s3_bucket_name = (
litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
)
self.s3_region_name = (
litellm.s3_callback_params.get("s3_region_name") or s3_region_name
)
self.s3_api_version = (
litellm.s3_callback_params.get("s3_api_version") or s3_api_version
)
self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
self.s3_region_name = params.get("s3_region_name") or s3_region_name
self.s3_api_version = params.get("s3_api_version") or s3_api_version
self.s3_use_ssl = (
litellm.s3_callback_params.get("s3_use_ssl", True)
if litellm.s3_callback_params.get("s3_use_ssl") is not None
params.get("s3_use_ssl", True)
if params.get("s3_use_ssl") is not None
else s3_use_ssl
)
self.s3_verify = (
litellm.s3_callback_params.get("s3_verify")
if litellm.s3_callback_params.get("s3_verify") is not None
params.get("s3_verify")
if params.get("s3_verify") is not None
else s3_verify
)
self.s3_endpoint_url = (
litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
)
self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
self.s3_aws_access_key_id = (
litellm.s3_callback_params.get("s3_aws_access_key_id")
or s3_aws_access_key_id
params.get("s3_aws_access_key_id") or s3_aws_access_key_id
)
self.s3_aws_secret_access_key = (
litellm.s3_callback_params.get("s3_aws_secret_access_key")
or s3_aws_secret_access_key
params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
)
self.s3_aws_session_token = (
litellm.s3_callback_params.get("s3_aws_session_token")
or s3_aws_session_token
params.get("s3_aws_session_token") or s3_aws_session_token
)
self.s3_aws_session_name = (
litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
params.get("s3_aws_session_name") or s3_aws_session_name
)
self.s3_aws_profile_name = (
litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
params.get("s3_aws_profile_name") or s3_aws_profile_name
)
self.s3_aws_role_name = (
litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
)
self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
self.s3_aws_web_identity_token = (
litellm.s3_callback_params.get("s3_aws_web_identity_token")
or s3_aws_web_identity_token
params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
)
self.s3_aws_sts_endpoint = (
litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
)
self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
# done reading litellm.s3_callback_params
self.s3_config = params.get("s3_config") or s3_config
self.s3_path = params.get("s3_path") or s3_path
self.s3_use_team_prefix = (
bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
or s3_use_team_prefix
bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
)
self.s3_use_key_prefix = (
bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
or s3_use_key_prefix
bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
)
self.s3_strip_base64_files = (
bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
or s3_strip_base64_files
bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
)
self.s3_use_virtual_hosted_style = (
bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
bool(params.get("s3_use_virtual_hosted_style", False))
or s3_use_virtual_hosted_style
)

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal
from litellm.integrations.agentops import AgentOps
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.argilla import ArgillaLogger
from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
@ -73,6 +74,7 @@ class CustomLoggerRegistry:
"opik": OpikLogger,
"argilla": ArgillaLogger,
"opentelemetry": OpenTelemetry,
"azure_sentinel": AzureSentinelLogger,
"azure_storage": AzureBlobStorageLogger,
"humanloop": HumanloopLogger,
# OTEL compatible loggers

12
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -621,6 +621,18 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
or "https://integrate.api.nvidia.com/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
elif custom_llm_provider == "nvidia_riva":
# NVIDIA Riva is gRPC-based; api_base must be a host:port like
# `grpc.nvcf.nvidia.com:443` or `localhost:50051`. There is no
# public-default endpoint, so we do not fill one in here.
api_base = api_base or get_secret_str("NVIDIA_RIVA_API_BASE") # type: ignore
# Fall back to NVIDIA_NIM_API_KEY because users running both NVCF
# services typically reuse the same nvapi-* key.
dynamic_api_key = (
api_key
or get_secret_str("NVIDIA_RIVA_API_KEY")
or get_secret_str("NVIDIA_NIM_API_KEY")
)
elif custom_llm_provider == "cerebras":
api_base = (
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"

35
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
"""
Updates messages with model file ids.
For managed files (unified file IDs), uses model_file_id_mapping if it
resolves the id, otherwise decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly. Mirrors the Responses-API
sibling `update_responses_input_with_model_file_ids`.
model_file_id_mapping: Dict[str, Dict[str, str]] = {
"litellm_proxy/file_id": {
"model_id": "provider_file_id"
}
}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
)
for message in messages:
if message.get("role") == "user":
@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
if not isinstance(c, dict):
# Content list items aren't always dicts. e.g.
# text_completion forwards a token-ids list/list-of-
# lists through this path. Skip non-dict items
# instead of indexing into them.
continue
if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):
@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
if file_id:
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
or file_id
if model_file_id_mapping
else None
)
if (
not provider_file_id
and _is_base64_encoded_unified_file_id(file_id)
):
unified_file_id = convert_b64_uid_to_unified_uid(
file_id
)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
file_object_file_field["file_id"] = (
provider_file_id or file_id
)
file_object_file_field["file_id"] = provider_file_id
if format:
file_object_file_field["format"] = format
return messages

69
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -2124,27 +2124,62 @@ def anthropic_process_openai_file_message(
)
_EMPTY_TEXT_PLACEHOLDER = (
"[System: Empty message content sanitised to satisfy protocol]"
)
def _sanitize_empty_text_content(
message: AllMessageValues,
) -> AllMessageValues:
"""
Case C: Sanitize empty text content
- Replace empty or whitespace-only text content with a placeholder message.
- Handles both string content and list-of-blocks content (rewriting only
the empty text blocks in place; non-text blocks like images are left
untouched).
Returns:
The message with sanitized content if needed, otherwise the original message
"""
if message.get("role") in ["user", "assistant"]:
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = (
"[System: Empty message content sanitised to satisfy protocol]"
)
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
if message.get("role") not in ["user", "assistant"]:
return message
content = message.get("content")
if isinstance(content, str):
if not content or not content.strip():
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = _EMPTY_TEXT_PLACEHOLDER
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
)
return message
if isinstance(content, list):
# Walk the blocks and rewrite any empty text blocks. We rewrite (rather
# than drop) so callers don't end up with an entirely empty content
# list, which Anthropic also rejects.
new_blocks: List[Any] = []
rewrote_any = False
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text")
if not isinstance(text, str) or not text or not text.strip():
new_block = dict(block)
new_block["text"] = _EMPTY_TEXT_PLACEHOLDER
new_blocks.append(new_block)
rewrote_any = True
continue
new_blocks.append(block)
if rewrote_any:
message = cast(AllMessageValues, dict(message)) # Make a copy
message["content"] = new_blocks # type: ignore
verbose_logger.debug(
f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message"
)
return message
@ -2427,6 +2462,18 @@ def anthropic_messages_pt( # noqa: PLR0915
# Sanitize messages for tool calling issues when modify_params=True
messages = sanitize_messages_for_tool_calling(messages)
# Anthropic rejects empty text content blocks with:
# "messages: text content blocks must be non-empty"
# OpenAI/other providers silently tolerate `{"role": "user", "content": ""}`,
# so callers (and upstream agent frameworks like pydantic-ai) routinely
# send empty user/assistant turns. We always rewrite these to a placeholder
# for Anthropic-shaped requests, independent of `litellm.modify_params`,
# because there is no way to "pass through" an empty text block — the
# request will always 400 otherwise. The richer tool-call sanitization
# (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on
# `modify_params` because it actually mutates conversation structure.
messages = [_sanitize_empty_text_content(m) for m in messages]
# add role=tool support to allow function call result/error submission
user_message_types = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.

308
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -31,6 +31,8 @@ DefaultLoggedRealTimeEventTypes = [
"session.created",
"response.create",
"response.done",
"conversation.item.added", # GA
"conversation.item.done", # GA
]
@ -44,6 +46,7 @@ class RealTimeStreaming:
model: str = "",
user_api_key_dict: Optional[Any] = None,
request_data: Optional[Dict] = None,
backend_uses_beta_protocol: Optional[bool] = None,
):
self.websocket = websocket
self.backend_ws = backend_ws
@ -54,6 +57,14 @@ class RealTimeStreaming:
self.session_tools: List[Dict] = []
self.tool_calls: List[Dict] = []
# Detect whether the client is explicitly opting into the beta protocol.
self._client_wants_beta = self._detect_beta_header(websocket)
self._backend_uses_beta_protocol = (
self._client_wants_beta
if backend_uses_beta_protocol is None
else backend_uses_beta_protocol
)
_logged_real_time_event_types = litellm.logged_real_time_event_types
if _logged_real_time_event_types is None:
@ -76,6 +87,27 @@ class RealTimeStreaming:
# response.create can be rewritten to include the failure context.
self._pending_guardrail_message: Optional[str] = None
_SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
_AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = {
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
"g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
}
# GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
_GA_TO_BETA_EVENT_TYPES: Dict[str, str] = {
"conversation.item.added": "conversation.item.created",
"response.output_text.delta": "response.text.delta",
"response.output_audio.delta": "response.audio.delta",
"response.output_audio_transcript.delta": "response.audio_transcript.delta",
"response.output_text.done": "response.text.done",
"response.output_audio.done": "response.audio.done",
"response.output_audio_transcript.done": "response.audio_transcript.done",
}
_GA_TO_BETA_CONTENT_TYPES: Dict[str, str] = {
"output_text": "text",
"output_audio": "audio",
}
def _should_store_message(
self,
message_obj: Union[dict, OpenAIRealtimeEvents],
@ -92,24 +124,27 @@ class RealTimeStreaming:
if isinstance(message, bytes):
message = message.decode("utf-8")
if isinstance(message, dict):
message_obj = message
# TypedDict union members do not narrow to plain dict for mypy.
message_obj: Dict[str, Any] = cast(Dict[str, Any], message)
else:
message_obj = json.loads(message)
message_obj = cast(Dict[str, Any], json.loads(cast(str, message)))
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
try:
if (
not isinstance(message, dict)
or message_obj.get("type") == "session.created"
or message_obj.get("type") == "session.updated"
):
message_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
elif not isinstance(message, dict):
message_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
event_type = message_obj.get("type", "")
if event_type in self._SESSION_EVENT_TYPES:
typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore
else:
# Use the base object as a safe catch-all for all other event types
# (both beta and GA), so unknown/new event names never raise here.
typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore
except Exception as e:
verbose_logger.debug(f"Error parsing message for logging: {e}")
raise e
if self._should_store_message(message_obj):
self.messages.append(message_obj)
# Don't re-raise — a parse failure must not drop or delay the message
if self._should_store_message(message_obj):
self.messages.append(message_obj) # type: ignore[arg-type]
return
if self._should_store_message(typed_obj):
self.messages.append(typed_obj)
def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None:
"""Extract user text content from client WebSocket events for spend logging."""
@ -147,6 +182,8 @@ class RealTimeStreaming:
tools = session.get("tools")
if tools and isinstance(tools, list):
self.session_tools = tools
# GA: session.type is required; log it for traceability but no action needed
verbose_logger.debug(f"Realtime session.type: {session.get('type')}")
except (json.JSONDecodeError, AttributeError, TypeError):
pass
@ -228,6 +265,23 @@ class RealTimeStreaming:
else:
await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined]
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
if self._backend_uses_beta_protocol:
session: Dict[str, Any] = {
"turn_detection": {"create_response": False},
}
else:
session = {
"type": "realtime",
"audio": {
"input": {
"turn_detection": {"create_response": False},
}
},
}
return json.dumps({"type": "session.update", "session": session})
def _has_realtime_guardrails(self) -> bool:
"""Return True if any callback is registered for realtime guardrail event types."""
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -435,14 +489,7 @@ class RealTimeStreaming:
):
self.store_message(event_str)
await self.websocket.send_text(event_str)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
continue
## GUARDRAIL: run on transcription events in provider_config path too
if (
@ -484,14 +531,7 @@ class RealTimeStreaming:
):
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
await self._send_to_backend(
json.dumps(
{
"type": "session.update",
"session": {"turn_detection": {"create_response": False}},
}
)
)
await self._send_to_backend(self._make_disable_auto_response_message())
return True
if (
@ -542,7 +582,20 @@ class RealTimeStreaming:
continue
## LOGGING
self.store_message(raw_response)
await self.websocket.send_text(raw_response)
# If the client opted into beta protocol, translate GA event
# names/shapes back to the beta equivalents before forwarding.
if self._client_wants_beta:
try:
event_dict = json.loads(raw_response)
translated = self._translate_event_to_beta(event_dict)
if translated is None:
continue # drop GA-only events (e.g. conversation.item.done)
await self.websocket.send_text(json.dumps(translated))
except Exception:
await self.websocket.send_text(raw_response)
else:
await self.websocket.send_text(raw_response)
except websockets.exceptions.ConnectionClosed as e: # type: ignore
verbose_logger.exception(
@ -553,6 +606,183 @@ class RealTimeStreaming:
finally:
await self.log_messages()
@staticmethod
def _detect_beta_header(websocket: Any) -> bool:
"""Return True if the client sent 'OpenAI-Beta: realtime=v1'.
Checks the raw ASGI scope headers so it works for both FastAPI WebSocket
objects and any test doubles that expose a .scope dict.
"""
try:
headers = websocket.scope.get("headers", [])
for name, value in headers:
if isinstance(name, bytes):
name = name.decode("latin-1")
if isinstance(value, bytes):
value = value.decode("latin-1")
if name.lower() == "openai-beta" and "realtime=v1" in value.lower():
return True
except Exception:
pass
return False
@staticmethod
def _remap_beta_session_to_ga(session: dict) -> dict:
"""
Convert a beta-style session.update payload to the GA nested schema.
Beta → GA field mappings
─────────────────────────────────────────────────────────────────────
session.type (inject "realtime" if absent)
session.modalities → session.output_modalities
session.voice → session.audio.output.voice
session.input_audio_format → session.audio.input.format (with type/rate)
session.output_audio_format → session.audio.output.format (with type/rate)
session.turn_detection → session.audio.input.turn_detection
session.input_audio_transcription → session.audio.input.transcription
─────────────────────────────────────────────────────────────────────
Fields not in the mapping (instructions, tools, etc.) are passed through.
GA clients that already use the nested shape are unaffected.
"""
# Work on a shallow copy so we don't mutate the caller's dict
session = dict(session)
# 1. Ensure session.type is present
if "type" not in session:
session["type"] = "realtime"
# 2. Rename modalities → output_modalities and normalise combinations.
# Beta allowed ["audio", "text"] together; GA only supports ["audio"] or
# ["text"] as single-element lists. When both are present we prefer
# ["audio"] because audio mode already delivers transcripts via events.
if "modalities" in session:
mods = session.pop("modalities")
if "output_modalities" not in session:
mods_set = {m.lower() for m in (mods or [])}
if "audio" in mods_set:
session["output_modalities"] = ["audio"]
elif "text" in mods_set:
session["output_modalities"] = ["text"]
# 3-7. Lift flat audio fields into the nested audio object
audio: Dict[str, Any] = {}
inp: Dict[str, Any] = {}
out: Dict[str, Any] = {}
# voice → audio.output.voice
if "voice" in session:
out["voice"] = session.pop("voice")
# input_audio_format → audio.input.format
if "input_audio_format" in session:
raw = session.pop("input_audio_format")
inp["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# output_audio_format → audio.output.format
if "output_audio_format" in session:
raw = session.pop("output_audio_format")
out["format"] = (
RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
if isinstance(raw, str)
else raw
)
# turn_detection → audio.input.turn_detection
if "turn_detection" in session:
inp["turn_detection"] = session.pop("turn_detection")
# input_audio_transcription → audio.input.transcription
if "input_audio_transcription" in session:
inp["transcription"] = session.pop("input_audio_transcription")
if inp:
audio["input"] = inp
if out:
audio["output"] = out
if audio:
# Merge with any existing GA-style `audio` block the client already set,
# letting the remapped values take precedence within each sub-key.
existing = session.get("audio") or {}
for sub_key, sub_val in audio.items():
if (
sub_key in existing
and isinstance(existing[sub_key], dict)
and isinstance(sub_val, dict)
):
existing[sub_key] = {**existing[sub_key], **sub_val}
else:
existing[sub_key] = sub_val
session["audio"] = existing
return session
@staticmethod
def _translate_event_to_beta(event: dict) -> Optional[dict]:
"""Translate a single GA event dict to its beta equivalent.
Returns None if the event should be dropped entirely (e.g. the GA-only
conversation.item.done has no beta counterpart).
Returns the (possibly mutated copy of the) event otherwise.
"""
event_type = event.get("type", "")
# conversation.item.done has no beta equivalent — the client already
# received conversation.item.created (translated from .added).
if event_type == "conversation.item.done":
return None
# Shallow-copy so we don't mutate the stored message
translated = dict(event)
# Rename the type field
if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES:
translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type]
# Fix content block types inside items (response.done output list,
# conversation.item.created item content, etc.)
if "item" in translated and isinstance(translated["item"], dict):
translated["item"] = RealTimeStreaming._translate_item_content_types(
dict(translated["item"])
)
if "response" in translated and isinstance(translated["response"], dict):
resp = dict(translated["response"])
if "output" in resp and isinstance(resp["output"], list):
resp["output"] = [
(
RealTimeStreaming._translate_item_content_types(dict(o))
if isinstance(o, dict)
else o
)
for o in resp["output"]
]
translated["response"] = resp
return translated
@staticmethod
def _translate_item_content_types(item: dict) -> dict:
"""Replace GA content type names with beta names inside a single item."""
if "content" not in item or not isinstance(item["content"], list):
return item
new_content = []
for block in item["content"]:
if (
isinstance(block, dict)
and block.get("type") in RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES
):
block = dict(block)
block["type"] = RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES[
block["type"]
]
new_content.append(block)
item["content"] = new_content
return item
async def client_ack_messages(self):
try:
while True:
@ -594,6 +824,19 @@ class RealTimeStreaming:
self._pending_guardrail_message = None
continue
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
# session shape unchanged.
if (
msg_type == "session.update"
and not self._backend_uses_beta_protocol
):
session = msg_obj.get("session", {})
if isinstance(session, dict):
session = self._remap_beta_session_to_ga(session)
msg_obj["session"] = session
message = json.dumps(msg_obj)
except (json.JSONDecodeError, AttributeError):
pass
@ -627,3 +870,8 @@ class RealTimeStreaming:
await forward_task
except asyncio.CancelledError:
pass
def client_sent_openai_beta_realtime_header(websocket: Any) -> bool:
"""True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``."""
return RealTimeStreaming._detect_beta_header(websocket)

2
litellm/litellm_core_utils/url_utils.py
View file

@ -365,7 +365,7 @@ _MAX_REDIRECTS = 10
def _extract_redirect_url(response: Any, request_url: str) -> str:
"""Extract and resolve the redirect target from a response's Location header."""
location = response.headers.get("location")
if not location:
if not isinstance(location, str) or not location:
raise SSRFError("Redirect response has no Location header")
# Resolve relative URLs against the request URL
return str(httpx.URL(request_url).join(location))

52
litellm/llms/anthropic/chat/handler.py
View file

@ -65,7 +65,7 @@ from litellm.types.utils import (
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers
from .transformation import AnthropicConfig
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -83,6 +83,7 @@ async def make_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_aclient
@ -117,6 +118,7 @@ async def make_call(
sync_stream=False,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -141,6 +143,7 @@ def make_sync_call(
timeout: Optional[Union[float, httpx.Timeout]],
json_mode: bool,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
) -> Tuple[Any, httpx.Headers]:
if client is None:
client = litellm.module_level_client # re-use a module level client
@ -183,6 +186,7 @@ def make_sync_call(
sync_stream=True,
json_mode=json_mode,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
# LOGGING
@ -237,6 +241,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
streamwrapper = CustomStreamWrapper(
completion_stream=completion_stream,
@ -462,6 +471,11 @@ class AnthropicChatCompletion(BaseLLM):
timeout=timeout,
json_mode=json_mode,
speed=optional_params.get("speed") if optional_params else None,
tool_name_reverse_map=(
litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(litellm_params, dict)
else None
),
)
return CustomStreamWrapper(
completion_stream=completion_stream,
@ -526,6 +540,7 @@ class ModelResponseIterator:
sync_stream: bool,
json_mode: Optional[bool] = False,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
self.streaming_response = streaming_response
self.response_iterator = self.streaming_response
@ -533,6 +548,13 @@ class ModelResponseIterator:
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
# rewritten-name -> caller's original. Built per-request from the
# forward map in AnthropicConfig._build_request_tool_name_maps; only
# contains entries we actually rewrote, so a tool legitimately named
# `foo_bar` is *not* reverse-mapped just because some other tool was
# rewritten to `foo_bar` in a different request. Empty/None is the
# common case (no '/' or other invalid chars in any tool name).
self.tool_name_reverse_map: Dict[str, str] = tool_name_reverse_map or {}
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
@ -557,6 +579,10 @@ class ModelResponseIterator:
# Accumulate compaction blocks for multi-turn reconstruction
self.compaction_blocks: List[Dict[str, Any]] = []
# Accumulate streamed thinking text so final usage can split reasoning
# tokens from regular output tokens.
self.reasoning_content_chunks: List[str] = []
# Track server tool use inputs and results for code_interpreter_results
self._server_tool_inputs: Dict[str, Any] = {}
self.tool_results: List[Dict[str, Any]] = []
@ -587,9 +613,14 @@ class ModelResponseIterator:
return False
def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
reasoning_content = (
"".join(self.reasoning_content_chunks)
if self.reasoning_content_chunks
else None
)
return AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
reasoning_content=None,
reasoning_content=reasoning_content,
speed=self.speed,
)
@ -636,10 +667,13 @@ class ModelResponseIterator:
"thinking" in content_block["delta"]
or "signature" in content_block["delta"]
):
thinking_content = content_block["delta"].get("thinking")
if isinstance(thinking_content, str) and thinking_content:
self.reasoning_content_chunks.append(thinking_content)
thinking_blocks = [
ChatCompletionThinkingBlock(
type="thinking",
thinking=content_block["delta"].get("thinking") or "",
thinking=thinking_content or "",
signature=str(content_block["delta"].get("signature") or ""),
)
]
@ -792,6 +826,16 @@ class ModelResponseIterator:
or content_block_start["content_block"]["type"] == "server_tool_use"
):
self.tool_index += 1
# Reverse-map the (sanitized) tool name back to the
# caller's original. No-op when the map is empty.
_stream_tool_name = content_block_start["content_block"]["name"]
if (
self.tool_name_reverse_map
and _stream_tool_name in self.tool_name_reverse_map
):
_stream_tool_name = self.tool_name_reverse_map[
_stream_tool_name
]
# Use empty string for arguments in content_block_start - actual arguments
# come in subsequent content_block_delta chunks and get accumulated.
# Using str(input) here would prepend '{}' causing invalid JSON accumulation.
@ -799,7 +843,7 @@ class ModelResponseIterator:
id=content_block_start["content_block"]["id"],
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=content_block_start["content_block"]["name"],
name=_stream_tool_name,
arguments="",
),
index=self.tool_index,

364
litellm/llms/anthropic/chat/transformation.py
View file

@ -105,6 +105,115 @@ else:
LoggingClass = Any
# Anthropic requires tool names to match ^[a-zA-Z0-9_-]{1,128}$. Any other
# character (commonly '/' or '.' from OpenAPI-derived MCP tools, e.g.
# "actions/download-job-logs-for-workflow-run") must be replaced before
# the request is sent.
#
# A naive "replace [^a-zA-Z0-9_-] with _" is unsafe because it's lossy:
# `foo/bar` and `foo_bar` both collapse to `foo_bar`. Two tools with the
# same sanitized name would either 400 at Anthropic (duplicate) or, worse,
# cause the response side to mis-translate `foo_bar` (a name the caller
# really did register) back to `foo/bar`.
#
# Instead we build a *per-request* forward map (original -> sanitized)
# whose codomain is unique within the request: when two originals collapse
# to the same candidate, or when a sanitized name collides with an already-
# valid name elsewhere in the request, we append numeric suffixes
# (`_2`, `_3`, ...) until the result is free.
#
# The reverse map (sanitized -> original) only contains entries where the
# original was actually rewritten. So a tool whose name is already valid
# round-trips identically and is *never* mistakenly re-mapped on the
# response side.
_ANTHROPIC_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
_ANTHROPIC_TOOL_NAME_MAX_LEN = 128
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
# params`` IS (it becomes the JSON body via ``data = {**optional_params}``).
# Keep these two channels strictly separate -- never stash internal
# coordination state in ``optional_params``.
ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY = "_anthropic_tool_name_map"
def _basic_sanitize_anthropic_tool_name(name: str) -> str:
"""Lossy: replace [^a-zA-Z0-9_-] with '_' and truncate to 128.
Used as a candidate generator for the per-request forward map.
Callers should NOT use this directly for translation -- always go
through the forward map so collisions are resolved.
"""
if not isinstance(name, str) or not name:
return name
return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[
:_ANTHROPIC_TOOL_NAME_MAX_LEN
]
def _build_anthropic_tool_name_maps(
original_names: List[str],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build (forward, reverse) tool-name maps for a single request.
forward[original] = sanitized -- only present when name was rewritten
reverse[sanitized] = original -- inverse of `forward`
Properties:
- All sanitized names satisfy ^[a-zA-Z0-9_-]{1,128}$.
- Sanitized names are unique within the request (no two originals
collide on the wire).
- A name that's already valid AND doesn't collide with another tool's
sanitized form passes through untouched and is absent from the maps.
That's the key correctness property: response-side translation only
runs on entries we actually rewrote, so a tool legitimately named
`foo_bar` is never incorrectly retyped to `foo/bar` just because
some *other* request had that pair.
- Order-dependent: when two originals would clash, the *second* one
seen gets the disambiguating suffix. Callers should preserve the
caller's tool order (we do).
"""
forward: Dict[str, str] = {}
used: set = set()
# First pass: reserve slots for names that are already valid so they
# always have priority regardless of input order.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
used.add(candidate)
# Second pass: sanitize/disambiguate names that need rewriting.
for original in original_names:
if not isinstance(original, str) or not original:
continue
candidate = _basic_sanitize_anthropic_tool_name(original)
if candidate == original:
continue
# Skip duplicates of the same original name. Without this guard the
# second pass would assign a fresh suffix and overwrite the forward
# map entry, causing every reference to map to the suffixed name and
# leaving the original sanitized slot orphaned in `used` with no
# reverse mapping.
if original in forward:
continue
# Disambiguate against names already chosen this request.
unique = candidate
n = 1
while unique in used:
n += 1
suffix = f"_{n}"
# Keep within the 128-char cap.
head = candidate[: _ANTHROPIC_TOOL_NAME_MAX_LEN - len(suffix)]
unique = f"{head}{suffix}"
forward[original] = unique
used.add(unique)
reverse = {v: k for k, v in forward.items()}
return forward, reverse
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT: Dict[str, str] = {
"low": "low",
"minimal": "low",
@ -486,7 +595,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
}
def _map_tool_choice(
self, tool_choice: Optional[str], parallel_tool_use: Optional[bool]
self,
tool_choice: Optional[str],
parallel_tool_use: Optional[bool],
) -> Optional[AnthropicMessagesToolChoice]:
_tool_choice: Optional[AnthropicMessagesToolChoice] = None
if tool_choice == "auto":
@ -527,7 +638,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return _tool_choice
def _map_tool_helper( # noqa: PLR0915
self, tool: ChatCompletionToolParam
self,
tool: ChatCompletionToolParam,
) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]:
returned_tool: Optional[AllAnthropicToolsValues] = None
mcp_server: Optional[AnthropicMcpServerTool] = None
@ -783,7 +895,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return initial_tool
def _map_tools(
self, tools: List
self,
tools: List,
) -> Tuple[List[AllAnthropicToolsValues], List[AnthropicMcpServerTool]]:
anthropic_tools = []
mcp_servers = []
@ -799,6 +912,174 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
mcp_servers.append(mcp_server_tool)
return anthropic_tools, mcp_servers
@staticmethod
def _rewrite_tool_names_in_messages(
messages: List[AllMessageValues],
name_forward_map: Dict[str, str],
) -> List[AllMessageValues]:
"""Return a copy of `messages` with tool_call/function_call names
rewritten using the per-request forward map.
Only mutates messages whose tool_call/function_call name is *in* the
forward map. Names absent from the map (already valid, no collision)
round-trip untouched. We only deep-copy the entries we actually
change to keep this O(turns-with-rewritten-tools), not O(history).
"""
if not name_forward_map:
return messages
new_messages: List[AllMessageValues] = []
for msg in messages:
if not isinstance(msg, dict):
new_messages.append(msg)
continue
tool_calls = msg.get("tool_calls")
function_call = msg.get("function_call")
if not tool_calls and not function_call:
new_messages.append(msg)
continue
new_msg = dict(msg)
if isinstance(tool_calls, list):
new_calls = []
for tc in tool_calls:
if not isinstance(tc, dict):
new_calls.append(tc)
continue
fn = tc.get("function")
fn_name = fn.get("name") if isinstance(fn, dict) else None
if (
isinstance(fn, dict)
and isinstance(fn_name, str)
and fn_name in name_forward_map
):
new_fn = dict(fn)
new_fn["name"] = name_forward_map[fn_name]
new_tc = dict(tc)
new_tc["function"] = new_fn
new_calls.append(new_tc)
else:
new_calls.append(tc)
new_msg["tool_calls"] = new_calls
fc_name = (
function_call.get("name") if isinstance(function_call, dict) else None
)
if (
isinstance(function_call, dict)
and isinstance(fc_name, str)
and fc_name in name_forward_map
):
new_fc = dict(function_call)
new_fc["name"] = name_forward_map[fc_name]
new_msg["function_call"] = new_fc
new_messages.append(cast(AllMessageValues, new_msg))
return new_messages
@staticmethod
def _build_request_tool_name_maps(
tools: List,
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Build the (forward, reverse) tool-name maps for an OpenAI tools list.
Operates on **OpenAI-format** tool dicts (pre-``_map_tools``). The
production sanitization path uses ``_sanitize_tool_names_in_request``
instead, which operates on **Anthropic-format** tools (post-
``_map_tools``, where ``type == "custom"``). This helper exists for
callers that need to compute the maps from the raw OpenAI shape --
e.g. test setup or future pre-mapping consumers.
See _build_anthropic_tool_name_maps for the collision rules. Pulls
the original name out of either ``{"function": {"name": ...}}``
(legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape).
"""
original_names: List[str] = []
for tool in tools or []:
if not isinstance(tool, dict):
continue
original = (
tool.get("function", {}).get("name")
if isinstance(tool.get("function"), dict)
else None
)
if original is None:
original = tool.get("name")
if isinstance(original, str) and original:
original_names.append(original)
return _build_anthropic_tool_name_maps(original_names)
@staticmethod
def _sanitize_tool_names_in_request(
optional_params: Dict[str, Any],
) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']``
in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``.
Returns ``(forward, reverse)`` for use by message-history rewriting
and response translation. ``forward[original] = sanitized`` is only
populated for names that were actually rewritten -- i.e. either
contained an invalid character or collided with another tool's
sanitized form. Names already valid AND unique pass through and are
absent from both maps.
Only ``type == "custom"`` tools (the OpenAI function-tool shape) are
considered. Hosted tools (``web_search``, ``bash``, ``code_execution``,
``computer_*``, ``mcp``, ...) own reserved names defined by Anthropic
and must not be touched.
"""
tools = optional_params.get("tools")
if not isinstance(tools, list) or not tools:
return {}, {}
# 1. Collect originals from the Anthropic-shaped custom-tool entries.
# Order matters: the first occurrence wins the canonical slot;
# later collisions get numeric suffixes (see
# ``_build_anthropic_tool_name_maps``).
original_names: List[str] = []
for t in tools:
if not isinstance(t, dict):
continue
if t.get("type") != "custom":
continue
name = t.get("name")
if isinstance(name, str) and name:
original_names.append(name)
if not original_names:
return {}, {}
forward, reverse = _build_anthropic_tool_name_maps(original_names)
if not forward:
# Every name was already valid -- nothing to do.
return forward, reverse
# 2. Apply forward map. Build a new list with copy-on-change entries
# so a caller reusing the same tool list/dicts across requests
# doesn't see its inputs permanently rewritten (which would also
# drop the original key from `forward` on the next request).
new_tools: List[Any] = []
for t in tools:
if (
isinstance(t, dict)
and t.get("type") == "custom"
and isinstance(t.get("name"), str)
and t["name"] in forward
):
new_tools.append({**t, "name": forward[t["name"]]})
else:
new_tools.append(t)
optional_params["tools"] = new_tools
# 3. Same for ``tool_choice`` when it targets a named tool. Copy
# rather than mutate for the same reason as above.
tool_choice = optional_params.get("tool_choice")
if isinstance(tool_choice, dict) and tool_choice.get("type") == "tool":
tc_name = tool_choice.get("name")
if isinstance(tc_name, str) and tc_name in forward:
optional_params["tool_choice"] = {
**tool_choice,
"name": forward[tc_name],
}
return forward, reverse
def _detect_tool_search_tools(self, tools: Optional[List]) -> bool:
"""Check if tool search tools are present in the tools list."""
if not tools:
@ -1125,6 +1406,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
non_default_params=non_default_params
)
# NB: ``map_openai_params`` deliberately does NOT sanitize tool names
# here. Names are the *original* OpenAI names at this stage, and must
# remain so until ``transform_request`` -- which is the single
# chokepoint where Anthropic, Bedrock-Anthropic, and Vertex-Anthropic
# all pass through. Doing it there guarantees:
# 1. one source of truth for the per-request forward/reverse maps,
# 2. the maps land on ``litellm_params`` (internal), never on
# ``optional_params`` (which is serialized into the request body
# via ``data = {**optional_params}`` and would 400 with
# ``Extra inputs are not permitted``).
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = (
@ -1135,7 +1427,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
value if isinstance(value, int) else max(1, int(round(value)))
)
elif param == "tools":
# check if optional params already has tools
anthropic_tools, mcp_servers = self._map_tools(value)
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=anthropic_tools
@ -1565,6 +1856,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers, optional_params=optional_params
)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
#
# - This function is the single boundary shared by AnthropicConfig,
# AmazonAnthropicConfig (Bedrock invoke), VertexAIAnthropicConfig,
# and AzureAnthropicConfig (all call ``super().transform_request``
# or ``AnthropicConfig.transform_request(self, ...)``). Sanitizing
# once here covers every Anthropic-shaped request.
# - The forward/reverse maps are coordination state; they belong on
# ``litellm_params`` (internal-only), never on ``optional_params``
# (which becomes the JSON body via ``{**optional_params}``).
# - It keeps ``map_openai_params`` a pure param translator with no
# side-channel state.
#
# The reverse map only contains entries for names that were actually
# rewritten -- so a tool legitimately named ``foo_bar`` is never
# incorrectly retyped to ``foo/bar`` on the response side.
# See _build_anthropic_tool_name_maps for the collision-handling
# rules and rationale.
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Separate system prompt from rest of message
anthropic_system_message_list = self.translate_system_message(messages=messages)
# Handling anthropic API Prompt Caching
@ -1837,8 +2156,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
speed: Optional[str] = None,
) -> Usage:
# NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
prompt_tokens = usage_object.get("input_tokens", 0) or 0
completion_tokens = usage_object.get("output_tokens", 0) or 0
raw_prompt_tokens = usage_object.get("input_tokens", 0) or 0
prompt_tokens: int = (
int(raw_prompt_tokens) if isinstance(raw_prompt_tokens, (int, float)) else 0
)
raw_completion_tokens = usage_object.get("output_tokens", 0) or 0
completion_tokens: int = (
int(raw_completion_tokens)
if isinstance(raw_completion_tokens, (int, float))
else 0
)
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
@ -1907,11 +2234,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
text_tokens=raw_input_tokens,
)
# Always populate completion_token_details, not just when there's reasoning_content
reasoning_tokens = (
estimated_reasoning_tokens = (
token_counter(text=reasoning_content, count_response_tokens=True)
if reasoning_content
else 0
)
reasoning_tokens = min(estimated_reasoning_tokens, completion_tokens)
completion_token_details = CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
text_tokens=(
@ -2041,6 +2369,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode: Optional[bool] = None,
prefix_prompt: Optional[str] = None,
speed: Optional[str] = None,
tool_name_reverse_map: Optional[Dict[str, str]] = None,
):
_hidden_params: Dict = {}
_hidden_params["additional_headers"] = process_anthropic_headers(
@ -2065,6 +2394,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
compaction_blocks,
) = self.extract_response_content(completion_response=completion_response)
# Reverse-map rewritten tool names back to caller's originals so a
# downstream OpenAI-style dispatcher can match on the registered name.
# See _build_anthropic_tool_name_maps for why this is keyed on the
# per-request reverse map (so a tool legitimately named `foo_bar` is
# never incorrectly retyped to `foo/bar`). No-op when the map is
# empty (the common case).
if tool_name_reverse_map and tool_calls:
for tc in tool_calls:
fn = tc.get("function") if isinstance(tc, dict) else None
if fn is None:
continue
_name = fn.get("name")
if isinstance(_name, str) and _name in tool_name_reverse_map:
fn["name"] = tool_name_reverse_map[_name]
if (
prefix_prompt is not None
and not text_content.startswith(prefix_prompt)
@ -2191,6 +2535,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
prefix_prompt = self.get_prefix_prompt(messages=messages)
speed = optional_params.get("speed")
tool_name_reverse_map: Optional[Dict[str, str]] = None
if isinstance(litellm_params, dict):
_candidate = litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY)
if isinstance(_candidate, dict):
tool_name_reverse_map = _candidate
model_response = self.transform_parsed_response(
completion_response=completion_response,
@ -2199,6 +2548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
json_mode=json_mode,
prefix_prompt=prefix_prompt,
speed=speed,
tool_name_reverse_map=tool_name_reverse_map,
)
return model_response

7
litellm/llms/azure/azure.py
View file

@ -44,6 +44,7 @@ from .common_utils import (
select_azure_base_url_or_endpoint,
)
from .image_generation import get_azure_image_generation_config
from .image_generation.http_utils import azure_deployment_image_generation_json_body
class AzureOpenAIAssistantsAPIConfig:
@ -966,9 +967,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json = azure_deployment_image_generation_json_body(api_base, data)
return await async_handler.post(
url=api_base,
json=data,
json=request_json,
headers=headers,
)
@ -1085,9 +1087,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json = azure_deployment_image_generation_json_body(api_base, data)
return sync_handler.post(
url=api_base,
json=data,
json=request_json,
headers=headers,
)

18
litellm/llms/azure/image_edit/transformation.py
View file

@ -9,6 +9,19 @@ from litellm.utils import _add_path_to_api_base
class AzureImageEditConfig(OpenAIImageEditConfig):
@staticmethod
def azure_deployment_image_edit_form_data(data: dict, request_url: str) -> dict:
"""
Azure OpenAI ``.../openai/deployments/{deployment}/images/edits`` routes by
deployment in the URL; including ``model`` in multipart fields can break
the same way as image generations (LiteLLM #26316).
Non-deployment edit URLs keep ``model`` when present.
"""
if "images/edits" in request_url and "/openai/deployments/" in request_url:
return {k: v for k, v in data.items() if k != "model"}
return data
def validate_environment(
self,
headers: dict,
@ -83,3 +96,8 @@ class AzureImageEditConfig(OpenAIImageEditConfig):
final_url = httpx.URL(new_url).copy_with(params=query_params)
return str(final_url)
def finalize_image_edit_request_data(
self, data: dict, resolved_request_url: str
) -> dict:
return self.azure_deployment_image_edit_form_data(data, resolved_request_url)

2
litellm/llms/azure/image_generation/__init__.py
View file

@ -6,11 +6,13 @@ from litellm.llms.base_llm.image_generation.transformation import (
from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
from .gpt_transformation import AzureGPTImageGenerationConfig
from .http_utils import azure_deployment_image_generation_json_body
__all__ = [
"AzureDallE2ImageGenerationConfig",
"AzureDallE3ImageGenerationConfig",
"AzureGPTImageGenerationConfig",
"azure_deployment_image_generation_json_body",
]

17
litellm/llms/azure/image_generation/http_utils.py Normal file
View file

@ -0,0 +1,17 @@
"""HTTP helpers for Azure OpenAI image generation (REST, not SDK)."""
def azure_deployment_image_generation_json_body(api_base: str, data: dict) -> dict:
"""
Build the JSON body for Azure OpenAI image generation POSTs.
For ``.../openai/deployments/{deployment}/images/generations``, routing uses the
deployment in the URL only; sending ``model`` in the body (especially the deployment
name) breaks some models (e.g. gpt-image-2). See LiteLLM #26316.
Provider-style URLs (e.g. ``/providers/...`` for FLUX on Azure AI) keep all keys
so non–OpenAI-deployment payloads still work.
"""
if "images/generations" in api_base and "/openai/deployments/" in api_base:
return {k: v for k, v in data.items() if k != "model"}
return data

6
litellm/llms/azure/realtime/handler.py
View file

@ -89,9 +89,10 @@ class AzureOpenAIRealtime(AzureChatCompletion):
if api_base is None:
raise ValueError("api_base is required for Azure OpenAI calls")
if api_version is None and (
backend_uses_beta_protocol = (
realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
):
)
if api_version is None and backend_uses_beta_protocol:
raise ValueError("api_version is required for Azure OpenAI calls")
url = self._construct_url(
@ -114,6 +115,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
logging_obj,
user_api_key_dict=user_api_key_dict,
request_data={"litellm_metadata": litellm_metadata or {}},
backend_uses_beta_protocol=backend_uses_beta_protocol,
)
await realtime_streaming.bidirectional_forward()

12
litellm/llms/base_llm/image_edit/transformation.py
View file

@ -102,6 +102,18 @@ class BaseImageEditConfig(ABC):
) -> Tuple[Dict, RequestFiles]:
pass
def finalize_image_edit_request_data(
self, data: dict, resolved_request_url: str
) -> dict:
"""
Last pass on the request dict after ``transform_image_edit_request``, using the
exact URL string used for the HTTP POST (same as ``get_complete_url`` output).
The handler sends this dict as ``data=`` for multipart providers or ``json=``
for JSON-only providers; default implementation returns ``data`` unchanged.
"""
return data
@abstractmethod
def transform_image_edit_response(
self,

33
litellm/llms/base_llm/managed_resources/base_managed_resource.py
View file

@ -18,6 +18,11 @@ from typing import (
)
from litellm import verbose_logger
from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
can_access_resource,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import SpecialEnums
@ -169,6 +174,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
"model_mappings": model_mappings,
"flat_model_resource_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
@ -190,6 +196,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
"model_mappings": json.dumps(model_mappings),
"flat_model_resource_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
@ -316,15 +323,17 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
Returns:
True if user has access, False otherwise
"""
user_id = user_api_key_dict.user_id
# Use cached method instead of direct DB query
resource = await self.get_unified_resource_id(
unified_resource_id, litellm_parent_otel_span
)
if resource:
return resource.get("created_by") == user_id
return can_access_resource(
user_api_key_dict=user_api_key_dict,
created_by=resource.get("created_by"),
resource_team_id=resource.get("team_id"),
)
return False
@ -549,11 +558,11 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
Returns:
Dictionary with list of resources and pagination info
"""
where_clause: Dict[str, Any] = {}
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return build_list_page([])
# Filter by user who created the resource
if user_api_key_dict.user_id:
where_clause["created_by"] = user_api_key_dict.user_id
where_clause: Dict[str, Any] = {**owner_filter}
if after:
where_clause["id"] = {"gt": after}
@ -598,10 +607,6 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
)
continue
return {
"object": "list",
"data": resource_objects,
"first_id": resource_objects[0].id if resource_objects else None,
"last_id": resource_objects[-1].id if resource_objects else None,
"has_more": len(resource_objects) == (limit or 20),
}
return build_list_page(
resource_objects, has_more=len(resource_objects) == (limit or 20)
)

99
litellm/llms/base_llm/managed_resources/isolation.py Normal file
View file

@ -0,0 +1,99 @@
"""
Tenant-isolation helpers for managed file/batch/vector-store resources.
Returns a Prisma filter and an ownership check that scope managed resources
to the caller's identity: proxy admins see everything, user-keyed callers
see records they created, and service-account keys (no user_id) fall back
to the resource's owning team. Callers with no admin role and no
identifying ids are denied so an empty user_id can never select an
unscoped query.
"""
from typing import Any, Dict, List, Optional
from litellm.proxy._types import (
UserAPIKeyAuth,
user_api_key_has_admin_view as _user_has_admin_view,
)
def build_list_page(items: List[Any], has_more: bool = False) -> Dict[str, Any]:
"""Build the OpenAI-style paginated list response shape used by managed
file/batch/vector-store listings. ``first_id`` and ``last_id`` are
sourced from each item's ``.id`` attribute."""
return {
"object": "list",
"data": items,
"first_id": items[0].id if items else None,
"last_id": items[-1].id if items else None,
"has_more": has_more,
}
def build_owner_filter(
user_api_key_dict: UserAPIKeyAuth,
) -> Optional[Dict[str, Any]]:
"""Return a Prisma `where` fragment that scopes a managed-resource listing
to records the caller is allowed to see.
- ``{}`` means no scoping (proxy admins).
- ``{"created_by": <user_id>}`` for user-keyed callers.
- ``{"team_id": <team_id>}`` for service-account callers
that have a team but no user_id.
- ``{"OR": [...]}`` when the caller has both — listing must include
both their own resources and team-shared ones so it stays consistent
with ``can_access_resource``.
- ``None`` means deny: callers MUST skip the query rather than fall
back to an unscoped fetch.
"""
if _user_has_admin_view(user_api_key_dict):
return {}
user_id = user_api_key_dict.user_id
team_id = user_api_key_dict.team_id
if user_id is not None and team_id is not None:
return {
"OR": [
{"created_by": user_id},
{"team_id": team_id},
]
}
if user_id is not None:
return {"created_by": user_id}
if team_id is not None:
return {"team_id": team_id}
return None
def can_access_resource(
user_api_key_dict: UserAPIKeyAuth,
created_by: Optional[str],
resource_team_id: Optional[str],
) -> bool:
"""Return True iff the caller may read/modify a managed resource.
The resource's ``created_by`` and ``team_id`` fields must be non-None
to match the caller's identity — guarding against the ``None == None``
bypass that previously let service-account keys read every keyless
resource.
"""
if _user_has_admin_view(user_api_key_dict):
return True
user_id = user_api_key_dict.user_id
if user_id is not None and created_by is not None and created_by == user_id:
return True
team_id = user_api_key_dict.team_id
if (
team_id is not None
and resource_team_id is not None
and resource_team_id == team_id
):
return True
return False

205
litellm/llms/bedrock/base_aws_llm.py
View file

@ -7,6 +7,8 @@ from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
Callable,
ClassVar,
Dict,
List,
Literal,
@ -63,8 +65,16 @@ class AwsAuthError(Exception):
class BaseAWSLLM:
# Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request).
# Storage is in-process memory only: default ``DualCache()`` has no Redis backend unless attached
# elsewhere. Entry TTL: static access-key + secret + region use ``_get_default_ttl_for_boto3_credentials``
# (~59 minutes); ambient env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s
# ``default_ttl`` (600 seconds / 10 minutes). AssumeRole, web identity, profiles, and explicit
# session-token tuples are not cached — see ``get_credentials`` and ``_get_or_set_cached_credentials``.
_shared_iam_cache: ClassVar[DualCache] = DualCache()
def __init__(self) -> None:
self.iam_cache = DualCache()
self.iam_cache = BaseAWSLLM._shared_iam_cache
super().__init__()
self.aws_authentication_params = [
"aws_access_key_id",
@ -103,6 +113,79 @@ class BaseAWSLLM:
credential_str = json.dumps(credential_args, sort_keys=True)
return hashlib.sha256(credential_str.encode()).hexdigest()
def _get_or_set_cached_credentials(
self,
credential_args: Dict[str, Optional[str]],
credential_fetcher: Callable[[], Tuple[Any, Optional[int]]],
) -> Any:
"""
Read-through IAM cache on the process-wide ``DualCache``.
Only the in-memory layer is used by default (no Redis on ``_shared_iam_cache`` unless
configured globally). TTL on write: static access-key fetches pass
``_get_default_ttl_for_boto3_credentials()`` (~59 minutes); ambient env passes ``ttl=None``,
which ``InMemoryCache.set_cache`` resolves to ``default_ttl`` (600 seconds / 10 minutes by
default).
Used only for static access-key credentials and ambient credentials from
``_auth_with_env_vars`` (including when skipping AssumeRole because the runtime identity
already matches ``aws_role_name``).
AssumeRole, web identity exchange, profiles, and explicit session-token tuples are not
cached here — shared ``Credentials`` / refresh state must not span logical sessions.
"""
cache_key = self.get_cache_key(credential_args)
_cached = self.iam_cache.get_cache(cache_key)
if _cached:
return _cached
credentials, ttl = credential_fetcher()
self.iam_cache.set_cache(cache_key, credentials, ttl=ttl)
return credentials
@staticmethod
def _is_auth_with_web_identity_token(
aws_web_identity_token: Optional[str],
aws_role_name: Optional[str],
aws_session_name: Optional[str],
) -> bool:
return (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
)
@staticmethod
def _is_auth_with_aws_role(aws_role_name: Optional[str]) -> bool:
return aws_role_name is not None
@staticmethod
def _is_auth_with_aws_profile(aws_profile_name: Optional[str]) -> bool:
return aws_profile_name is not None
@staticmethod
def _is_auth_with_aws_session_token_tuple(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
)
@staticmethod
def _is_auth_with_access_key_and_secret_key(
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_region_name: Optional[str],
) -> bool:
return (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
)
@tracer.wrap()
def get_credentials(
self,
@ -184,95 +267,97 @@ class BaseAWSLLM:
aws_external_id,
)
# create cache key for non-expiring auth flows
args = {
k: v
for k, v in locals().items()
if k.startswith("aws_") or k == "ssl_verify"
}
cache_key = self.get_cache_key(args)
_cached_credentials = self.iam_cache.get_cache(cache_key)
if _cached_credentials:
return _cached_credentials
#########################################################
# Handle diff boto3 auth flows
# for each helper
# Return:
# Credentials - boto3.Credentials
# cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time.
#
# iam_cache: static keys and ambient env only (including skip-AssumeRole path).
# Do not cache AssumeRole / web identity / profile / explicit session-token paths here.
#########################################################
if (
aws_web_identity_token is not None
and aws_role_name is not None
and aws_session_name is not None
if self._is_auth_with_web_identity_token(
aws_web_identity_token,
aws_role_name,
aws_session_name,
):
credentials, _cache_ttl = self._auth_with_web_identity_token(
aws_web_identity_token=aws_web_identity_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_web_identity_token=cast(str, aws_web_identity_token),
aws_role_name=cast(str, aws_role_name),
aws_session_name=cast(str, aws_session_name),
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
)
elif aws_role_name is not None:
# Check if we're already running as the target role and can skip assumption
# This handles IRSA (EKS), ECS task roles, and EC2 instance profiles
if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify):
return credentials
elif self._is_auth_with_aws_role(aws_role_name):
# Same role (IRSA/ECS/EC2): ambient creds via _get_or_set_cached_credentials like the
# default env branch; never pre-read cache (must run _is_already_running_as_role first).
if self._is_already_running_as_role(
cast(str, aws_role_name), ssl_verify=ssl_verify
):
verbose_logger.debug(
"Already running as target role %s, using ambient credentials",
aws_role_name,
)
credentials, _cache_ttl = self._auth_with_env_vars()
else:
verbose_logger.debug(
"Using role assumption: calling _auth_with_aws_role"
return self._get_or_set_cached_credentials(
args, self._auth_with_env_vars
)
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = (
f"litellm-session-{int(datetime.now().timestamp())}"
)
credentials, _cache_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
aws_role_name=aws_role_name,
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
elif aws_profile_name is not None: ### CHECK SESSION ###
credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_session_token is not None
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
# If aws_session_name is not provided, generate a default one
if aws_session_name is None:
aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
credentials, _assume_ttl = self._auth_with_aws_role(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
)
elif (
aws_access_key_id is not None
and aws_secret_access_key is not None
and aws_region_name is not None
):
credentials, _cache_ttl = self._auth_with_access_key_and_secret_key(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_role_name=cast(str, aws_role_name),
aws_session_name=aws_session_name,
aws_region_name=aws_region_name,
aws_sts_endpoint=aws_sts_endpoint,
aws_external_id=aws_external_id,
ssl_verify=ssl_verify,
)
return credentials
elif self._is_auth_with_aws_profile(aws_profile_name):
credentials, _cache_ttl = self._auth_with_aws_profile(
cast(str, aws_profile_name)
)
return credentials
elif self._is_auth_with_aws_session_token_tuple(
aws_access_key_id,
aws_secret_access_key,
aws_session_token,
):
credentials, _cache_ttl = self._auth_with_aws_session_token(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_session_token=cast(str, aws_session_token),
)
return credentials
elif self._is_auth_with_access_key_and_secret_key(
aws_access_key_id,
aws_secret_access_key,
aws_region_name,
):
return self._get_or_set_cached_credentials(
args,
lambda: self._auth_with_access_key_and_secret_key(
aws_access_key_id=cast(str, aws_access_key_id),
aws_secret_access_key=cast(str, aws_secret_access_key),
aws_region_name=cast(str, aws_region_name),
),
)
else:
credentials, _cache_ttl = self._auth_with_env_vars()
self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl)
return credentials
return self._get_or_set_cached_credentials(args, self._auth_with_env_vars)
def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]:
try:

24
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -299,29 +299,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
)
def _get_response_stream_shape(self):
"""Get the response stream shape for parsing, reusing existing logic."""
try:
# Try to reuse the cached shape from the existing decoder
from litellm.llms.bedrock.chat.invoke_handler import (
get_response_stream_shape,
)
from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE
return get_response_stream_shape()
except ImportError:
# Fallback: create our own shape
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model(
"bedrock-runtime", "service-2"
)
bedrock_service_model = ServiceModel(bedrock_service_dict)
return bedrock_service_model.shape_for("ResponseStream")
except Exception as e:
verbose_logger.warning(f"Could not load response stream shape: {e}")
return None
return BEDROCK_RESPONSE_STREAM_SHAPE
def _extract_response_content(self, events: InvokeAgentEventList) -> str:
"""Extract the final response content from parsed events."""

34
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -67,9 +67,13 @@ from litellm.types.utils import (
from litellm.utils import CustomStreamWrapper, get_secret
from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
from ..common_utils import (
BEDROCK_RESPONSE_STREAM_SHAPE,
BedrockError,
ModelResponseIterator,
get_bedrock_tool_name,
)
_response_stream_shape_cache = None
bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
max_size_in_memory=50, default_ttl=600
)
@ -1391,20 +1395,6 @@ class BedrockLLM(BaseAWSLLM):
return None
def get_response_stream_shape():
global _response_stream_shape_cache
if _response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2")
bedrock_service_model = ServiceModel(bedrock_service_dict)
_response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream")
return _response_stream_shape_cache
class AWSEventStreamDecoder:
def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
from botocore.parsers import EventStreamJSONParser
@ -1838,8 +1828,18 @@ class AWSEventStreamDecoder:
yield self._chunk_parser(chunk_data=_data)
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
raise BedrockError(
status_code=500,
message=(
"Bedrock event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
parsed_response = self.parser.parse(
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:
decoded_body = response_dict["body"].decode()

58
litellm/llms/bedrock/common_utils.py
View file

@ -14,6 +14,7 @@ if TYPE_CHECKING:
import httpx
import litellm
from litellm import verbose_logger
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -917,38 +918,57 @@ def get_bedrock_chat_config(model: str):
return litellm.AmazonInvokeConfig()
def _load_bedrock_response_stream_shape():
"""
Load the ResponseStream shape from botocore's bundled bedrock-runtime schema.
Called once at module import time; the result is stored in
``BEDROCK_RESPONSE_STREAM_SHAPE`` and reused for the process lifetime.
Returns ``None`` if botocore is unavailable or the service model cannot be
loaded, so the module still imports cleanly.
"""
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
service_dict = loader.load_service_model("bedrock-runtime", "service-2")
return ServiceModel(service_dict).shape_for("ResponseStream")
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load bedrock-runtime response stream shape "
"— Bedrock event-stream decoding will be unavailable. Error: %s",
e,
)
return None
# Eagerly resolved once per process — avoids per-instance or per-request disk I/O.
BEDROCK_RESPONSE_STREAM_SHAPE = _load_bedrock_response_stream_shape()
class BedrockEventStreamDecoderBase:
"""
Base class for event stream decoding for Bedrock
"""
_response_stream_shape_cache = None
def __init__(self):
from botocore.parsers import EventStreamJSONParser
self.parser = EventStreamJSONParser()
def get_response_stream_shape(self):
if self._response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
bedrock_service_dict = loader.load_service_model(
"bedrock-runtime", "service-2"
)
bedrock_service_model = ServiceModel(bedrock_service_dict)
self._response_stream_shape_cache = bedrock_service_model.shape_for(
"ResponseStream"
)
return self._response_stream_shape_cache
def _parse_message_from_event(self, event) -> Optional[str]:
if BEDROCK_RESPONSE_STREAM_SHAPE is None:
raise BedrockError(
status_code=500,
message=(
"Bedrock event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(
response_dict, self.get_response_stream_shape()
response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:

6
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -10,14 +10,12 @@ from ..base_aws_llm import BaseAWSLLM
from ..common_utils import BedrockEventStreamDecoderBase, BedrockModelInfo
if TYPE_CHECKING:
from httpx import URL
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import CostResponseTypes
if TYPE_CHECKING:
from httpx import URL
class BedrockPassthroughConfig(
BaseAWSLLM, BedrockModelInfo, BedrockEventStreamDecoderBase, BasePassthroughConfig
):

53
litellm/llms/custom_httpx/http_handler.py
View file

@ -1,4 +1,5 @@
import asyncio
import concurrent.futures
import inspect
import os
import socket
@ -133,6 +134,11 @@ _DEFAULT_TIMEOUT = httpx.Timeout(
timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
)
_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0
_STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=50,
thread_name_prefix="litellm-streaming-error-body-read",
)
def _prepare_request_data_and_content(
@ -386,17 +392,30 @@ def _safe_get_response_text(response: httpx.Response) -> str:
return ""
async def _safe_aread_response(response: httpx.Response) -> bytes:
async def _safe_aread_response(
response: httpx.Response, timeout: Optional[float] = None
) -> bytes:
"""Safely read async response body, falling back to empty bytes on errors."""
try:
if timeout is not None:
return await asyncio.wait_for(response.aread(), timeout=timeout)
return await response.aread()
except Exception:
return b""
def _safe_read_response(response: httpx.Response) -> bytes:
def _safe_read_response(
response: httpx.Response, timeout: Optional[float] = None
) -> bytes:
"""Safely read sync response body, falling back to empty bytes on errors."""
try:
if timeout is not None:
future = _STREAMING_ERROR_BODY_READ_EXECUTOR.submit(response.read)
try:
return future.result(timeout=timeout)
except Exception:
response.close()
return b""
return response.read()
except Exception:
return b""
@ -405,8 +424,19 @@ def _safe_read_response(response: httpx.Response) -> bytes:
def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
"""Raise a MaskedHTTPStatusError for sync HTTP handlers."""
if stream:
_body = mask_sensitive_info(_safe_read_response(e.response))
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
try:
_body = mask_sensitive_info(
_safe_read_response(
e.response,
timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
)
)
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
finally:
try:
e.response.close()
except Exception:
pass
_text = mask_sensitive_info(_safe_get_response_text(e.response))
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
@ -414,8 +444,19 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None:
"""Raise a MaskedHTTPStatusError for async HTTP handlers."""
if stream:
_body = mask_sensitive_info(await _safe_aread_response(e.response))
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
try:
_body = mask_sensitive_info(
await _safe_aread_response(
e.response,
timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
)
)
raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
finally:
try:
await e.response.aclose()
except Exception:
pass
_text = mask_sensitive_info(_safe_get_response_text(e.response))
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None

7
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -5255,7 +5255,6 @@ class BaseLLMHTTPHandler:
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
"OpenAI-Beta": "realtime=v1",
}
if extra_headers:
@ -5579,6 +5578,9 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
headers=headers,
)
data = image_edit_provider_config.finalize_image_edit_request_data(
data, api_base
)
## LOGGING
logging_obj.pre_call(
@ -5677,6 +5679,9 @@ class BaseLLMHTTPHandler:
litellm_params=litellm_params,
headers=headers,
)
data = image_edit_provider_config.finalize_image_edit_request_data(
data, api_base
)
## LOGGING
logging_obj.pre_call(

70
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -2,7 +2,18 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import (
Any,
Coroutine,
Dict,
List,
Literal,
Optional,
Tuple,
Union,
cast,
overload,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_get_image_mime_type_from_url,
@ -21,6 +32,61 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
class HostedVLLMChatConfig(OpenAIGPTConfig):
def _convert_custom_tools_to_function_tools(
self, tools: List[Dict[str, Any]]
) -> List[Dict[str, Any]]:
"""
vLLM chat completions currently accepts only OpenAI function tools.
Convert custom tools into function tools so request validation does not fail.
"""
converted_tools: List[Dict[str, Any]] = []
for idx, tool in enumerate(tools):
if not isinstance(tool, dict):
converted_tools.append(tool)
continue
if tool.get("type") != "custom":
converted_tools.append(tool)
continue
custom_tool = tool.get("custom", {})
if not isinstance(custom_tool, dict):
custom_tool = {}
tool_name = (
custom_tool.get("name") or tool.get("name") or f"custom_tool_{idx}"
)
tool_description = custom_tool.get("description") or tool.get("description")
tool_parameters = custom_tool.get("input_schema") or tool.get(
"input_schema"
)
if not isinstance(tool_parameters, dict):
tool_parameters = {
"type": "object",
"properties": {
"input": {
"type": "string",
"description": "Raw tool input payload.",
}
},
"required": ["input"],
}
function_tool: Dict[str, Any] = {
"type": "function",
"function": {
"name": str(tool_name),
"parameters": tool_parameters,
},
}
if isinstance(tool_description, str):
function_tool["function"]["description"] = tool_description
converted_tools.append(function_tool)
return converted_tools
def get_supported_openai_params(self, model: str) -> List[str]:
params = super().get_supported_openai_params(model)
params.extend(["reasoning_effort", "thinking"])
@ -39,6 +105,8 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
_tools = _remove_additional_properties(_tools)
# remove 'strict' from tools
_tools = _remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)
if _tools is not None:
non_default_params["tools"] = _tools

187
litellm/llms/litellm_proxy/skills/handler.py
View file

@ -9,42 +9,47 @@ import uuid
from typing import Any, Dict, List, Optional
from litellm._logging import verbose_logger
from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest, UserAPIKeyAuth
from litellm.proxy.common_utils.resource_ownership import (
get_primary_resource_owner_scope,
get_resource_owner_scopes,
is_proxy_admin,
user_can_access_resource_owner,
)
# Skills are looked up on every chat completion that has skills enabled
# (`SkillsInjectionHook` calls ``fetch_skill_from_db``). 60s LRU/TTL cache
# absorbs the hot read before it reaches Prisma. ``_NEGATIVE_SKILL_SENTINEL``
# lets us cache a true "skill does not exist" so repeated misses also
# avoid the DB — ``InMemoryCache`` returns ``None`` indistinguishably for
# "miss" and "cached as None".
_NEGATIVE_SKILL_SENTINEL = "__litellm_skill_not_found__"
_SKILL_CACHE = InMemoryCache(max_size_in_memory=10000, default_ttl=60)
def _prisma_skill_to_litellm(prisma_skill) -> LiteLLM_SkillsTable:
"""
Convert a Prisma skill record to LiteLLM_SkillsTable.
"""Convert a Prisma skill record to LiteLLM_SkillsTable.
Handles Base64 decoding of file_content field.
Handles Base64 decoding of file_content field — model_dump() converts
Base64 fields to base64-encoded strings.
"""
import base64
data = prisma_skill.model_dump()
# Decode Base64 file_content back to bytes
# model_dump() converts Base64 field to base64-encoded string
if data.get("file_content") is not None:
if isinstance(data["file_content"], str):
data["file_content"] = base64.b64decode(data["file_content"])
elif isinstance(data["file_content"], bytes):
# Already bytes, no conversion needed
pass
return LiteLLM_SkillsTable(**data)
class LiteLLMSkillsHandler:
"""
Handler for LiteLLM database-backed skills operations.
This class provides static methods for CRUD operations on skills
stored in the LiteLLM proxy database (LiteLLM_SkillsTable).
"""
"""CRUD for skills stored in ``litellm_skillstable``."""
@staticmethod
async def _get_prisma_client():
"""Get the prisma client from proxy server."""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
@ -58,20 +63,21 @@ class LiteLLMSkillsHandler:
async def create_skill(
data: NewSkillRequest,
user_id: Optional[str] = None,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
) -> LiteLLM_SkillsTable:
"""
Create a new skill in the LiteLLM database.
Args:
data: NewSkillRequest with skill details
user_id: Optional user ID for tracking
Returns:
LiteLLM_SkillsTable record
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
skill_id = f"litellm_skill_{uuid.uuid4()}"
owner = get_primary_resource_owner_scope(user_api_key_dict) or user_id
if owner is None:
# Identity-less callers (no user_id / team_id / org_id /
# api_key / token) can't be uniquely stamped on the row.
# Stamping a placeholder would let any two such callers see
# each other's skills via the shared owner. ValueError keeps
# this module FastAPI-free per the project layering rule.
raise ValueError(
"Unable to record skill ownership: caller has no identity scope."
)
skill_data: Dict[str, Any] = {
"skill_id": skill_id,
@ -79,17 +85,15 @@ class LiteLLMSkillsHandler:
"description": data.description,
"instructions": data.instructions,
"source": "custom",
"created_by": user_id,
"updated_by": user_id,
"created_by": owner,
"updated_by": owner,
}
# Handle metadata
if data.metadata is not None:
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
skill_data["metadata"] = safe_dumps(data.metadata)
# Handle file content - wrap bytes in Base64 for Prisma
if data.file_content is not None:
from prisma.fields import Base64
@ -104,112 +108,103 @@ class LiteLLMSkillsHandler:
)
new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data)
return _prisma_skill_to_litellm(new_skill)
@staticmethod
async def list_skills(
limit: int = 20,
offset: int = 0,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
) -> List[LiteLLM_SkillsTable]:
"""
List skills from the LiteLLM database.
Args:
limit: Maximum number of skills to return
offset: Number of skills to skip
Returns:
List of LiteLLM_SkillsTable records
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(
f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}"
)
skills = await prisma_client.db.litellm_skillstable.find_many(
take=limit,
skip=offset,
order={"created_at": "desc"},
)
find_many_kwargs: Dict[str, Any] = {
"take": limit,
"skip": offset,
"order": {"created_at": "desc"},
}
if user_api_key_dict is not None and not is_proxy_admin(user_api_key_dict):
owner_scopes = get_resource_owner_scopes(user_api_key_dict)
if not owner_scopes:
return []
find_many_kwargs["where"] = {"created_by": {"in": owner_scopes}}
skills = await prisma_client.db.litellm_skillstable.find_many(
**find_many_kwargs
)
return [_prisma_skill_to_litellm(s) for s in skills]
@staticmethod
async def get_skill(skill_id: str) -> LiteLLM_SkillsTable:
async def _load_skill(skill_id: str) -> Optional[Any]:
"""Cache-first read of the Prisma skill row. Owner-scope filtering
happens on the cached row, so the cache is per-skill not per-caller.
"""
Get a skill by ID from the LiteLLM database.
cached = _SKILL_CACHE.get_cache(skill_id)
if cached == _NEGATIVE_SKILL_SENTINEL:
return None
if cached is not None:
return cached
Args:
skill_id: The skill ID to retrieve
Returns:
LiteLLM_SkillsTable record
Raises:
ValueError: If skill not found
"""
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}")
skill = await prisma_client.db.litellm_skillstable.find_unique(
where={"skill_id": skill_id}
)
_SKILL_CACHE.set_cache(
skill_id, skill if skill is not None else _NEGATIVE_SKILL_SENTINEL
)
return skill
if skill is None:
@staticmethod
async def get_skill(
skill_id: str,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
) -> LiteLLM_SkillsTable:
verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}")
skill = await LiteLLMSkillsHandler._load_skill(skill_id)
# Same "not found" message for both "missing" and "cross-tenant"
# so callers can't enumerate skill IDs they don't own.
if skill is None or not user_can_access_resource_owner(
getattr(skill, "created_by", None), user_api_key_dict
):
raise ValueError(f"Skill not found: {skill_id}")
return _prisma_skill_to_litellm(skill)
@staticmethod
async def delete_skill(skill_id: str) -> Dict[str, str]:
"""
Delete a skill by ID from the LiteLLM database.
Args:
skill_id: The skill ID to delete
Returns:
Dict with id and type of deleted skill
Raises:
ValueError: If skill not found
"""
async def delete_skill(
skill_id: str,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
) -> Dict[str, str]:
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}")
# Check if skill exists
skill = await prisma_client.db.litellm_skillstable.find_unique(
where={"skill_id": skill_id}
)
if skill is None:
skill = await LiteLLMSkillsHandler._load_skill(skill_id)
if skill is None or not user_can_access_resource_owner(
getattr(skill, "created_by", None), user_api_key_dict
):
raise ValueError(f"Skill not found: {skill_id}")
# Delete the skill
await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id})
_SKILL_CACHE.set_cache(skill_id, _NEGATIVE_SKILL_SENTINEL)
return {"id": skill_id, "type": "skill_deleted"}
@staticmethod
async def fetch_skill_from_db(skill_id: str) -> Optional[LiteLLM_SkillsTable]:
"""
Fetch a skill from the database (used by skills injection hook).
This is a convenience method that returns None instead of raising
an exception if the skill is not found.
Args:
skill_id: The skill ID to fetch
Returns:
LiteLLM_SkillsTable or None if not found
"""
async def fetch_skill_from_db(
skill_id: str,
user_api_key_dict: Optional[UserAPIKeyAuth] = None,
) -> Optional[LiteLLM_SkillsTable]:
"""Skills-injection-hook helper: returns None instead of raising on
not-found / not-authorized so the hook can silently skip."""
try:
return await LiteLLMSkillsHandler.get_skill(skill_id)
return await LiteLLMSkillsHandler.get_skill(
skill_id, user_api_key_dict=user_api_key_dict
)
except ValueError:
return None
except Exception as e:

65
litellm/llms/litellm_proxy/skills/transformation.py
View file

@ -18,6 +18,7 @@ from litellm.types.utils import LlmProviders
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy._types import UserAPIKeyAuth
class LiteLLMSkillsTransformationHandler:
@ -44,6 +45,7 @@ class LiteLLMSkillsTransformationHandler:
file_type: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
user_id: Optional[str] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
@ -99,6 +101,7 @@ class LiteLLMSkillsTransformationHandler:
file_type=file_type,
metadata=metadata,
user_id=user_id,
user_api_key_dict=user_api_key_dict,
)
import asyncio
@ -113,6 +116,7 @@ class LiteLLMSkillsTransformationHandler:
file_type=file_type,
metadata=metadata,
user_id=user_id,
user_api_key_dict=user_api_key_dict,
)
)
@ -126,6 +130,7 @@ class LiteLLMSkillsTransformationHandler:
file_type: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
user_id: Optional[str] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> Skill:
"""Async implementation of create_skill."""
# Lazy import to avoid SDK dependency on proxy
@ -145,6 +150,7 @@ class LiteLLMSkillsTransformationHandler:
db_skill = await LiteLLMSkillsHandler.create_skill(
data=skill_request,
user_id=user_id,
user_api_key_dict=user_api_key_dict,
)
return self._db_skill_to_response(db_skill)
@ -156,6 +162,7 @@ class LiteLLMSkillsTransformationHandler:
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
**kwargs,
) -> Union[ListSkillsResponse, Coroutine[Any, Any, ListSkillsResponse]]:
"""
@ -182,18 +189,27 @@ class LiteLLMSkillsTransformationHandler:
)
if _is_async:
return self._async_list_skills(limit=limit, offset=offset)
return self._async_list_skills(
limit=limit,
offset=offset,
user_api_key_dict=user_api_key_dict,
)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_list_skills(limit=limit, offset=offset)
self._async_list_skills(
limit=limit,
offset=offset,
user_api_key_dict=user_api_key_dict,
)
)
async def _async_list_skills(
self,
limit: int = 20,
offset: int = 0,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> ListSkillsResponse:
"""Async implementation of list_skills."""
# Lazy import to avoid SDK dependency on proxy
@ -202,6 +218,7 @@ class LiteLLMSkillsTransformationHandler:
db_skills = await LiteLLMSkillsHandler.list_skills(
limit=limit,
offset=offset,
user_api_key_dict=user_api_key_dict,
)
skills = [self._db_skill_to_response(s) for s in db_skills]
@ -217,6 +234,7 @@ class LiteLLMSkillsTransformationHandler:
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
**kwargs,
) -> Union[Skill, Coroutine[Any, Any, Skill]]:
"""
@ -242,20 +260,33 @@ class LiteLLMSkillsTransformationHandler:
)
if _is_async:
return self._async_get_skill(skill_id=skill_id)
return self._async_get_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_get_skill(skill_id=skill_id)
self._async_get_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
)
async def _async_get_skill(self, skill_id: str) -> Skill:
async def _async_get_skill(
self,
skill_id: str,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> Skill:
"""Async implementation of get_skill."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
db_skill = await LiteLLMSkillsHandler.get_skill(skill_id=skill_id)
db_skill = await LiteLLMSkillsHandler.get_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
return self._db_skill_to_response(db_skill)
def delete_skill_handler(
@ -264,6 +295,7 @@ class LiteLLMSkillsTransformationHandler:
_is_async: bool = False,
logging_obj: Optional["LiteLLMLoggingObj"] = None,
litellm_call_id: Optional[str] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
**kwargs,
) -> Union[DeleteSkillResponse, Coroutine[Any, Any, DeleteSkillResponse]]:
"""
@ -289,20 +321,33 @@ class LiteLLMSkillsTransformationHandler:
)
if _is_async:
return self._async_delete_skill(skill_id=skill_id)
return self._async_delete_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
import asyncio
return asyncio.get_event_loop().run_until_complete(
self._async_delete_skill(skill_id=skill_id)
self._async_delete_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
)
async def _async_delete_skill(self, skill_id: str) -> DeleteSkillResponse:
async def _async_delete_skill(
self,
skill_id: str,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
) -> DeleteSkillResponse:
"""Async implementation of delete_skill."""
# Lazy import to avoid SDK dependency on proxy
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
result = await LiteLLMSkillsHandler.delete_skill(skill_id=skill_id)
result = await LiteLLMSkillsHandler.delete_skill(
skill_id=skill_id,
user_api_key_dict=user_api_key_dict,
)
return DeleteSkillResponse(
id=result["id"],
type=result.get("type", "skill_deleted"),

0
litellm/llms/nvidia_riva/__init__.py Normal file
View file

0
litellm/llms/nvidia_riva/audio_transcription/__init__.py Normal file
View file

232
litellm/llms/nvidia_riva/audio_transcription/audio_utils.py Normal file
View file

@ -0,0 +1,232 @@
"""
Audio resampling utilities for the NVIDIA Riva STT provider.
We intentionally avoid a hard dependency on ``ffmpeg`` so this works in
slim Python environments. Format coverage:
- ``soundfile`` handles wav / flac / ogg out of the box (libsndfile).
- ``audioread`` is tried for everything ``soundfile`` cannot decode (mp3,
m4a, mp4, webm, ...). This is a soft optional dependency.
If neither library can decode the input we raise a clear error instructing
the caller to convert the audio upstream.
"""
import io
import os
import tempfile
from dataclasses import dataclass
from typing import Any, Tuple, cast
from litellm.llms.nvidia_riva.audio_transcription.transformation import (
RIVA_TARGET_NUM_CHANNELS,
RIVA_TARGET_SAMPLE_RATE_HZ,
)
from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
# Keep this as Any: the module intentionally avoids importing numpy at module
# import time (optional dependency), and project-wide mypy config evaluates this
# file in contexts where conditional type aliases can degrade to "FloatArray?".
FloatArray = Any
_INSTALL_HINT = (
"Install Riva STT extras to enable automatic audio resampling: "
"`pip install 'litellm[stt-nvidia-riva]'`"
)
@dataclass
class ResampledAudio:
pcm_bytes: bytes
duration_seconds: float
sample_rate_hz: int
num_channels: int
def resample_to_riva_pcm(file_bytes: bytes) -> ResampledAudio:
"""
Decode ``file_bytes`` and produce 16 kHz mono LINEAR_PCM (int16 little
endian) suitable for streaming to Riva, plus the audio duration in
seconds (used for cost calculation when Riva does not return usage).
"""
try:
import numpy as np # type: ignore
except ImportError as e:
raise NvidiaRivaException(
status_code=500,
message=f"numpy is required for Riva audio resampling. {_INSTALL_HINT}",
) from e
samples_float, source_rate = _decode_to_float32(file_bytes)
# Downmix to mono by averaging channels.
if samples_float.ndim == 2 and samples_float.shape[1] > 1:
samples_float = samples_float.mean(axis=1)
elif samples_float.ndim == 2:
samples_float = samples_float[:, 0]
samples_float = np.asarray(samples_float, dtype=np.float32).ravel()
if source_rate != RIVA_TARGET_SAMPLE_RATE_HZ:
samples_float = _resample(
samples_float, source_rate, RIVA_TARGET_SAMPLE_RATE_HZ
)
# Clip + convert float [-1, 1] to int16 little-endian PCM.
np.clip(samples_float, -1.0, 1.0, out=samples_float)
pcm_int16 = (samples_float * 32767.0).astype("<i2")
pcm_bytes = pcm_int16.tobytes()
duration_seconds = float(pcm_int16.size) / float(RIVA_TARGET_SAMPLE_RATE_HZ)
return ResampledAudio(
pcm_bytes=pcm_bytes,
duration_seconds=duration_seconds,
sample_rate_hz=RIVA_TARGET_SAMPLE_RATE_HZ,
num_channels=RIVA_TARGET_NUM_CHANNELS,
)
def _decode_to_float32(file_bytes: bytes) -> Tuple["FloatArray", int]:
"""
Decode arbitrary audio bytes into a float32 array shaped either
``(n_samples,)`` (mono) or ``(n_samples, n_channels)`` plus the source
sample rate.
Tries ``soundfile`` first (wav/flac/ogg), then falls back to
``audioread`` for compressed formats. Raises a clear error if neither
works.
"""
import numpy as np # type: ignore
sf_error: Exception | None = None
try:
import soundfile as sf # type: ignore
with io.BytesIO(file_bytes) as buf:
data, source_rate = sf.read(buf, dtype="float32", always_2d=False)
return cast("FloatArray", data), int(source_rate)
except ImportError as e:
sf_error = e
except Exception as e:
# soundfile raises RuntimeError / LibsndfileError for formats it
# cannot decode (mp3 on older libsndfile, m4a, webm, ...).
sf_error = e
try:
import audioread # type: ignore
except ImportError as e:
raise NvidiaRivaException(
status_code=400,
message=(
"Could not decode audio for Riva STT. Install audio extras "
f"(`pip install 'litellm[stt-nvidia-riva]'`) or convert your "
f"audio to wav/flac/ogg before calling the API. "
f"Underlying error: {sf_error}"
),
) from e
# audioread backends (FFmpeg subprocess, GStreamer, Core Audio) require a
# filesystem path, so spill the bytes to a temp file. mkstemp is portable
# to Windows where re-opening a NamedTemporaryFile is not allowed.
fd, tmp_path = tempfile.mkstemp(suffix=".audio")
try:
with os.fdopen(fd, "wb") as tmp_file:
tmp_file.write(file_bytes)
try:
with audioread.audio_open(tmp_path) as src:
source_rate = int(src.samplerate)
channels = int(src.channels)
chunks = []
for buf in src:
chunks.append(np.frombuffer(buf, dtype=np.int16))
if not chunks:
raise NvidiaRivaException(
status_code=400,
message="Audio decode produced no samples.",
)
interleaved = np.concatenate(chunks).astype(np.float32) / 32768.0
if channels > 1:
interleaved = interleaved.reshape(-1, channels)
return cast("FloatArray", interleaved), source_rate
except NvidiaRivaException:
raise
except Exception as e:
raise NvidiaRivaException(
status_code=400,
message=(
"Could not decode audio for Riva STT. Convert your audio to "
f"wav/flac/ogg before calling the API. Underlying error: {e}"
),
) from e
finally:
try:
os.unlink(tmp_path)
except OSError:
pass
def _resample(
samples: "FloatArray", source_rate: int, target_rate: int
) -> "FloatArray":
"""
Resample mono float32 ``samples`` from ``source_rate`` to ``target_rate``.
Prefers high-quality polyphase resampling when ``soxr`` or ``scipy`` is
available (anti-aliased, important for downsampling 44.1/48 kHz -> 16 kHz
where naive interpolation folds high frequencies back into the speech
band). Falls back to linear interpolation if neither is installed —
acceptable for speech-only mono input but lossy for wideband content.
"""
import numpy as np # type: ignore
if source_rate == target_rate or samples.size == 0:
return samples
try:
import soxr # type: ignore
return cast(
"FloatArray",
np.asarray(
soxr.resample(samples, source_rate, target_rate), dtype=np.float32
),
)
except ImportError:
pass
try:
from math import gcd
from scipy.signal import resample_poly # type: ignore
g = gcd(int(source_rate), int(target_rate))
up = int(target_rate) // g
down = int(source_rate) // g
return cast(
"FloatArray", np.asarray(resample_poly(samples, up, down), dtype=np.float32)
)
except ImportError:
pass
return _linear_resample(samples, source_rate, target_rate)
def _linear_resample(
samples: "FloatArray", source_rate: int, target_rate: int
) -> "FloatArray":
"""Linear-interpolation fallback. See :func:`_resample` for caveats."""
import numpy as np # type: ignore
duration = samples.size / float(source_rate)
target_length = int(round(duration * target_rate))
if target_length <= 1:
return samples.astype(np.float32)
src_indices = np.linspace(0, samples.size - 1, num=target_length, dtype=np.float64)
left = np.floor(src_indices).astype(np.int64)
right = np.minimum(left + 1, samples.size - 1)
frac = (src_indices - left).astype(np.float32)
return ((1.0 - frac) * samples[left] + frac * samples[right]).astype(np.float32)

444
litellm/llms/nvidia_riva/audio_transcription/handler.py Normal file
View file

@ -0,0 +1,444 @@
"""
NVIDIA Riva STT handler.
This module bridges litellm's transcription dispatch to NVIDIA Riva's gRPC
streaming ASR API. We do *not* go through ``base_llm_http_handler`` because
Riva is gRPC-only: HTTP-shaped abstractions (``httpx.Response``,
``api_base/v1/...`` URLs, multipart bodies) do not apply.
The handler is intentionally a thin orchestration layer:
1. Resample the inbound audio to 16 kHz mono LINEAR_PCM (Riva's required
wire format).
2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs
from the structured dict produced by
:class:`NvidiaRivaAudioTranscriptionConfig`.
3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS)
vs self-hosted (any host:port, optional TLS) modes.
4. Stream the audio through Riva's ``streaming_response_generator`` and
aggregate ``is_final`` results into a single transcript.
5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed
on ``_hidden_params`` so cost calculation works.
``riva-client`` is imported lazily so ``litellm`` core remains usable
without the optional STT extras installed.
"""
import asyncio
import inspect
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from litellm.litellm_core_utils.audio_utils.utils import (
get_audio_file_name,
process_audio_file,
)
from litellm.llms.nvidia_riva.audio_transcription.audio_utils import (
resample_to_riva_pcm,
)
from litellm.llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig,
RIVA_TARGET_NUM_CHANNELS,
RIVA_TARGET_SAMPLE_RATE_HZ,
)
from litellm.llms.nvidia_riva.common_utils import (
NvidiaRivaException,
grpc_error_to_litellm_exception,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import (
Logging as LiteLLMLoggingObj,
)
# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
# NVIDIA's recommended chunk size for streaming ASR — small enough for
# responsive endpointing, large enough to keep per-RPC overhead low.
_DEFAULT_CHUNK_SAMPLES = 1600
_DEFAULT_CHUNK_BYTES = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample
_RIVA_INSTALL_HINT = (
"NVIDIA Riva client is not installed. "
"Install with `pip install 'litellm[stt-nvidia-riva]'`."
)
class NvidiaRivaAudioTranscription:
"""Sync + async entry point for Riva ASR."""
def audio_transcriptions(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
atranscription: bool = False,
provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None,
):
if provider_config is None:
provider_config = NvidiaRivaAudioTranscriptionConfig()
if atranscription:
return self.async_audio_transcriptions(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config,
)
return self._run_sync(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config,
atranscription=atranscription,
)
async def async_audio_transcriptions(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None,
) -> TranscriptionResponse:
# ``riva-client`` exposes a sync streaming generator, so we offload
# the blocking call to a worker thread to keep the event loop free.
return await asyncio.to_thread(
self._run_sync,
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params=litellm_params,
model_response=model_response,
timeout=timeout,
logging_obj=logging_obj,
api_key=api_key,
api_base=api_base,
provider_config=provider_config or NvidiaRivaAudioTranscriptionConfig(),
atranscription=True,
)
def _run_sync(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
api_key: Optional[str],
api_base: Optional[str],
provider_config: NvidiaRivaAudioTranscriptionConfig,
atranscription: bool = False,
) -> TranscriptionResponse:
if not api_base:
raise NvidiaRivaException(
status_code=400,
message=(
"NVIDIA Riva requires `api_base` (host:port for the gRPC "
"endpoint, e.g. `grpc.nvcf.nvidia.com:443` or "
"`localhost:50051`). Set it in litellm_params or via "
"NVIDIA_RIVA_API_BASE."
),
)
processed = process_audio_file(audio_file)
resampled = resample_to_riva_pcm(processed.file_content)
request_payload = provider_config.transform_audio_transcription_request(
model=model,
audio_file=audio_file,
optional_params=optional_params,
litellm_params={
**litellm_params,
"api_base": api_base,
"api_key": api_key,
},
).data
if not isinstance(request_payload, dict):
raise NvidiaRivaException(
status_code=500,
message="NvidiaRivaAudioTranscriptionConfig produced an unexpected request payload type.",
)
recognition_config_dict: Dict[str, Any] = request_payload["recognition_config"]
# The wire format is fixed by our resampler; override anything stale
# the caller passed in so the gRPC config matches the bytes we send.
recognition_config_dict["sample_rate_hertz"] = RIVA_TARGET_SAMPLE_RATE_HZ
recognition_config_dict["audio_channel_count"] = RIVA_TARGET_NUM_CHANNELS
recognition_config_dict["encoding"] = "LINEAR_PCM"
response_format = request_payload.get("response_format") or "json"
timestamp_granularities = request_payload.get("timestamp_granularities")
riva_module, riva_asr_module = _import_riva()
auth_obj = self._construct_auth(
riva_module=riva_module,
api_base=api_base,
api_key=api_key,
optional_params=optional_params,
)
recognition_config = self._build_recognition_config_proto(
riva_asr_module=riva_asr_module,
recognition_config_dict=recognition_config_dict,
)
streaming_config = riva_asr_module.StreamingRecognitionConfig(
config=recognition_config, interim_results=False
)
logging_obj.pre_call(
input=None,
api_key=api_key,
additional_args={
"api_base": api_base,
"atranscription": atranscription,
"complete_input_dict": {
"recognition_config": recognition_config_dict,
"nvcf_function_id_set": bool(
optional_params.get("nvcf_function_id")
),
"use_ssl": optional_params.get("use_ssl"),
},
},
)
try:
asr_service = riva_module.ASRService(auth_obj)
audio_chunks = self._iter_audio_chunks(resampled.pcm_bytes)
stream_kwargs: Dict[str, Any] = {
"audio_chunks": audio_chunks,
"streaming_config": streaming_config,
}
# Forward the deadline so the stream cannot block forever if the
# server stalls. Older riva-client versions do not accept a
# ``timeout`` kwarg, so pass it only when supported.
if timeout is not None and self._supports_timeout_kwarg(
asr_service.streaming_response_generator
):
stream_kwargs["timeout"] = float(timeout)
stream = asr_service.streaming_response_generator(**stream_kwargs)
final_results = self._collect_final_results(stream)
except NvidiaRivaException:
raise
except Exception as e:
raise grpc_error_to_litellm_exception(e) from e
transcription = NvidiaRivaAudioTranscriptionConfig.build_transcription_response(
final_results=final_results,
response_format=response_format,
duration_seconds=resampled.duration_seconds,
timestamp_granularities=timestamp_granularities,
)
stringified_response = dict(transcription)
logging_obj.post_call(
input=get_audio_file_name(audio_file),
api_key=api_key,
additional_args={"complete_input_dict": recognition_config_dict},
original_response=stringified_response,
)
hidden_params = {
"model": model,
"custom_llm_provider": "nvidia_riva",
"audio_transcription_duration": resampled.duration_seconds,
}
final_response: TranscriptionResponse = convert_to_model_response_object( # type: ignore
response_object=stringified_response,
model_response_object=model_response,
hidden_params=hidden_params,
response_type="audio_transcription",
)
return final_response
def _construct_auth(
self,
riva_module: Any,
api_base: str,
api_key: Optional[str],
optional_params: dict,
) -> Any:
"""
Build a ``riva.client.Auth`` object.
- When ``nvcf_function_id`` is provided we attach the NVCF
``function-id`` and bearer ``authorization`` metadata, and default
``use_ssl`` to True (NVCF endpoints are TLS-only).
- Otherwise (self-hosted) we default ``use_ssl`` to False but still
honor an explicit override — self-hosted Riva behind an ingress
with TLS termination is a real deployment topology.
"""
nvcf_function_id = optional_params.get("nvcf_function_id")
use_ssl_override = optional_params.get("use_ssl")
use_ssl = (
bool(use_ssl_override)
if use_ssl_override is not None
else bool(nvcf_function_id)
)
metadata: List[Tuple[str, str]] = []
if nvcf_function_id:
metadata.append(("function-id", str(nvcf_function_id)))
if api_key:
metadata.append(("authorization", f"Bearer {api_key}"))
try:
return riva_module.Auth(
uri=api_base, use_ssl=use_ssl, metadata_args=metadata
)
except TypeError:
# Older riva-client signatures used positional-only args.
return riva_module.Auth(None, use_ssl, api_base, metadata)
def _build_recognition_config_proto(
self, riva_asr_module: Any, recognition_config_dict: Dict[str, Any]
):
encoding_name = (
recognition_config_dict.get("encoding") or "LINEAR_PCM"
).upper()
encoding_enum = getattr(
riva_asr_module.AudioEncoding,
encoding_name,
riva_asr_module.AudioEncoding.LINEAR_PCM,
)
config = riva_asr_module.RecognitionConfig(
encoding=encoding_enum,
sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]),
language_code=recognition_config_dict["language_code"],
audio_channel_count=int(recognition_config_dict["audio_channel_count"]),
enable_automatic_punctuation=bool(
recognition_config_dict.get("enable_automatic_punctuation", True)
),
enable_word_time_offsets=bool(
recognition_config_dict.get("enable_word_time_offsets", False)
),
max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)),
model=recognition_config_dict.get("model", "") or "",
verbatim_transcripts=bool(
recognition_config_dict.get("verbatim_transcripts", False)
),
profanity_filter=bool(
recognition_config_dict.get("profanity_filter", False)
),
)
endpointing = recognition_config_dict.get("endpointing_config")
if isinstance(endpointing, dict) and endpointing:
try:
ep = riva_asr_module.EndpointingConfig(**endpointing)
config.endpointing_config.CopyFrom(ep)
except Exception:
# If the user supplied an unknown EndpointingConfig field
# (older Riva server), fall back to Riva's defaults rather
# than failing the whole request.
pass
return config
@staticmethod
def _supports_timeout_kwarg(callable_obj: Any) -> bool:
try:
sig = inspect.signature(callable_obj)
except (TypeError, ValueError):
return False
params = sig.parameters
if "timeout" in params:
return True
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
@staticmethod
def _iter_audio_chunks(pcm_bytes: bytes):
for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES):
chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES]
if not chunk:
continue
yield chunk
@staticmethod
def _collect_final_results(stream) -> List[Dict[str, Any]]:
"""
Walk the gRPC stream, ignore empty / non-final chunks, and return a
list of normalized final-result dicts. Matching the user's note: the
``id`` blocks with no ``results`` are streaming heartbeats and must
be skipped.
"""
final_results: List[Dict[str, Any]] = []
for response in stream:
results = getattr(response, "results", None) or []
for result in results:
if not getattr(result, "is_final", False):
continue
alternatives = getattr(result, "alternatives", None) or []
if not alternatives:
continue
top = alternatives[0]
transcript = getattr(top, "transcript", "") or ""
words_proto = getattr(top, "words", None) or []
words = []
for word in words_proto:
words.append(
{
"word": getattr(word, "word", ""),
"start_time_ms": int(getattr(word, "start_time", 0) or 0),
"end_time_ms": int(getattr(word, "end_time", 0) or 0),
}
)
final_results.append({"transcript": transcript, "words": words})
return final_results
def _import_riva():
"""
Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``.
We try the SDK first (preferred) and fall back to importing the proto
module separately when the SDK packaging changes between versions.
"""
try:
import riva.client as riva_client # type: ignore
except ImportError as e:
raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
riva_asr_module = riva_client
if not hasattr(riva_asr_module, "RecognitionConfig"):
try:
import riva.client.proto.riva_asr_pb2 as riva_asr_pb2 # type: ignore
riva_asr_module = riva_asr_pb2
except ImportError as e:
raise NvidiaRivaException(
status_code=500, message=_RIVA_INSTALL_HINT
) from e
return riva_client, riva_asr_module

284
litellm/llms/nvidia_riva/audio_transcription/transformation.py Normal file
View file

@ -0,0 +1,284 @@
"""
Translates from OpenAI's `/v1/audio/transcriptions` to NVIDIA Riva's gRPC
streaming recognize API.
Riva is gRPC-only, so unlike most providers in this directory the request
"transformation" produced here is a structured dict consumed directly by the
gRPC handler (rather than HTTP form-data). The handler builds Riva
``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs from this
dict at call time.
Reference: https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html
"""
from typing import Any, Dict, List, Optional, Union
from httpx import Headers, Response
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import (
AllMessageValues,
OpenAIAudioTranscriptionOptionalParams,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from ...base_llm.audio_transcription.transformation import (
AudioTranscriptionRequestData,
BaseAudioTranscriptionConfig,
)
from ..common_utils import NvidiaRivaException
# Riva expects a fixed wire format for the audio chunks we stream in.
RIVA_TARGET_SAMPLE_RATE_HZ = 16000
RIVA_TARGET_NUM_CHANNELS = 1
RIVA_TARGET_ENCODING = "LINEAR_PCM"
class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
"""
Config for NVIDIA Riva ASR (gRPC).
Supports both NVCF-hosted (``api_base=grpc.nvcf.nvidia.com:443`` +
``nvcf_function_id``) and self-hosted deployments (any ``host:port``,
optional TLS via ``use_ssl``).
"""
def get_supported_openai_params(
self, model: str
) -> List[OpenAIAudioTranscriptionOptionalParams]:
# Riva natively understands language + word timestamps.
# `response_format` is honored at response-shaping time in the handler.
return ["language", "response_format", "timestamp_granularities"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
for key, value in non_default_params.items():
if value is None:
continue
if key == "language":
optional_params["language_code"] = self._normalize_language_code(value)
elif key == "timestamp_granularities":
# OpenAI accepts ["word"], ["segment"], or both. Riva only
# natively exposes word timing, so we toggle it on whenever
# "word" is requested. Segment timing is reconstructed in the
# response transformer.
if isinstance(value, list) and "word" in value:
optional_params["enable_word_time_offsets"] = True
optional_params["timestamp_granularities"] = value
elif key == "response_format":
# Stored verbatim; consumed by transform_audio_transcription_response.
optional_params["response_format"] = value
else:
optional_params[key] = value
return optional_params
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return NvidiaRivaException(
message=error_message, status_code=status_code, headers=headers
)
def transform_audio_transcription_request(
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
) -> AudioTranscriptionRequestData:
"""
Build a structured dict that the gRPC handler consumes. We do *not*
construct protobufs here, so this module remains importable without
``nvidia-riva-client`` being installed (matching how other providers
defer SDK imports to handler-call time).
"""
recognition_config = self._build_recognition_config_dict(
model=model,
optional_params=optional_params,
)
endpointing_config = self._build_endpointing_config_dict(optional_params)
if endpointing_config is not None:
recognition_config["endpointing_config"] = endpointing_config
request_payload: Dict[str, Any] = {
"recognition_config": recognition_config,
"response_format": optional_params.get("response_format") or "json",
"timestamp_granularities": optional_params.get("timestamp_granularities"),
}
return AudioTranscriptionRequestData(data=request_payload, files=None)
def transform_audio_transcription_response(
self,
raw_response: Response,
) -> TranscriptionResponse:
# Not used: Riva responses come from a gRPC stream, not an httpx
# response. The handler calls _build_transcription_response directly.
raise NotImplementedError(
"NvidiaRivaAudioTranscriptionConfig.transform_audio_transcription_response "
"is not used. The handler builds the TranscriptionResponse directly "
"from Riva's gRPC streaming results."
)
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
# gRPC auth is constructed in the handler, not via HTTP headers.
return headers
def _build_recognition_config_dict(
self, model: str, optional_params: dict
) -> Dict[str, Any]:
"""
Build the Riva ``RecognitionConfig`` shape as a plain dict.
``model`` is intentionally left empty when the user has not supplied
``riva_model_name``. Riva auto-selects the right deployment from
``language_code`` + ``sample_rate_hertz``. NVIDIA's internal
deployment names (e.g. ``parakeet-1.1b-en-US-asr-streaming-...``)
change across NIM versions, regions, and self-hosted builds, so
hardcoding any name here would break unpredictably.
"""
return {
"language_code": optional_params.get("language_code", "en-US"),
"sample_rate_hertz": optional_params.get(
"sample_rate_hertz", RIVA_TARGET_SAMPLE_RATE_HZ
),
"encoding": optional_params.get("encoding", RIVA_TARGET_ENCODING),
"audio_channel_count": optional_params.get(
"audio_channel_count", RIVA_TARGET_NUM_CHANNELS
),
"enable_automatic_punctuation": optional_params.get(
"enable_automatic_punctuation", True
),
"enable_word_time_offsets": bool(
optional_params.get("enable_word_time_offsets", False)
),
"max_alternatives": optional_params.get("max_alternatives", 1),
"model": optional_params.get("riva_model_name", ""),
"verbatim_transcripts": optional_params.get("verbatim_transcripts", False),
"profanity_filter": optional_params.get("profanity_filter", False),
}
def _build_endpointing_config_dict(
self, optional_params: dict
) -> Optional[Dict[str, Any]]:
"""
Translate an OpenAI-style ``chunking_strategy`` into Riva's
``EndpointingConfig`` shape, or pass through an explicit
``endpointing_config`` dict.
Returns ``None`` when neither is provided so Riva uses its built-in
VAD defaults.
"""
explicit = optional_params.get("endpointing_config")
if isinstance(explicit, dict):
return dict(explicit)
chunking = optional_params.get("chunking_strategy")
if chunking in (None, "auto"):
return None
if isinstance(chunking, dict) and chunking.get("type") == "server_vad":
config: Dict[str, Any] = {}
if "threshold" in chunking:
threshold = float(chunking["threshold"])
config["start_threshold"] = threshold
config["stop_threshold"] = threshold
if "silence_duration_ms" in chunking:
config["stop_history"] = int(chunking["silence_duration_ms"])
if "prefix_padding_ms" in chunking:
config["stop_history_eou"] = int(chunking["prefix_padding_ms"])
return config or None
return None
@staticmethod
def _normalize_language_code(language: str) -> str:
"""
OpenAI accepts bare ISO-639 codes like ``en``; Riva requires BCP-47
like ``en-US``. Normalize the most common bare codes; pass through
anything that already looks like BCP-47.
"""
if not isinstance(language, str) or not language:
return "en-US"
if "-" in language:
return language
bare_to_bcp47 = {
"en": "en-US",
"es": "es-ES",
"de": "de-DE",
"fr": "fr-FR",
"it": "it-IT",
"pt": "pt-BR",
"ja": "ja-JP",
"ko": "ko-KR",
"zh": "zh-CN",
"ru": "ru-RU",
"hi": "hi-IN",
"ar": "ar-SA",
}
return bare_to_bcp47.get(language.lower(), language)
@staticmethod
def build_transcription_response(
final_results: List[Dict[str, Any]],
response_format: str,
duration_seconds: Optional[float],
timestamp_granularities: Optional[List[str]],
) -> TranscriptionResponse:
"""
Aggregate a list of normalized "final result" dicts into a
``TranscriptionResponse`` shaped for the requested ``response_format``.
Each entry in ``final_results`` is expected to look like::
{
"transcript": str,
"words": [{"word": str, "start_time_ms": int, "end_time_ms": int}, ...],
}
which the handler produces by walking the gRPC stream and keeping
only ``result.is_final`` entries (empty/non-final chunks are
ignored).
"""
full_transcript = "".join(
(item.get("transcript") or "") for item in final_results
).strip()
response = TranscriptionResponse(text=full_transcript)
response["task"] = "transcribe"
if response_format == "verbose_json":
words: List[Dict[str, Any]] = []
if timestamp_granularities and "word" in timestamp_granularities:
for item in final_results:
for word in item.get("words", []) or []:
words.append(
{
"word": word.get("word", ""),
"start": (float(word.get("start_time_ms", 0)) / 1000.0),
"end": float(word.get("end_time_ms", 0)) / 1000.0,
}
)
if words:
response["words"] = words
if duration_seconds is not None:
response["duration"] = duration_seconds
return response

92
litellm/llms/nvidia_riva/common_utils.py Normal file
View file

@ -0,0 +1,92 @@
"""
Common utilities and exceptions for the NVIDIA Riva STT provider
"""
from typing import Any, Optional
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class NvidiaRivaException(BaseLLMException):
"""
Exception raised for NVIDIA Riva (gRPC) errors.
``status_code`` is an HTTP-equivalent code derived from the underlying
gRPC ``StatusCode`` (when available) so that litellm's existing error
classifiers (RateLimitError, AuthenticationError, etc.) keep working.
"""
pass
# Mapping from grpc.StatusCode.name -> equivalent HTTP status code.
# Kept as a plain dict (rather than importing grpc enums) so this module is
# importable without grpc installed.
_GRPC_STATUS_CODE_TO_HTTP: dict = {
"OK": 200,
"CANCELLED": 499,
"UNKNOWN": 500,
"INVALID_ARGUMENT": 400,
"DEADLINE_EXCEEDED": 504,
"NOT_FOUND": 404,
"ALREADY_EXISTS": 409,
"PERMISSION_DENIED": 403,
"RESOURCE_EXHAUSTED": 429,
"FAILED_PRECONDITION": 400,
"ABORTED": 409,
"OUT_OF_RANGE": 400,
"UNIMPLEMENTED": 501,
"INTERNAL": 500,
"UNAVAILABLE": 503,
"DATA_LOSS": 500,
"UNAUTHENTICATED": 401,
}
def _extract_grpc_status_name(error: Any) -> Optional[str]:
"""
Best-effort extraction of a gRPC StatusCode name from an arbitrary error.
Works for ``grpc.RpcError`` instances (which expose ``.code()``) as well
as plain exceptions whose string representation contains a status name.
"""
code_fn = getattr(error, "code", None)
if callable(code_fn):
try:
code = code_fn()
except Exception:
code = None
name = getattr(code, "name", None)
if isinstance(name, str):
return name
return None
def _extract_grpc_details(error: Any) -> Optional[str]:
"""Best-effort extraction of a human-readable detail string from a gRPC error."""
details_fn = getattr(error, "details", None)
if callable(details_fn):
try:
details = details_fn()
except Exception:
details = None
if isinstance(details, str) and details:
return details
return None
def grpc_error_to_litellm_exception(error: Exception) -> NvidiaRivaException:
"""
Convert a gRPC error (or any exception raised from the Riva client) into
a ``NvidiaRivaException`` with an appropriate HTTP-equivalent status code.
"""
status_name = _extract_grpc_status_name(error)
http_status = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500)
detail = _extract_grpc_details(error) or str(error)
message = (
f"NVIDIA Riva gRPC error ({status_name}): {detail}"
if status_name
else f"NVIDIA Riva error: {detail}"
)
return NvidiaRivaException(status_code=http_status, message=message)

46
litellm/llms/openai/realtime/handler.py
View file

@ -6,12 +6,15 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
from typing import Any, Optional, cast
from litellm._logging import _redact_string
from litellm._logging import _redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.types.realtime import RealtimeQueryParams
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ....litellm_core_utils.realtime_streaming import RealTimeStreaming
from ....litellm_core_utils.realtime_streaming import (
RealTimeStreaming,
client_sent_openai_beta_realtime_header,
)
from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
from ..openai import OpenAIChatCompletion
@ -33,21 +36,24 @@ class OpenAIRealtime(OpenAIChatCompletion):
"""
return "https://api.openai.com/"
def _get_additional_headers(self, api_key: str) -> dict:
def _get_additional_headers(
self,
api_key: str,
*,
openai_beta_realtime: bool = False,
) -> dict:
"""
Get additional headers beyond Authorization.
Override this in subclasses to customize headers (e.g., remove OpenAI-Beta).
Headers for the upstream OpenAI Realtime WebSocket.
Args:
api_key: API key for authentication
Returns:
Dictionary of additional headers
When the client sent ``OpenAI-Beta: realtime=v1`` on the proxy WebSocket,
``openai_beta_realtime`` is True and the same header is forwarded upstream
so the legacy beta API is used. GA clients omit that header on the client
connection and must send GA-shaped ``session.update`` payloads.
"""
return {
"Authorization": f"Bearer {api_key}",
"OpenAI-Beta": "realtime=v1",
}
headers: dict = {"Authorization": f"Bearer {api_key}"}
if openai_beta_realtime:
headers["OpenAI-Beta"] = "realtime=v1"
return headers
def _get_ssl_config(self, url: str) -> Any:
"""
@ -120,8 +126,16 @@ class OpenAIRealtime(OpenAIChatCompletion):
# Get provider-specific SSL configuration
ssl_config = self._get_ssl_config(url)
# Get provider-specific headers
headers = self._get_additional_headers(api_key)
openai_beta_realtime = client_sent_openai_beta_realtime_header(websocket)
if not openai_beta_realtime:
verbose_logger.debug(
"OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). "
"If your client expects beta event names, add 'OpenAI-Beta: realtime=v1' "
"to the WebSocket headers sent to the LiteLLM proxy."
)
headers = self._get_additional_headers(
api_key, openai_beta_realtime=openai_beta_realtime
)
# Log a masked request preview consistent with other endpoints.
logging_obj.pre_call(

51
litellm/llms/sagemaker/common_utils.py
View file

@ -9,7 +9,27 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import StreamingChatCompletionChunk
_response_stream_shape_cache = None
def _load_sagemaker_response_stream_shape():
try:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
service_dict = loader.load_service_model("sagemaker-runtime", "service-2")
return ServiceModel(service_dict).shape_for(
"InvokeEndpointWithResponseStreamOutput"
)
except Exception as e:
verbose_logger.warning(
"litellm: could not pre-load sagemaker-runtime response stream shape "
"— SageMaker event-stream decoding will be unavailable. Error: %s",
e,
)
return None
SAGEMAKER_RESPONSE_STREAM_SHAPE = _load_sagemaker_response_stream_shape()
class SagemakerError(BaseLLMException):
@ -187,8 +207,18 @@ class AWSEventStreamDecoder:
verbose_logger.error(f"Final error parsing accumulated JSON: {e}")
def _parse_message_from_event(self, event) -> Optional[str]:
if SAGEMAKER_RESPONSE_STREAM_SHAPE is None:
raise SagemakerError(
status_code=500,
message=(
"SageMaker event-stream shape could not be loaded from botocore. "
"Ensure botocore is correctly installed."
),
)
response_dict = event.to_response_dict()
parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
parsed_response = self.parser.parse(
response_dict, SAGEMAKER_RESPONSE_STREAM_SHAPE
)
if response_dict["status_code"] != 200:
raise ValueError(f"Bad response code, expected 200: {response_dict}")
@ -204,20 +234,3 @@ class AWSEventStreamDecoder:
return None
return chunk.decode() # type: ignore[no-any-return]
def get_response_stream_shape():
global _response_stream_shape_cache
if _response_stream_shape_cache is None:
from botocore.loaders import Loader
from botocore.model import ServiceModel
loader = Loader()
sagemaker_service_dict = loader.load_service_model(
"sagemaker-runtime", "service-2"
)
sagemaker_service_model = ServiceModel(sagemaker_service_dict)
_response_stream_shape_cache = sagemaker_service_model.shape_for(
"InvokeEndpointWithResponseStreamOutput"
)
return _response_stream_shape_cache

7
litellm/llms/xai/realtime/handler.py
View file

@ -28,7 +28,12 @@ class XAIRealtime(OpenAIRealtime):
"""xAI uses a different API base URL."""
return XAI_API_BASE
def _get_additional_headers(self, api_key: str) -> dict:
def _get_additional_headers(
self,
api_key: str,
*,
openai_beta_realtime: bool = False,
) -> dict:
"""
xAI does NOT require the OpenAI-Beta header.
Only send Authorization header.

43
litellm/main.py
View file

@ -211,6 +211,12 @@ from .llms.oobabooga.chat import oobabooga
from .llms.openai.completion.handler import OpenAITextCompletion
from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler
from .llms.openai.openai import OpenAIChatCompletion
from .llms.nvidia_riva.audio_transcription.handler import (
NvidiaRivaAudioTranscription,
)
from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig,
)
from .llms.openai.transcriptions.handler import OpenAIAudioTranscription
from .llms.openai_like.chat.handler import OpenAILikeChatHandler
from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler
@ -266,6 +272,7 @@ from .types.utils import (
openai_chat_completions = OpenAIChatCompletion()
openai_text_completions = OpenAITextCompletion()
openai_audio_transcriptions = OpenAIAudioTranscription()
nvidia_riva_audio_transcriptions = NvidiaRivaAudioTranscription()
openai_image_variations = OpenAIImageVariationsHandler()
groq_chat_completions = GroqChatCompletion()
sap_gen_ai_hub_chat_completions = GenAIHubOrchestration()
@ -1452,14 +1459,14 @@ def completion( # type: ignore # noqa: PLR0915
if eos_token:
custom_prompt_dict[model]["eos_token"] = eos_token
if kwargs.get("model_file_id_mapping"):
messages = update_messages_with_model_file_ids(
messages=messages,
model_id=kwargs.get("model_info", {}).get("id", None),
model_file_id_mapping=cast(
Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping")
),
)
messages = update_messages_with_model_file_ids(
messages=messages,
model_id=kwargs.get("model_info", {}).get("id", None),
model_file_id_mapping=cast(
Dict[str, Dict[str, str]],
kwargs.get("model_file_id_mapping") or {},
),
)
provider_config: Optional[BaseConfig] = None
if custom_llm_provider is not None and custom_llm_provider in [
@ -6605,6 +6612,26 @@ def transcription(
litellm_params=litellm_params_dict,
shared_session=shared_session,
)
elif custom_llm_provider == "nvidia_riva":
# NVIDIA Riva is gRPC-based, not HTTP. It has its own dedicated handler
# rather than going through base_llm_http_handler.
response = nvidia_riva_audio_transcriptions.audio_transcriptions(
model=model,
audio_file=file,
optional_params=optional_params,
litellm_params=litellm_params_dict,
model_response=model_response,
atranscription=atranscription,
timeout=timeout,
logging_obj=litellm_logging_obj,
api_base=api_base,
api_key=api_key,
provider_config=(
provider_config
if isinstance(provider_config, NvidiaRivaAudioTranscriptionConfig)
else None
),
)
elif provider_config is not None:
response = base_llm_http_handler.audio_transcriptions(
model=model,

55
litellm/model_prices_and_context_window_backup.json
View file

@ -28874,6 +28874,19 @@
"mode": "chat",
"output_cost_per_token": 0.0
},
"sambanova/MiniMax-M2.7": {
"input_cost_per_token": 3e-07,
"litellm_provider": "sambanova",
"max_input_tokens": 204800,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.sambanova.ai/plans/pricing",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"sambanova/DeepSeek-R1": {
"input_cost_per_token": 5e-06,
"litellm_provider": "sambanova",
@ -34927,6 +34940,48 @@
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-4.3": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-4.3-latest": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_above_200k_tokens": 2.5e-06,
"litellm_provider": "xai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-beta": {
"input_cost_per_token": 5e-06,
"litellm_provider": "xai",

196
litellm/proxy/_experimental/mcp_server/auth/token_exchange.py Normal file
View file

@ -0,0 +1,196 @@
"""
OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers.
Exchanges a user's incoming JWT (subject_token) for a scoped access token
at an IDP's token exchange endpoint. The exchanged token is then used to
authenticate requests to the upstream MCP server.
See: https://datatracker.ietf.org/doc/html/rfc8693
"""
import asyncio
import hashlib
import weakref
from typing import TYPE_CHECKING, Dict, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import (
MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
from litellm.types.mcp_server.mcp_server_manager import MCPServer
# RFC 8693 grant type constant
TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
class TokenExchangeHandler:
"""Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers.
Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so
repeated calls with the same user token skip the IDP round-trip.
"""
def __init__(self) -> None:
self._cache = InMemoryCache(
max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
)
# WeakValueDictionary so locks are GC'd once no coroutine holds a reference,
# preventing unbounded growth with many rotating user tokens.
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
weakref.WeakValueDictionary()
)
def _get_lock(self, cache_key: str) -> asyncio.Lock:
lock = self._locks.get(cache_key)
if lock is None:
lock = asyncio.Lock()
self._locks[cache_key] = lock
return lock
@staticmethod
def _cache_key(subject_token: str, server_id: str) -> str:
raw = f"{subject_token}:{server_id}"
return hashlib.sha256(raw.encode()).hexdigest()
async def exchange_token(
self,
subject_token: str,
server: "MCPServer",
) -> str:
"""Exchange *subject_token* for a scoped access token.
Returns the exchanged ``access_token`` string (suitable for a
``Bearer`` header).
Raises ``ValueError`` on configuration or IDP errors.
"""
cache_key = self._cache_key(subject_token, server.server_id)
# Fast path
cached = self._cache.get_cache(cache_key)
if cached is not None:
return cached
# Slow path — one exchange at a time per (user, server) pair
async with self._get_lock(cache_key):
cached = self._cache.get_cache(cache_key)
if cached is not None:
return cached
token, ttl = await self._do_exchange(subject_token, server)
self._cache.set_cache(cache_key, token, ttl=ttl)
return token
async def _do_exchange(
self,
subject_token: str,
server: "MCPServer",
) -> Tuple[str, int]:
"""POST to the token exchange endpoint with RFC 8693 parameters.
Returns ``(access_token, ttl_seconds)``.
"""
endpoint = server.token_exchange_endpoint or server.token_url
if not endpoint:
raise ValueError(
f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
f"but no token_exchange_endpoint or token_url configured"
)
if not server.client_id or not server.client_secret:
raise ValueError(
f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
f"but missing client_id or client_secret"
)
data: Dict[str, str] = {
"grant_type": TOKEN_EXCHANGE_GRANT_TYPE,
"subject_token": subject_token,
"subject_token_type": server.subject_token_type
or DEFAULT_SUBJECT_TOKEN_TYPE,
"client_id": server.client_id,
"client_secret": server.client_secret,
}
if server.audience:
data["audience"] = server.audience
if server.scopes:
data["scope"] = " ".join(server.scopes)
verbose_logger.debug(
"Exchanging token for MCP server %s at %s (audience=%s)",
server.server_id,
endpoint,
server.audience,
)
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
try:
response = await client.post(endpoint, data=data)
response.raise_for_status()
except httpx.HTTPStatusError as exc:
verbose_logger.debug(
"Token exchange IDP error for MCP server %s (status %d)",
server.server_id,
exc.response.status_code,
)
raise ValueError(
f"Token exchange for MCP server '{server.server_id}' "
f"failed with status {exc.response.status_code}"
) from exc
body = response.json()
if not isinstance(body, dict):
raise ValueError(
f"Token exchange response for MCP server '{server.server_id}' "
f"returned non-object JSON (got {type(body).__name__})"
)
access_token = body.get("access_token")
if not access_token:
raise ValueError(
f"Token exchange response for MCP server '{server.server_id}' "
f"missing 'access_token'"
)
raw_expires_in = body.get("expires_in")
try:
expires_in = (
int(raw_expires_in)
if raw_expires_in is not None
else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
)
except (TypeError, ValueError):
expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
ttl = max(
expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
)
verbose_logger.info(
"Token exchange succeeded for MCP server %s (expires in %ds)",
server.server_id,
expires_in,
)
return access_token, ttl
def invalidate(self, subject_token: str, server_id: str) -> None:
"""Remove a cached exchanged token (e.g. after a 401)."""
cache_key = self._cache_key(subject_token, server_id)
self._cache.delete_cache(cache_key)
# Module-level singleton
mcp_token_exchange_handler = TokenExchangeHandler()

174
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -411,6 +411,15 @@ class MCPServerManager:
aws_role_name=server_config.get("aws_role_name", None),
aws_session_name=server_config.get("aws_session_name", None),
instructions=server_config.get("instructions", None),
# Token Exchange (OBO) fields
token_exchange_endpoint=server_config.get(
"token_exchange_endpoint", None
),
audience=server_config.get("audience", None),
subject_token_type=server_config.get(
"subject_token_type",
"urn:ietf:params:oauth:token-type:access_token",
),
)
self._assign_unique_short_prefix(new_server)
self.config_mcp_servers[server_id] = new_server
@ -765,10 +774,23 @@ class MCPServerManager:
aws_role_name=aws_creds.get("aws_role_name"),
aws_session_name=aws_creds.get("aws_session_name"),
instructions=mcp_server.instructions,
# Token Exchange (OBO) fields — read from credentials JSON blob
token_exchange_endpoint=(
credentials_dict.get("token_exchange_endpoint")
if credentials_dict
else None
),
audience=(credentials_dict.get("audience") if credentials_dict else None),
subject_token_type=(
credentials_dict.get("subject_token_type") if credentials_dict else None
)
or "urn:ietf:params:oauth:token-type:access_token",
)
return new_server
async def _maybe_register_openapi_tools(self, server: MCPServer):
async def _maybe_register_openapi_tools(
self, server: MCPServer, *, initialize_mapping: bool = True
):
"""Register OpenAPI tools if the server has a spec_path configured."""
if server.spec_path:
verbose_logger.info(
@ -779,7 +801,8 @@ class MCPServerManager:
server=server,
base_url=server.url or "",
)
self.initialize_tool_name_to_mcp_server_name_mapping()
if initialize_mapping:
self.initialize_tool_name_to_mcp_server_name_mapping()
async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
@ -1136,6 +1159,29 @@ class MCPServerManager:
#########################################################
# Methods that call the upstream MCP servers
#########################################################
@staticmethod
def _extract_bearer_token(
oauth2_headers: Optional[Dict[str, str]],
raw_headers: Optional[Dict[str, str]],
) -> Optional[str]:
"""Extract the bare Bearer token from oauth2_headers or raw_headers.
Returns the token string without the ``Bearer `` prefix, or ``None``
if no Authorization header is found.
"""
auth_value: Optional[str] = None
if oauth2_headers and "Authorization" in oauth2_headers:
auth_value = oauth2_headers["Authorization"]
elif raw_headers:
# raw_headers may have lowercase keys depending on the ASGI server
normalized = {k.lower(): v for k, v in raw_headers.items()}
auth_value = normalized.get("authorization")
if auth_value:
if auth_value.startswith("Bearer "):
return auth_value[len("Bearer ") :]
return auth_value
return None
def _build_stdio_env(
self,
server: MCPServer,
@ -1169,25 +1215,30 @@ class MCPServerManager:
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
extra_headers: Optional[Dict[str, str]] = None,
stdio_env: Optional[Dict[str, str]] = None,
subject_token: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
Auth resolution (single place for all auth logic):
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
2. OAuth2 Token Exchange (OBO) — exchange user token for scoped token
3. OAuth2 client_credentials token — auto-fetched and cached
4. ``server.authentication_token`` — static token from config/DB
Args:
server: The server configuration.
mcp_auth_header: Optional per-request auth override.
extra_headers: Additional headers to forward.
stdio_env: Environment variables for stdio transport.
subject_token: Optional user JWT for token exchange (OBO) flow.
Returns:
Configured MCP client instance.
"""
auth_value = await resolve_mcp_auth(server, mcp_auth_header)
auth_value = await resolve_mcp_auth(
server, mcp_auth_header, subject_token=subject_token
)
transport = server.transport or MCPTransport.sse
@ -1978,7 +2029,11 @@ class MCPServerManager:
_SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
def _assign_unique_short_prefix(self, server: MCPServer) -> None:
def _assign_unique_short_prefix(
self,
server: MCPServer,
registry: Optional[Dict[str, MCPServer]] = None,
) -> None:
"""Resolve and cache a collision-free short tool prefix on ``server``.
Called at registration time for every MCP server entering the
@ -2002,7 +2057,8 @@ class MCPServerManager:
return
used: Dict[str, str] = {}
for other in self.get_registry().values():
registry_for_collision_check = registry or self.get_registry()
for other in registry_for_collision_check.values():
if other.server_id == server.server_id:
continue
if other.short_prefix:
@ -2534,9 +2590,12 @@ class MCPServerManager:
if server_auth_header is None:
server_auth_header = mcp_auth_header
# oauth2 headers
# Extract subject token for OAuth2 Token Exchange (OBO) flow
subject_token: Optional[str] = None
extra_headers: Optional[Dict[str, str]] = None
if mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
elif mcp_server.auth_type == MCPAuth.oauth2:
if mcp_server.has_client_credentials:
# For M2M OAuth servers, Authorization must come from token fetch.
extra_headers = None
@ -2604,6 +2663,7 @@ class MCPServerManager:
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
stdio_env=stdio_env,
subject_token=subject_token,
)
call_tool_params = MCPCallToolRequestParams(
@ -2916,46 +2976,72 @@ class MCPServerManager:
# against the *full* set so dedup is deterministic regardless of
# iteration order.
for server in db_mcp_servers:
existing_server = previous_registry.get(server.server_id)
try:
existing_server = previous_registry.get(server.server_id)
if (
existing_server is not None
and existing_server.updated_at is not None
and server.updated_at is not None
and existing_server.updated_at == server.updated_at
):
# Re-use existing server instance to avoid re-running build_mcp_server_from_table()
# which can perform network discovery for OAuth2 servers.
new_registry[server.server_id] = existing_server
continue
if (
existing_server is not None
and existing_server.updated_at is not None
and server.updated_at is not None
and existing_server.updated_at == server.updated_at
):
# Re-use existing server instance to avoid re-running build_mcp_server_from_table()
# which can perform network discovery for OAuth2 servers.
new_registry[server.server_id] = existing_server
continue
_warn_on_server_name_fields(
server_id=server.server_id,
alias=getattr(server, "alias", None),
server_name=getattr(server, "server_name", None),
)
verbose_logger.debug(
f"Building server from DB: {server.server_id} ({server.server_name})"
)
new_server = await self.build_mcp_server_from_table(server)
# Carry the cached short_prefix from the previous registry entry
# (if any) so the prefix is stable across reloads.
if existing_server is not None and existing_server.short_prefix:
new_server.short_prefix = existing_server.short_prefix
new_registry[server.server_id] = new_server
_warn_on_server_name_fields(
server_id=server.server_id,
alias=getattr(server, "alias", None),
server_name=getattr(server, "server_name", None),
)
verbose_logger.debug(
f"Building server from DB: {server.server_id} ({server.server_name})"
)
new_server = await self.build_mcp_server_from_table(server)
# Carry the cached short_prefix from the previous registry entry
# (if any) so the prefix is stable across reloads.
if existing_server is not None and existing_server.short_prefix:
new_server.short_prefix = existing_server.short_prefix
new_registry[server.server_id] = new_server
except Exception as e:
verbose_logger.exception(
"Skipping MCP server %s (%s) during DB reload: %s",
server.server_id,
getattr(server, "alias", None),
e,
)
# Swap in the new registry first so _assign_unique_short_prefix
# sees the complete set when checking for collisions.
self.registry = new_registry
for new_server in new_registry.values():
self._assign_unique_short_prefix(new_server)
# Register OpenAPI tools *after* the final short prefix is assigned
# so the tools are stored in the global registry under the same
# prefix that lookups will use.
await self._maybe_register_openapi_tools(new_server)
# Assign short prefixes against the full candidate set without
# publishing the staged registry to concurrent callers.
registered_registry: Dict[str, MCPServer] = {}
registered_openapi_tools = False
for server_id, new_server in new_registry.items():
try:
self._assign_unique_short_prefix(new_server, registry=new_registry)
# Register OpenAPI tools *after* the final short prefix is assigned
# so the tools are stored in the global registry under the same
# prefix that lookups will use.
await self._maybe_register_openapi_tools(
new_server, initialize_mapping=False
)
registered_registry[server_id] = new_server
if new_server.spec_path:
registered_openapi_tools = True
except Exception as e:
verbose_logger.exception(
"Skipping MCP server %s (%s) during DB reload: %s",
new_server.server_id,
getattr(new_server, "alias", None),
e,
)
self.registry = registered_registry
if registered_openapi_tools:
self.initialize_tool_name_to_mcp_server_name_mapping()
verbose_logger.debug(
"MCP registry refreshed (%s servers in registry)", len(new_registry)
"MCP registry refreshed (%s servers in registry)", len(registered_registry)
)
def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]:

42
litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py
View file

@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy._experimental.mcp_server.auth import token_exchange
from litellm.types.llms.custom_http import httpxSpecialProvider
if TYPE_CHECKING:
@ -50,12 +51,23 @@ class MCPOAuth2TokenCache(InMemoryCache):
def _get_lock(self, server_id: str) -> asyncio.Lock:
return self._locks.setdefault(server_id, asyncio.Lock())
async def async_get_token(self, server: "MCPServer") -> Optional[str]:
@staticmethod
def _has_client_credentials_config(server: "MCPServer") -> bool:
return bool(server.client_id and server.client_secret and server.token_url)
async def async_get_token(
self,
server: "MCPServer",
*,
require_client_credentials_flow: bool = True,
) -> Optional[str]:
"""Return a valid access token, fetching or refreshing as needed.
Returns ``None`` when the server lacks client credentials config.
"""
if not server.has_client_credentials:
if require_client_credentials_flow and not server.has_client_credentials:
return None
if not self._has_client_credentials_config(server):
return None
server_id = server.server_id
@ -263,16 +275,38 @@ mcp_per_user_token_cache = MCPPerUserTokenCache()
async def resolve_mcp_auth(
server: "MCPServer",
mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
subject_token: Optional[str] = None,
) -> Optional[Union[str, Dict[str, str]]]:
"""Resolve the auth value for an MCP server.
Priority:
1. ``mcp_auth_header`` — per-request/per-user override
2. OAuth2 client_credentials token — auto-fetched and cached
3. ``server.authentication_token`` — static token from config/DB
2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token
3. OAuth2 client_credentials token — auto-fetched and cached
4. ``server.authentication_token`` — static token from config/DB
"""
if mcp_auth_header:
return mcp_auth_header
if server.has_token_exchange_config:
if subject_token:
return await token_exchange.mcp_token_exchange_handler.exchange_token(
subject_token, server
)
# No subject_token — fall back to client_credentials using the same client
# credentials and token_url so M2M scenarios still work.
if server.client_id and server.client_secret and server.token_url:
return await mcp_oauth2_token_cache.async_get_token(
server,
require_client_credentials_flow=False,
)
# OBO configured but no subject_token and missing client credentials — warn
# rather than silently proceeding unauthenticated.
verbose_logger.warning(
"MCP server '%s' is configured for token exchange (OBO) but no subject_token "
"was provided and client credentials (client_id/client_secret/token_url) are "
"incomplete. The request will proceed without authentication.",
server.server_id,
)
if server.has_client_credentials:
return await mcp_oauth2_token_cache.async_get_token(server)
return server.authentication_token

53
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -6,10 +6,34 @@ import asyncio
import contextvars
import json
import os
import re
from pathlib import PurePosixPath
from typing import Any, Dict, List, Optional
from urllib.parse import quote
# Tool names emitted from OpenAPI specs must work across all major LLM providers.
# OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to
# ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use
# tag-namespaced operationIds like "actions/download-job-logs-for-workflow-run"
# which include '/'. Sanitize here so the same regex passes everywhere downstream.
_OPENAPI_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
_OPENAPI_TOOL_NAME_MAX_LEN = 128
def sanitize_openapi_tool_name(raw_name: str) -> str:
"""Map an OpenAPI operationId / fallback to a provider-safe tool name.
Replaces any character outside ``[a-zA-Z0-9_-]`` with ``_`` and caps the
result at 128 chars (the most restrictive of the major providers).
Lowercased to match the existing convention in
``register_tools_from_openapi``.
"""
if not raw_name:
return raw_name
sanitized = _OPENAPI_TOOL_NAME_INVALID_CHARS.sub("_", raw_name).lower()
return sanitized[:_OPENAPI_TOOL_NAME_MAX_LEN]
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -399,17 +423,36 @@ def create_tool_function(
def register_tools_from_openapi(spec: Dict[str, Any], base_url: str):
"""Register MCP tools from OpenAPI specification."""
paths = spec.get("paths", {})
used_names: set = set()
for path, path_item in paths.items():
for method in ["get", "post", "put", "delete", "patch"]:
if method in path_item:
operation = path_item[method]
# Generate tool name
operation_id = operation.get(
"operationId", f"{method}_{path.replace('/', '_')}"
)
tool_name = operation_id.replace(" ", "_").lower()
# Generate tool name. Sanitize to ^[a-zA-Z0-9_-]+$ (lowercase)
# so the resulting name is valid across OpenAI/Anthropic/Bedrock.
# Many specs (e.g. GitHub REST) use tag-namespaced operationIds
# like "actions/download-job-logs-for-workflow-run" which
# contain '/' and would 400 at the LLM provider boundary.
operation_id = operation.get("operationId", f"{method}_{path}")
tool_name = sanitize_openapi_tool_name(operation_id)
# Disambiguate collisions: two operationIds that differ only
# by sanitized characters (e.g. "foo/list" and "foo.list")
# would both become "foo_list". Append _2, _3, … to keep
# every tool reachable, mirroring the Anthropic-side logic
# in _build_anthropic_tool_name_maps.
unique = tool_name
n = 1
while unique in used_names:
n += 1
suffix = f"_{n}"
unique = (
tool_name[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix
)
tool_name = unique
used_names.add(tool_name)
# Get description
description = operation.get(

28
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -857,6 +857,7 @@ if MCP_AVAILABLE:
########################################################
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
_inherit_credentials_from_existing_server,
)
def _extract_credentials(
@ -975,9 +976,11 @@ if MCP_AVAILABLE:
async def _preview_openapi_tools(spec_path: str) -> dict:
"""Generate tool previews from an OpenAPI spec without creating a server."""
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
_OPENAPI_TOOL_NAME_MAX_LEN,
build_input_schema,
load_openapi_spec_async,
resolve_operation_params,
sanitize_openapi_tool_name,
)
try:
@ -985,8 +988,9 @@ if MCP_AVAILABLE:
paths = spec.get("paths", {})
components = spec.get("components", {})
tools: List[dict] = []
used_names: set = set()
for path, path_item in paths.items():
for method in ("get", "post", "put", "patch", "delete"):
for method in ("get", "post", "put", "delete", "patch"):
operation = path_item.get(method)
if operation is None:
continue
@ -995,7 +999,23 @@ if MCP_AVAILABLE:
operation, path_item, components
)
op_id = operation.get("operationId", f"{method}_{path}")
raw_op_id = operation.get("operationId", f"{method}_{path}")
# Match what register_tools_from_openapi does so the preview
# the user sees in the dashboard equals the names that get
# registered (and shipped to LLM providers, which enforce
# ^[a-zA-Z0-9_-]+$). See sanitize_openapi_tool_name docstring.
op_id = sanitize_openapi_tool_name(raw_op_id)
unique = op_id
n = 1
while unique in used_names:
n += 1
suffix = f"_{n}"
unique = (
op_id[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix
)
op_id = unique
used_names.add(op_id)
summary = operation.get("summary", "")
description = operation.get("description", summary)
input_schema = build_input_schema(resolved_op)
@ -1068,6 +1088,10 @@ if MCP_AVAILABLE:
},
)
new_mcp_server_request = _inherit_credentials_from_existing_server(
new_mcp_server_request
)
# For OpenAPI spec servers, generate tools from the spec directly
if new_mcp_server_request.spec_path:
return await _preview_openapi_tools(new_mcp_server_request.spec_path)

2
litellm/proxy/_experimental/out/404/index.html → litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

47
litellm/proxy/_experimental/out/__next.__PAGE__.txt
View file

@ -1,27 +1,30 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/cd677ff381b90c30.js","/litellm-asset-prefix/_next/static/chunks/738efbc0d941f9d2.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","/litellm-asset-prefix/_next/static/chunks/35e1b31334447d6e.js","/litellm-asset-prefix/_next/static/chunks/40ec1228b231a0d1.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d8cd2d44272d51c8.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/fe5abdcdb57db543.js","/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/7149faf92f484aca.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/4b8dcb3ad5dfb8de.js","/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/ddcd1fd842a79e55.js","/litellm-asset-prefix/_next/static/chunks/3daef8922b68e600.js","/litellm-asset-prefix/_next/static/chunks/4f6e5b838f18b8e6.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/052aaa8d01e02cd3.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/65968777d52ff874.js","/litellm-asset-prefix/_next/static/chunks/a1b5b0c54192471e.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/341e7c75250f4f40.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/50d8a95e62930b35.js","/litellm-asset-prefix/_next/static/chunks/2005c732f6d6cbb4.js","/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/e7cc7b98b893b20d.js","/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","/litellm-asset-prefix/_next/static/chunks/e1f572e8226962e3.js"],"default"]
17:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
18:"$Sreact.suspense"
3:I[952683,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js"],"default"]
1a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
1b:"$Sreact.suspense"
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/cd677ff381b90c30.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/738efbc0d941f9d2.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/c53c9c7afec96700.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1fe0596a309ad6cf.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/c0b640cc12a2b90e.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/35e1b31334447d6e.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/40ec1228b231a0d1.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d8cd2d44272d51c8.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/fe5abdcdb57db543.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/47a838c67cdd745e.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/7149faf92f484aca.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/4b8dcb3ad5dfb8de.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/eaa9f9b9bb3e054b.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/ddcd1fd842a79e55.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/3daef8922b68e600.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/4f6e5b838f18b8e6.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/052aaa8d01e02cd3.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15"],"$L16"]}],"loading":null,"isPartial":false}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0493aafc4891dd29.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/f7e1d08418645368.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/b3d198d6c56a21b8.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/403c4d96324c23a6.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/37e77c06e99eb8ff.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/adb8beb738574863.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/0549bc9afa7d4888.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0b470ffc60999bf4.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/c847ecdf8c790b0b.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/baadbd26839e7b66.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/ee5f9a39a526e423.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/4e17b625d75327a7.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/7b788dd93ad868b3.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/a06cc76a774dd182.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/264fd32eefec52b6.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/86828bdbafb8b581.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/10dc4591ef08a91f.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/e099566e8bd4ee4e.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/fbe12a36d22e9554.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/134f728fa7099e3e.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/679dbd657c8b5aef.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/94f7208f5087e27c.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4e06277331e725da.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/3b30ab8eaa03bc21.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/fb125648f2dae104.js","async":true}],["$","script","script-29",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-30",{"src":"/litellm-asset-prefix/_next/static/chunks/a09028cd611c08ef.js","async":true}],["$","script","script-31",{"src":"/litellm-asset-prefix/_next/static/chunks/6967a3b4ecbd3785.js","async":true}],["$","script","script-32",{"src":"/litellm-asset-prefix/_next/static/chunks/3e917c79aadd945b.js","async":true}],["$","script","script-33",{"src":"/litellm-asset-prefix/_next/static/chunks/9bbebdeb3f1cb03f.js","async":true}],"$L6","$L7","$L8","$L9","$La","$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17","$L18"],"$L19"]}],"loading":null,"isPartial":false}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/65968777d52ff874.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/a1b5b0c54192471e.js","async":true}]
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/341e7c75250f4f40.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/2971c4658f1bcd7d.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/50d8a95e62930b35.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/2005c732f6d6cbb4.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/6eee262391715440.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/7c797521435cb59c.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/43f6fc3c2ab9cf23.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/e7cc7b98b893b20d.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/ca91b0fa4d619698.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f572e8226962e3.js","async":true}]
16:["$","$L17",null,{"children":["$","$18",null,{"name":"Next.MetadataOutlet","children":"$@19"}]}]
19:null
6:["$","script","script-34",{"src":"/litellm-asset-prefix/_next/static/chunks/0a65da2cd24e2ab6.js","async":true}]
7:["$","script","script-35",{"src":"/litellm-asset-prefix/_next/static/chunks/908828a91f602d8b.js","async":true}]
8:["$","script","script-36",{"src":"/litellm-asset-prefix/_next/static/chunks/5f2d62a75803a3f7.js","async":true}]
9:["$","script","script-37",{"src":"/litellm-asset-prefix/_next/static/chunks/ca5fbafaf3826374.js","async":true}]
a:["$","script","script-38",{"src":"/litellm-asset-prefix/_next/static/chunks/d3ac82723ec9e30d.js","async":true}]
b:["$","script","script-39",{"src":"/litellm-asset-prefix/_next/static/chunks/9b0ee76cbdef1a2a.js","async":true}]
c:["$","script","script-40",{"src":"/litellm-asset-prefix/_next/static/chunks/1bc2898be56acd1b.js","async":true}]
d:["$","script","script-41",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}]
e:["$","script","script-42",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}]
f:["$","script","script-43",{"src":"/litellm-asset-prefix/_next/static/chunks/496b84010c33cf69.js","async":true}]
10:["$","script","script-44",{"src":"/litellm-asset-prefix/_next/static/chunks/8e3d0ce9505a304f.js","async":true}]
11:["$","script","script-45",{"src":"/litellm-asset-prefix/_next/static/chunks/e1f23fd814ac3500.js","async":true}]
12:["$","script","script-46",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}]
13:["$","script","script-47",{"src":"/litellm-asset-prefix/_next/static/chunks/99cf9cf99df5ccfc.js","async":true}]
14:["$","script","script-48",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}]
15:["$","script","script-49",{"src":"/litellm-asset-prefix/_next/static/chunks/0cdfadbcf4b8c9e4.js","async":true}]
16:["$","script","script-50",{"src":"/litellm-asset-prefix/_next/static/chunks/8f3bf592254c6c3b.js","async":true}]
17:["$","script","script-51",{"src":"/litellm-asset-prefix/_next/static/chunks/8c17e934bd227606.js","async":true}]
18:["$","script","script-52",{"src":"/litellm-asset-prefix/_next/static/chunks/b98447395b5d37ef.js","async":true}]
19:["$","$L1a",null,{"children":["$","$1b",null,{"name":"Next.MetadataOutlet","children":"$@1c"}]}]
1c:null

97
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -3,4 +3,4 @@
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

4
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -4,5 +4,5 @@
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,5 +1,5 @@
:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/91037395c95e366d.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
:HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"]
0:{"buildId":"zxkD4-EPlgfKHDTw8O869","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}
0:{"buildId":"8TZ2JbOi7SZ6BCj9ScTHW","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

0
litellm/proxy/_experimental/out/_next/static/zxkD4-EPlgfKHDTw8O869/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/8TZ2JbOi7SZ6BCj9ScTHW/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/zxkD4-EPlgfKHDTw8O869/_clientMiddlewareManifest.json → litellm/proxy/_experimental/out/_next/static/8TZ2JbOi7SZ6BCj9ScTHW/_clientMiddlewareManifest.json
View file

0
litellm/proxy/_experimental/out/_next/static/zxkD4-EPlgfKHDTw8O869/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/8TZ2JbOi7SZ6BCj9ScTHW/_ssgManifest.js
View file

1
litellm/proxy/_experimental/out/_next/static/chunks/0377ae18aae60c57.js Normal file
View file

File diff suppressed because one or more lines are too long

84
litellm/proxy/_experimental/out/_next/static/chunks/038e83c8eab81a79.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/03c715f5e0f1425c.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0493aafc4891dd29.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/052aaa8d01e02cd3.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/05d900c88781d712.js
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/_next/static/chunks/b83ca9892d2d63cf.js → litellm/proxy/_experimental/out/_next/static/chunks/0606c92ecd600e0c.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/06b2ea8c776c2e9b.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/088fea506f07e1cd.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/0b470ffc60999bf4.js Normal file
View file

File diff suppressed because one or more lines are too long

12
litellm/proxy/_experimental/out/_next/static/chunks/7c36bfe1ba5e3ba8.js → litellm/proxy/_experimental/out/_next/static/chunks/0c3e8651e0e97232.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/eaa9f9b9bb3e054b.js → litellm/proxy/_experimental/out/_next/static/chunks/0cdfadbcf4b8c9e4.js
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more