Merge remote-tracking branch 'upstream/litellm_internal_staging' into deepkeep-as-internal

.circleci/config.yml

@@ -350,7 +350,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -k \"langfuse\""
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -395,7 +403,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
 
       # Store test results
@@ -471,7 +487,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -495,7 +519,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -k \"assistants\""
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -528,14 +560,19 @@ jobs:
             # Add --timeout to kill hanging tests after 120s (2 min)
             # Add --durations=20 to show 20 slowest tests for debugging
             # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
-            IGNORE_DIRS=(
-              "tests/llm_translation/realtime"
-            )
-            IGNORE_ARGS=""
-            for dir in "${IGNORE_DIRS[@]}"; do
-              IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
-            done
-            uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
+            mkdir -p test-results
+            # Glob excludes the realtime/ subdirectory since it has its own job
+            TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v \
+                --junitxml=test-results/junit.xml \
+                --durations=20 \
+                -n 4 \
+                --timeout=120 --timeout_method=thread \
+                --retries 2 --retry-delay 5 \
+                --max-worker-restart=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -560,7 +597,17 @@ jobs:
           command: |
             # Add --timeout to kill hanging tests after 120s (2 min)
             # Add --durations=20 to show 20 slowest tests for debugging
-            uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=20 \
+                -n 4 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -593,7 +640,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -626,7 +681,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            export LITELLM_LOG=WARNING
+            TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -660,7 +726,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                --retries 3 --retry-delay 5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -702,7 +777,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 8"
           no_output_timeout: 15m
 
       # Store test results
@@ -725,7 +808,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -758,7 +850,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -793,7 +894,15 @@ jobs:
           name: Run enterprise tests
           command: |
             uv run --no-sync python -m prisma generate
-            uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-enterprise.xml \
+                --durations=10 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -815,7 +924,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -848,7 +966,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -882,7 +1009,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -916,7 +1052,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -939,7 +1083,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            export LITELLM_LOG=WARNING
+            TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                -n 4 \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -972,7 +1127,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -1012,14 +1175,19 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv \
+            mkdir -p test-results
+            TEST_FILES=$(printf "%s\n" \
               tests/local_testing/test_dual_cache.py \
               tests/local_testing/test_redis_batch_optimizations.py \
-              tests/local_testing/test_router_utils.py \
-              --cov=litellm --cov-report=xml \
-              -x -s -v --junitxml=test-results/junit.xml \
-              --durations=5 -n 2 \
-              --reruns 2 --reruns-delay 1
+              tests/local_testing/test_router_utils.py)
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 -n 2 \
+                --reruns 2 --reruns-delay 1"
           no_output_timeout: 20m
       - run:
           name: Rename the coverage files
@@ -1260,8 +1428,17 @@ jobs:
       - run:
           name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
           command: |
-            uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
+      - store_test_results:
+          path: test-results
 
   build_and_test:
     machine:
@@ -1331,7 +1508,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+            mkdir -p test-results
+            # Original used `tests/*.py` (top-level only); the `--ignore=...`
+            # flags were vestigial since shell globbing did not descend into
+            # subdirectories. Replicate by globbing only top-level test files.
+            TEST_FILES=$(circleci tests glob "tests/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -s -v -x \
+                --junitxml=test-results/junit.xml \
+                -n 4 \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1406,7 +1594,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -s -vv \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1475,7 +1670,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
       - run:
@@ -1518,7 +1720,14 @@ jobs:
       - run:
           name: Run second round of tests
           command: |
-            uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1587,8 +1796,17 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
+      - store_test_results:
+          path: test-results
       - run:
           name: Stop and remove first container
           when: always
@@ -1676,7 +1894,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
       # Store test results
@@ -1732,7 +1957,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Stop and remove containers
@@ -1805,9 +2037,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
+      - store_test_results:
+          path: test-results
       - run:
           name: Stop and remove first container
           command: |
@@ -1935,12 +2176,19 @@ jobs:
           name: Run Vertex AI, Google AI Studio Node.js tests
           command: |
             cd tests/pass_through_tests
-            npx jest . --verbose
+            NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
           no_output_timeout: 30m
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1997,9 +2245,16 @@ jobs:
       - run:
           name: Run Claude Agent SDK E2E Tests
           command: |
+            mkdir -p test-results
             export LITELLM_PROXY_URL="http://localhost:4000"
             export LITELLM_API_KEY="sk-1234"
-            uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
+            TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -2138,17 +2393,23 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
+          # The cimg/python:3.12-browsers image already ships the Chromium system
+          # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
+          # `--with-deps` triggers a redundant apt-get update + install that adds
+          # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
+          # so we install just the browser binary.
           command: |
             cd ui/litellm-dashboard
             npm ci
-            npx playwright install chromium --with-deps
+            npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
           # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.

.github/workflows/create-release-branch.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
         required: true
         type: string
       commit_hash:
@@ -14,7 +14,7 @@ on:
   workflow_call:
     inputs:
       tag:
-        description: "Release tag"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -40,8 +40,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
@@ -45,6 +45,13 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
+            // like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +96,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });

.github/workflows/mutation-test.yml

@@ -0,0 +1,131 @@
+name: "Mutation Test (manual)"
+
+# Manually-triggered mutation testing. Runs mutmut against the scope
+# configured in [tool.mutmut] in pyproject.toml (currently the
+# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly
+# weekly — clicked from the Actions tab when someone wants a fresh report.
+#
+# Uploads a structured `mutation-report.md` (Meta ACH-style: original +
+# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the
+# existing tests + a task instruction) as a workflow artifact. Failures
+# do not block anything because nothing depends on this workflow.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mutation-test-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mutation:
+    name: Run mutmut
+    runs-on: ubuntu-latest
+    # Whole-folder mutation against ~15 files / ~7.5k LOC can take hours.
+    # 350 minutes is just under the GitHub-hosted job cap of 360 minutes.
+    timeout-minutes: 350
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      # mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
+      # mutation trampolines. uv installs the project as editable by default,
+      # which puts the original source dir on sys.path via a .pth file and
+      # shadows the sandbox copy — so tests would never exercise the mutated
+      # code. Reinstalling non-editable removes the .pth shadow.
+      - name: Reinstall litellm non-editable (so mutants/ is not shadowed)
+        run: |
+          uv pip uninstall litellm
+          uv pip install . --no-deps
+
+      # pytest-retry's pytest_configure hook crashes with
+      # `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
+      # via mutmut's in-process pytest.main() call. The entry-point name
+      # doesn't normalize cleanly with `-p no:<name>`, so just remove the
+      # package outright. Reruns are wrong for mutation testing anyway —
+      # rerunning a "failed" mutant test would mask which mutants are killed.
+      - name: Remove pytest plugins that conflict with mutmut
+        run: |
+          uv pip uninstall pytest-retry || true
+
+      - name: Run mutmut
+        env:
+          # Make the mutants/ sandbox win over site-packages on sys.path so the
+          # trampolined files are imported instead of the installed copy.
+          PYTHONPATH: ${{ github.workspace }}/mutants
+        run: |
+          set -o pipefail
+          mkdir -p mutants
+          uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log
+
+      # Generate the structured report. The script embeds the enclosing
+      # function source for each survivor (via Python AST) and includes the
+      # existing test files, so an LLM agent has enough context to write
+      # killing tests without further file lookups. Modeled on Meta's ACH
+      # prompt template (arXiv 2501.12862).
+      - name: Generate detailed mutation report
+        if: always()
+        run: |
+          set +e
+          uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
+          uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
+          uv run --no-sync python scripts/mutation_report.py
+          # The full report can be very long for big test files; the run-page
+          # summary cuts off at 1 MB. Append the head of the report (summary
+          # + survivor list) and link out to the artifact for the full body.
+          {
+            head -c 900000 mutation-report.md
+            echo ""
+            echo ""
+            echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload mutmut artifacts
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            mutation-report.md
+            mutmut-results.txt
+            mutmut-run.log
+            mutants/mutmut-stats.json
+            mutants/mutmut-cicd-stats.json
+            mutants/litellm/proxy/management_endpoints/**/*.py
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/test-unit-proxy-db.yml

@@ -100,6 +100,7 @@ jobs:
             test-path: >-
               tests/proxy_unit_tests/test_auth_checks.py
               tests/proxy_unit_tests/test_user_api_key_auth.py
+              tests/proxy_unit_tests/test_deprecated_key_grace_period.py
             workers: 4
             dist: loadscope
             timeout: 15
@@ -141,6 +142,8 @@ jobs:
               tests/proxy_unit_tests/test_server_root_path.py
               tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
+              tests/proxy_unit_tests/test_request_size_limit_middleware.py
+              tests/proxy_unit_tests/test_multipart_bypass_repro.py
             workers: 4
             dist: loadscope
             timeout: 15

.gitignore

@@ -91,7 +91,6 @@ test.py
 litellm_config.yaml
 !.github/observatory/litellm_config.yaml
 .cursor
-.vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -102,3 +101,5 @@ STABILIZATION_TODO.md
 **/playwright-report
 **/*.storageState.json
 **/coverage
+test-config
+.vscode

.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

AGENTS.md

@@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates:
 
 ### Running the proxy server
 
-Start the proxy with a config file:
+Create a minimal config file and start the proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://fake-api.example.com
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+```
 
 ```bash
-uv run litellm --config dev_config.yaml --port 4000
+uv run litellm --config config.yaml --port 4000
 ```
 
 The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.

CLAUDE.md

@@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
 - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
 - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
-- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
 
 ### Setup Wizard (`litellm/setup_wizard.py`)
 - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).

Dockerfile

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
+    npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
     GLOBAL="$(npm root -g)" && \
     for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
         name="${pkg##*/}"; \
@@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
 # Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-COPY --from=builder /root/.cache /root/.cache
+# which is /root/.cache here. Copy only the Prisma subdirs — copying the
+# whole /root/.cache drags in the uv build cache (~660 MB, includes a
+# setuptools wheel that surfaces as a CVE finding even though it's not
+# on the runtime sys.path).
+COPY --from=builder /root/.cache/prisma /root/.cache/prisma
+COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

Makefile

@@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
 	$(UV_RUN) pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
 		-v --tb=short --maxfail=100 --timeout=300
+
+test-llm-translation-flush-vcr-cache:
+	$(UV_RUN) python tests/_flush_vcr_cache.py

README.md

@@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
     <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
     <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
     <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
-    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
     <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
     <td><h2>Netflix</h2></td>
     <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>

codecov.yaml

@@ -1,3 +1,8 @@
+codecov:
+  require_ci_to_pass: false # post coverage status even if CI has unrelated failures
+  notify:
+    wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
+
 component_management:
   individual_components:
     - component_id: "Router"
@@ -28,7 +33,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 1% # at maximum allow project coverage to drop by 1%
+        threshold: 0% # do not allow project coverage to drop
     patch:
       default:
         target: auto

cookbook/litellm-ollama-docker-image/requirements.txt

@@ -1 +1 @@
-litellm==1.83.5
+litellm==1.83.14

deploy/Dockerfile.ghcr_base

@@ -1,18 +0,0 @@
-# Use the provided base image
-FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Copy the configuration file into the container at /app
-COPY config.yaml .
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -100,6 +100,16 @@ spec:
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
             {{- end }}
+            {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+            - name: DATABASE_URL_READ_REPLICA
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.readReplicaUrlKey }}
+            {{- else if .Values.db.readReplicaUrl }}
+            - name: DATABASE_URL_READ_REPLICA
+              value: {{ .Values.db.readReplicaUrl | quote }}
+            {{- end }}
             - name: PROXY_MASTER_KEY
               valueFrom:
                 secretKeyRef:
@@ -116,21 +126,32 @@ spec:
                   name: {{ include "redis.secretName" .Subcharts.redis }}
                   key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
             {{- end }}
+            {{- /*
+              Inject LITELLM_LOG only when envVars does not already define it.
+            */}}
+            {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
+            - name: LITELLM_LOG
+              value: {{ .Values.logLevel | quote }}
+            {{- end }}
             {{- if .Values.envVars }}
             {{- range $key, $val := .Values.envVars }}
             - name: {{ $key }}
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
-            {{- if .Values.separateHealthApp }}
-            - name: SEPARATE_HEALTH_APP
-              value: "1"
-            - name: SEPARATE_HEALTH_PORT
-              value: {{ .Values.separateHealthPort | default "8081" | quote }}
-            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.migrationJob.enabled }}
+            # Schema updates are owned by the dedicated migrations Job; skip
+            # the proxy's startup `prisma db push` so N replicas don't race
+            # one DB on every rollout. Placed last (after envVars and
+            # extraEnvVars) so this override can't be silently shadowed by a
+            # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
+            # semantics — same pattern the migrations Job uses.
+            - name: DISABLE_SCHEMA_UPDATE
+              value: "true"
+            {{- end }}
           envFrom:
           {{- range .Values.environmentSecrets }}
             - secretRef:
@@ -158,15 +179,10 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
-            {{- if .Values.separateHealthApp }}
-            - name: health
-              containerPort: {{ .Values.separateHealthPort | default 8081 }}
-              protocol: TCP
-            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@@ -175,7 +191,7 @@ spec:
           readinessProbe:
             httpGet:
               path: {{ .Values.readinessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@@ -184,7 +200,7 @@ spec:
           startupProbe:
             httpGet:
               path: {{ .Values.startupProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.startupProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -257,16 +257,16 @@ tests:
           value: 0
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.periodSeconds
-          value: 10
+          value: 15
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.successThreshold
           value: 1
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.failureThreshold
-          value: 3
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.httpGet.path
           value: /health/readiness
@@ -278,7 +278,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.successThreshold
           value: 1
@@ -296,7 +296,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].startupProbe.successThreshold
           value: 1

deploy/charts/litellm-helm/values.yaml

@@ -88,26 +88,20 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
-# Separate health app configuration
-# When enabled, health checks will use a separate port and the application
-# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
-separateHealthApp: false
-separateHealthPort: 8081
-
-# Probe tuning for proxy container
+# Probes for LiteLLM gateway container
 livenessProbe:
   path: /health/liveliness
   initialDelaySeconds: 0
-  periodSeconds: 10
-  timeoutSeconds: 1
+  periodSeconds: 15
+  timeoutSeconds: 5
   successThreshold: 1
-  failureThreshold: 3
+  failureThreshold: 5
 
 readinessProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 3
 
@@ -115,7 +109,7 @@ startupProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 30
 
@@ -258,6 +252,26 @@ db:
     passwordKey: password
     # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
     endpointKey: ""
+    # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
+    # secret key instead of db.readReplicaUrl. Prefer this over the plain
+    # value: read-replica URLs typically embed credentials, and a value
+    # written to db.readReplicaUrl ends up visible in the rendered pod spec
+    # and the Helm release secret.
+    readReplicaUrlKey: ""
+
+  # Optional read-replica routing. When set, the proxy sends read-only
+  # queries (find_*, count, group_by, query_raw/_first) to this URL while
+  # writes continue to go to db.url. Useful for Aurora-style clusters with
+  # separate reader/writer endpoints. Leave empty to keep single-DB behavior.
+  # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
+  # alongside the writer (host/port/user/db are parsed from this URL once
+  # at startup; only the IAM token rotates).
+  #
+  # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
+  # this field — the plain value is rendered into the pod spec and the
+  # Helm release secret. This field is intended for credential-less URLs
+  # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
+  readReplicaUrl: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
@@ -329,6 +343,17 @@ migrationJob:
     helm:
       enabled: false
 
+# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
+# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
+# any `envFrom:` source. If you currently source LITELLM_LOG from an
+# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
+# disable injection — otherwise this value silently overrides your secret /
+# configmap entry.
+#
+# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
+# this injection entirely when envVars already defines LITELLM_LOG.
+logLevel: INFO
+
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {}
 

deploy/kubernetes/kub.yaml

@@ -1,56 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: litellm-deployment
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: litellm
-  template:
-    metadata:
-      labels:
-        app: litellm
-    spec:
-      containers:
-        - name: litellm-container
-          image: ghcr.io/berriai/litellm:main-latest
-          imagePullPolicy: Always
-          env:
-            - name: AZURE_API_KEY
-              value: "d6f****"
-            - name: AZURE_API_BASE
-              value: "https://openai"
-            - name: LITELLM_MASTER_KEY
-              value: "sk-1234"
-            - name: DATABASE_URL
-              value: "postgresql://ishaan*********"
-          args:
-            - "--config"
-            - "/app/proxy_config.yaml"  # Update the path to mount the config file
-          volumeMounts:                 # Define volume mount for proxy_config.yaml
-            - name: config-volume
-              mountPath: /app
-              readOnly: true
-          livenessProbe:
-            httpGet:
-              path: /health/liveliness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /health/readiness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-      volumes:  # Define volume to mount proxy_config.yaml
-        - name: config-volume
-          configMap:
-            name: litellm-config  

deploy/kubernetes/service.yaml

@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: litellm-service
-spec:
-  selector:
-    app: litellm
-  ports:
-    - protocol: TCP
-      port: 4000
-      targetPort: 4000
-  type: LoadBalancer

dev_config.yaml

@@ -1,13 +0,0 @@
-model_list:
-  - model_name: fake-openai-endpoint
-    litellm_params:
-      model: openai/fake-model
-      api_key: fake-key
-      api_base: https://exampleopenaiendpoint-production.up.railway.app/
-
-general_settings:
-  master_key: sk-1234
-
-litellm_settings:
-  drop_params: True
-  telemetry: False

docker-compose.yml

@@ -16,6 +16,11 @@ services:
       - "4000:4000" # Map the container port to the host, change the host port if necessary
     environment:
       DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      # Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
+      # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
+      # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
+      # is auto-refreshed alongside the writer.
+      # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
       STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
     env_file:
       - .env # Load local .env file

docker/Dockerfile.alpine

@@ -1,68 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
-
-FROM $UV_IMAGE AS uvbin
-
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-WORKDIR /app
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-
-RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile
-
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-# Copy dependency metadata first for layer caching
-COPY pyproject.toml uv.lock ./
-COPY enterprise/pyproject.toml enterprise/
-COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
-
-# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python3
-
-# Copy full source tree
-COPY . .
-
-# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python3
-
-RUN prisma generate --schema=./schema.prisma
-
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm
-
-WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-COPY --from=builder /app /app
-
-EXPOSE 4000/tcp
-
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-CMD ["--port", "4000"]

docker/Dockerfile.custom_ui

@@ -1,86 +0,0 @@
-# Use the provided base image
-# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
-FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 && \
-    apt-get install -y --no-install-recommends nodejs npm && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    apt-get purge -y npm
-
-# Copy the UI source into the container
-COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
-
-# Set an environment variable for UI_BASE_PATH
-# This can be overridden at build time
-# set UI_BASE_PATH to "<your server root path>/ui"
-ENV UI_BASE_PATH="/prod/ui"
-
-# Build the UI with the specified UI_BASE_PATH
-WORKDIR /app/ui/litellm-dashboard
-RUN npm ci
-RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
-
-# Create the destination directory
-RUN mkdir -p /app/litellm/proxy/_experimental/out
-
-# Move the built files to the appropriate location
-# Assuming the build output is in ./out directory
-RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
-    mv ./out/* /app/litellm/proxy/_experimental/out/
-
-# Switch back to the main app directory
-WORKDIR /app
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-# Run as non-root user
-RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \
-    && chown -R appuser:appuser /app
-USER appuser
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-  CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"]
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

docker/Dockerfile.database

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
     npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/Dockerfile.dev

@@ -1,121 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
-
-FROM $UV_IMAGE AS uvbin
-
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-WORKDIR /app
-USER root
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    g++ \
-    python3-dev \
-    libssl-dev \
-    pkg-config \
-    nodejs \
-    npm \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-# Copy dependency metadata first for layer caching
-COPY pyproject.toml uv.lock ./
-COPY enterprise/pyproject.toml enterprise/
-COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
-
-# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-# Copy full source tree
-COPY . .
-
-# Build Admin UI before final sync
-RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-RUN prisma generate --schema=./schema.prisma
-
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-USER root
-
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 \
-    && apt-get install -y --no-install-recommends \
-        libssl3 \
-        libatomic1 \
-        nodejs \
-        npm \
-    && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
-    && GLOBAL="$(npm root -g)" \
-    && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-       done \
-    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
-    && npm cache clean --force \
-    && apt-get purge -y npm
-
-WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-COPY --from=builder /app /app
-
-EXPOSE 4000/tcp
-
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-CMD ["--port", "4000"]

docker/Dockerfile.health_check

@@ -1,30 +0,0 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
-FROM $UV_IMAGE AS uvbin
-
-FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-WORKDIR /app
-
-# Copy the uv binary and the health check script.
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY pyproject.toml uv.lock /app/
-COPY scripts/health_check/health_check_client.py /app/health_check_client.py
-
-# Resolve and install the health-check dependencies from the project lockfile
-# so the runtime image stays self-contained and reproducible.
-RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \
-  && uv pip install --system -r /tmp/health-check-requirements.txt \
-  && rm /tmp/health-check-requirements.txt \
-  && rm /app/pyproject.toml /app/uv.lock \
-  && chmod +x /app/health_check_client.py
-
-# Run as non-root user
-RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser
-USER appuser
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
-  CMD ["python", "/app/health_check_client.py", "--help"]
-
-# Set entrypoint
-ENTRYPOINT ["python", "/app/health_check_client.py"]

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 ARG PROXY_EXTRAS_SOURCE=published
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -32,7 +32,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
     XDG_CACHE_HOME=/app/.cache
 
 # Copy dependency metadata first for layer caching
@@ -104,17 +103,15 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
-COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
 
 ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
     HOME=/app \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache \

docker/prod_entrypoint.sh

@@ -1,14 +1,8 @@
 #!/bin/sh
 
-if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
-    export LITELLM_ARGS="$@"
-    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
-    exec supervisord -c /etc/supervisord.conf
-fi
-
 if [ "$USE_DDTRACE" = "true" ]; then
     export DD_TRACE_OPENAI_ENABLED="False"
     exec ddtrace-run litellm "$@"
 else
     exec litellm "$@"
-fi
+fi

docker/supervisord.conf

@@ -1,46 +0,0 @@
-[supervisord]
-nodaemon=true
-loglevel=info
-logfile=/tmp/supervisord.log
-pidfile=/tmp/supervisord.pid
-
-[group:litellm]
-programs=main,health
-
-[program:main]
-command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
-autostart=true
-autorestart=true
-startretries=3
-priority=1
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[program:health]
-command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
-autostart=true
-autorestart=true
-startretries=3
-priority=2
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[eventlistener:process_monitor]
-command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
-events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
-autostart=true
-autorestart=true

docs/my-website/docs/providers/crusoe.md

@@ -0,0 +1,196 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Crusoe
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
+| Provider Route on LiteLLM | `crusoe/` |
+| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
+| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+<br />
+<br />
+
+**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
+
+## Available Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
+| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
+| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
+| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
+| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
+| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
+| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Crusoe Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Crusoe call
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Crusoe Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Write a short story about AI", "role": "user"}]
+
+# Crusoe call with streaming
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+### Function Calling
+
+```python showLineNumbers title="Crusoe Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the current weather in a location",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state, e.g. San Francisco, CA"
+                }
+            },
+            "required": ["location"]
+        }
+    }
+}]
+
+messages = [{"role": "user", "content": "What's the weather in Boston?"}]
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy Server
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: llama-3.3-70b
+    litellm_params:
+      model: crusoe/meta-llama/Llama-3.3-70B-Instruct
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-r1
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-R1-0528
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-v3
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-V3-0324
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: qwen3-235b
+    litellm_params:
+      model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: kimi-k2
+    litellm_params:
+      model: crusoe/moonshotai/Kimi-K2-Thinking
+      api_key: os.environ/CRUSOE_API_KEY
+```
+
+## Custom API Base
+
+**Option 1: Environment variable**
+
+```python showLineNumbers title="Custom API Base via env var"
+import os
+from litellm import completion
+
+os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
+os.environ["CRUSOE_API_KEY"] = ""  # your API key
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+)
+```
+
+**Option 2: Pass directly**
+
+```python showLineNumbers title="Custom API Base via parameter"
+from litellm import completion
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+    api_base="https://custom.crusoecloud.com/v1",
+    api_key="your-api-key",
+)
+```
+
+## Supported OpenAI Parameters
+
+- `temperature`
+- `max_tokens`
+- `max_completion_tokens`
+- `top_p`
+- `frequency_penalty`
+- `presence_penalty`
+- `stop`
+- `n`
+- `stream`
+- `tools`
+- `tool_choice`
+- `response_format`
+- `seed`
+- `user`
+- `logit_bias`
+- `logprobs`
+- `top_logprobs`

enterprise/enterprise_hooks/banned_keywords.py

@@ -11,6 +11,10 @@ from typing import Literal
 import litellm
 from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import (
+    is_text_content_call_type,
+    iter_message_text,
+)
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger
 from fastapi import HTTPException
@@ -73,10 +77,9 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
             - check if user id part of blocked list
             """
             self.print_verbose("Inside Banned Keyword List Pre-Call Hook")
-            if call_type == "completion" and "messages" in data:
-                for m in data["messages"]:
-                    if "content" in m and isinstance(m["content"], str):
-                        self.test_violation(test_str=m["content"])
+            if is_text_content_call_type(call_type):
+                for text in iter_message_text(data):
+                    self.test_violation(test_str=text)
 
         except HTTPException as e:
             raise e
@@ -93,11 +96,16 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
         user_api_key_dict: UserAPIKeyAuth,
         response,
     ):
-        if isinstance(response, litellm.ModelResponse) and isinstance(
-            response.choices[0], litellm.utils.Choices
-        ):
-            for word in self.banned_keywords_list:
-                self.test_violation(test_str=response.choices[0].message.content or "")
+        if not isinstance(response, litellm.ModelResponse):
+            return
+
+        for choice in response.choices:
+            if not isinstance(choice, litellm.utils.Choices):
+                continue
+            message = getattr(choice, "message", None)
+            content = getattr(message, "content", None)
+            if isinstance(content, str):
+                self.test_violation(test_str=content)
 
     async def async_post_call_streaming_hook(
         self,

enterprise/enterprise_hooks/google_text_moderation.py

@@ -12,6 +12,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
@@ -94,11 +95,9 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
         - Calls Google's Text Moderation API
         - Rejects request if it fails safety check
         """
-        if "messages" in data and isinstance(data["messages"], list):
-            text = ""
-            for m in data["messages"]:  # assume messages is a list
-                if "content" in m and isinstance(m["content"], str):
-                    text += m["content"]
+        # Covers multimodal list content + Responses-API input.
+        text = "".join(iter_message_text(data))
+        if text:
             document = self.language_document(content=text, type_=self.document_type)
 
             request = self.moderate_text_request(

enterprise/enterprise_hooks/openai_moderation.py

@@ -19,6 +19,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
@@ -37,11 +38,8 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
     ):
-        text = ""
-        if "messages" in data and isinstance(data["messages"], list):
-            for m in data["messages"]:  # assume messages is a list
-                if "content" in m and isinstance(m["content"], str):
-                    text += m["content"]
+        # Covers multimodal list content + Responses-API input.
+        text = "".join(iter_message_text(data))
 
         from litellm.proxy.proxy_server import llm_router
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import walk_user_text
 
 GUARDRAIL_NAME = "hide_secrets"
 
@@ -473,23 +474,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         if await self.should_run_check(user_api_key_dict) is False:
             return
 
-        if "messages" in data and isinstance(data["messages"], list):
-            for message in data["messages"]:
-                if "content" in message and isinstance(message["content"], str):
-                    detected_secrets = self.scan_message_for_secrets(message["content"])
+        # Covers multimodal list content + Responses-API input.
+        def _redact_message_text(text: str) -> str:
+            detected_secrets = self.scan_message_for_secrets(text)
+            for secret in detected_secrets:
+                text = text.replace(secret["value"], "[REDACTED]")
+            if detected_secrets:
+                secret_types = [secret["type"] for secret in detected_secrets]
+                verbose_proxy_logger.warning(
+                    f"Detected and redacted secrets in message: {secret_types}"
+                )
+            return text
 
-                    for secret in detected_secrets:
-                        message["content"] = message["content"].replace(
-                            secret["value"], "[REDACTED]"
-                        )
-
-                    if len(detected_secrets) > 0:
-                        secret_types = [secret["type"] for secret in detected_secrets]
-                        verbose_proxy_logger.warning(
-                            f"Detected and redacted secrets in message: {secret_types}"
-                        )
-                    else:
-                        verbose_proxy_logger.debug("No secrets detected on input.")
+        walk_user_text(data, _redact_message_text)
 
         if "prompt" in data:
             if isinstance(data["prompt"], str):
@@ -504,11 +501,15 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
                         f"Detected and redacted secrets in prompt: {secret_types}"
                     )
             elif isinstance(data["prompt"], list):
-                for item in data["prompt"]:
+                # Index back into the list — assigning to ``item`` would only
+                # rebind the loop variable and leave ``data["prompt"]``
+                # carrying the unredacted secret.
+                for idx, item in enumerate(data["prompt"]):
                     if isinstance(item, str):
                         detected_secrets = self.scan_message_for_secrets(item)
                         for secret in detected_secrets:
                             item = item.replace(secret["value"], "[REDACTED]")
+                        data["prompt"][idx] = item
                         if len(detected_secrets) > 0:
                             secret_types = [
                                 secret["type"] for secret in detected_secrets
@@ -517,31 +518,6 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
                                 f"Detected and redacted secrets in prompt: {secret_types}"
                             )
 
-        if "input" in data:
-            if isinstance(data["input"], str):
-                detected_secrets = self.scan_message_for_secrets(data["input"])
-                for secret in detected_secrets:
-                    data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
-                if len(detected_secrets) > 0:
-                    secret_types = [secret["type"] for secret in detected_secrets]
-                    verbose_proxy_logger.warning(
-                        f"Detected and redacted secrets in input: {secret_types}"
-                    )
-            elif isinstance(data["input"], list):
-                _input_in_request = data["input"]
-                for idx, item in enumerate(_input_in_request):
-                    if isinstance(item, str):
-                        detected_secrets = self.scan_message_for_secrets(item)
-                        for secret in detected_secrets:
-                            _input_in_request[idx] = item.replace(
-                                secret["value"], "[REDACTED]"
-                            )
-                        if len(detected_secrets) > 0:
-                            secret_types = [
-                                secret["type"] for secret in detected_secrets
-                            ]
-                            verbose_proxy_logger.warning(
-                                f"Detected and redacted secrets in input: {secret_types}"
-                            )
-                verbose_proxy_logger.debug("Data after redacting input %s", data)
+        # ``data["input"]`` (Responses API and embeddings/moderation) is
+        # already covered by ``walk_user_text`` above.
         return

enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py

@@ -10,18 +10,11 @@ has already authenticated the user) and you need to extract user information fro
 custom headers or other request attributes.
 """
 
-from typing import TYPE_CHECKING, Dict, Optional, Union, cast
+from typing import cast
 
 from fastapi import Request
 from fastapi.responses import RedirectResponse
 
-if TYPE_CHECKING:
-    from fastapi_sso.sso.base import OpenID
-else:
-    from typing import Any as OpenID
-
-from litellm.proxy.management_endpoints.types import CustomOpenID
-
 
 class EnterpriseCustomSSOHandler:
     """
@@ -60,9 +53,13 @@ class EnterpriseCustomSSOHandler:
         from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
         from litellm.proxy.proxy_server import (
             CommonProxyErrors,
+            general_settings,
             premium_user,
             user_custom_ui_sso_sign_in_handler,
         )
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
 
         if premium_user is not True:
             raise ValueError(CommonProxyErrors.not_premium_user.value)
@@ -72,6 +69,12 @@ class EnterpriseCustomSSOHandler:
                 "custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings."
             )
 
+        require_trusted_proxy_request(
+            request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
+        )
+
         custom_sso_login_handler = cast(
             CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler
         )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -15,6 +15,11 @@ from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.managed_resources.isolation import (
+    build_list_page,
+    build_owner_filter,
+    can_access_resource,
+)
 from litellm.proxy._types import (
     CallTypes,
     LiteLLM_ManagedFileTable,
@@ -99,6 +104,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
                 created_by=user_api_key_dict.user_id,
+                team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
             await self.internal_usage_cache.async_set_cache(
@@ -114,6 +120,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
             "created_by": user_api_key_dict.user_id,
+            "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
 
@@ -171,6 +178,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": user_api_key_dict.user_id,
+                    "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
                 },
@@ -229,15 +237,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     async def can_user_call_unified_file_id(
         self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
     ) -> bool:
-        ## check if the user has access to the unified file id
-
-        user_id = user_api_key_dict.user_id
         managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
             where={"unified_file_id": unified_file_id}
         )
 
         if managed_file:
-            return managed_file.created_by == user_id
+            return can_access_resource(
+                user_api_key_dict=user_api_key_dict,
+                created_by=managed_file.created_by,
+                resource_team_id=managed_file.team_id,
+            )
         raise HTTPException(
             status_code=404,
             detail=f"File not found: {unified_file_id}",
@@ -246,8 +255,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     async def can_user_call_unified_object_id(
         self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
     ) -> bool:
-        ## check if the user has access to the unified object id
-        user_id = user_api_key_dict.user_id
         managed_object = (
             await self.prisma_client.db.litellm_managedobjecttable.find_first(
                 where={"unified_object_id": unified_object_id}
@@ -255,7 +262,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
 
         if managed_object:
-            return managed_object.created_by == user_id
+            return can_access_resource(
+                user_api_key_dict=user_api_key_dict,
+                created_by=managed_object.created_by,
+                resource_team_id=managed_object.team_id,
+            )
         raise HTTPException(
             status_code=404,
             detail=f"Object not found: {unified_object_id}",
@@ -286,19 +297,18 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 "Filtering by 'target_model_names' is not supported when using managed batches."
             )
 
-        where_clause: Dict[str, Any] = {"file_purpose": "batch"}
+        owner_filter = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return build_list_page([])
 
-        # Filter by user who created the batch
-        if user_api_key_dict.user_id:
-            where_clause["created_by"] = user_api_key_dict.user_id
+        where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
 
         if after:
             where_clause["id"] = {"gt": after}
 
-        # Fetch more than needed to allow for post-fetch filtering
         fetch_limit = limit or 20
         if target_model_names:
-            # Fetch extra to account for filtering
+            # Oversample so post-fetch model-name filtering still has enough rows.
             fetch_limit = max(fetch_limit * 3, 100)
 
         batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
@@ -329,26 +339,28 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 )
                 continue
 
-        return {
-            "object": "list",
-            "data": batch_objects,
-            "first_id": batch_objects[0].id if batch_objects else None,
-            "last_id": batch_objects[-1].id if batch_objects else None,
-            "has_more": len(batch_objects) == (limit or 20),
-        }
+        return build_list_page(
+            batch_objects, has_more=len(batch_objects) == (limit or 20)
+        )
 
     async def get_user_created_file_ids(
         self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
     ) -> List[OpenAIFileObject]:
         """
-        Get all file ids created by the user for a list of model object ids
+        Get all file ids the caller is allowed to see for a list of model
+        object ids. Service-account keys (no user_id) are scoped to their
+        team via ``team_id``; admins see all matches.
 
         Returns:
          - List of OpenAIFileObject's
         """
+        owner_filter = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return []
+
         file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
             where={
-                "created_by": user_api_key_dict.user_id,
+                **owner_filter,
                 "flat_model_file_ids": {"hasSome": model_object_ids},
             }
         )

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -588,24 +588,21 @@ async def update_project(  # noqa: PLR0915
                 param="project_id",
             )
 
-        # Validate team exists and get team object for limit + permission checks
-        team_id_to_check = data.team_id or existing_project.team_id
-        team_obj_for_checks = None
-        if team_id_to_check is not None:
-            team_obj_for_checks = await _validate_team_exists(
-                team_id=team_id_to_check, prisma_client=prisma_client
+        # Permission to *edit* the project must be evaluated against the
+        # project's CURRENT team. Sourcing the team from `data.team_id`
+        # would let an admin of any team pass the check by supplying their
+        # own team_id, hijacking the project (VERIA-55).
+        target_team_id = data.team_id or existing_project.team_id
+        target_team_obj = None
+        if target_team_id is not None:
+            target_team_obj = await _validate_team_exists(
+                team_id=target_team_id, prisma_client=prisma_client
             )
 
-        # Check if user has permission to update this project
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
             team_id=existing_project.team_id,
             prisma_client=prisma_client,
-            team_object=(
-                LiteLLM_TeamTable(**team_obj_for_checks.model_dump())
-                if team_obj_for_checks
-                else None
-            ),
         )
 
         if not has_permission:
@@ -614,10 +611,32 @@ async def update_project(  # noqa: PLR0915
                 detail={"error": "Only admins or team admins can update projects"},
             )
 
+        # Reassigning to a different team also requires admin rights on the
+        # destination team — otherwise a team admin could shed projects into
+        # an unsuspecting team's namespace.
+        if data.team_id is not None and data.team_id != existing_project.team_id:
+            can_assign_to_target = await _check_user_permission_for_project(
+                user_api_key_dict=user_api_key_dict,
+                team_id=data.team_id,
+                prisma_client=prisma_client,
+                team_object=(
+                    LiteLLM_TeamTable(**target_team_obj.model_dump())
+                    if target_team_obj
+                    else None
+                ),
+            )
+            if not can_assign_to_target:
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "Cannot reassign project to a team you are not an admin of"
+                    },
+                )
+
         # Validate project limits against team limits
-        if team_obj_for_checks is not None:
+        if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**team_obj_for_checks.model_dump()),
+                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
                 data=data,
             )
 
@@ -857,10 +876,16 @@ async def project_info(
                 where={"team_id": project.team_id}
             )
             if team:
-                is_team_member = (
-                    user_api_key_dict.user_id in team.admins
-                    or user_api_key_dict.user_id in team.members
-                )
+                caller_user_id = user_api_key_dict.user_id
+                for m in team.members_with_roles or []:
+                    m_user_id = (
+                        m.get("user_id")
+                        if isinstance(m, dict)
+                        else getattr(m, "user_id", None)
+                    )
+                    if m_user_id == caller_user_id:
+                        is_team_member = True
+                        break
 
         if not (is_admin or is_team_member):
             raise HTTPException(
@@ -911,20 +936,20 @@ async def list_projects(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
-            # Get projects for teams the user belongs to
-            user_teams = await prisma_client.db.litellm_teamtable.find_many(
-                where={
-                    "OR": [
-                        {"members": {"has": user_api_key_dict.user_id}},
-                        {"admins": {"has": user_api_key_dict.user_id}},
-                    ]
-                }
+            # Look up the user's team memberships via the reverse-index on
+            # LiteLLM_UserTable.teams (maintained by team_member_add alongside
+            # members_with_roles). This avoids a full scan of all team rows.
+            user_record = await prisma_client.db.litellm_usertable.find_unique(
+                where={"user_id": user_api_key_dict.user_id},
+            )
+            user_team_ids = (
+                user_record.teams
+                if user_record is not None and user_record.teams
+                else []
             )
 
-            team_ids = [team.team_id for team in user_teams]
-
             projects = await prisma_client.db.litellm_projecttable.find_many(
-                where={"team_id": {"in": team_ids}},
+                where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )
 

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.39"
+version = "0.1.40"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
 Documentation = "https://docs.litellm.ai"
 
 [build-system]
-requires = ["uv_build==0.10.7"]
+requires = ["uv_build==0.11.8"]
 build-backend = "uv_build"
 
 [tool.uv]
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.39"
+version = "0.1.40"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

index.yaml

@@ -1,108 +0,0 @@
-apiVersion: v1
-entries:
-  litellm-helm:
-  - apiVersion: v2
-    appVersion: v1.43.18
-    created: "2024-08-19T23:58:25.331689+08:00"
-    dependencies:
-    - condition: db.deployStandalone
-      name: postgresql
-      repository: oci://registry-1.docker.io/bitnamicharts
-      version: '>=13.3.0'
-    - condition: redis.enabled
-      name: redis
-      repository: oci://registry-1.docker.io/bitnamicharts
-      version: '>=18.0.0'
-    description: Call all LLM APIs using the OpenAI format
-    digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531
-    name: litellm-helm
-    type: application
-    urls:
-    - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz
-    version: 0.2.3
-  postgresql:
-  - annotations:
-      category: Database
-      images: |
-        - name: os-shell
-          image: docker.io/bitnami/os-shell:12-debian-12-r16
-        - name: postgres-exporter
-          image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14
-        - name: postgresql
-          image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6
-      licenses: Apache-2.0
-    apiVersion: v2
-    appVersion: 16.2.0
-    created: "2024-08-19T23:58:25.335716+08:00"
-    dependencies:
-    - name: common
-      repository: oci://registry-1.docker.io/bitnamicharts
-      tags:
-      - bitnami-common
-      version: 2.x.x
-    description: PostgreSQL (Postgres) is an open source object-relational database
-      known for reliability and data integrity. ACID-compliant, it supports foreign
-      keys, joins, views, triggers and stored procedures.
-    digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167
-    home: https://bitnami.com
-    icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png
-    keywords:
-    - postgresql
-    - postgres
-    - database
-    - sql
-    - replication
-    - cluster
-    maintainers:
-    - name: VMware, Inc.
-      url: https://github.com/bitnami/charts
-    name: postgresql
-    sources:
-    - https://github.com/bitnami/charts/tree/main/bitnami/postgresql
-    urls:
-    - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz
-    version: 14.3.1
-  redis:
-  - annotations:
-      category: Database
-      images: |
-        - name: kubectl
-          image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3
-        - name: os-shell
-          image: docker.io/bitnami/os-shell:12-debian-12-r16
-        - name: redis
-          image: docker.io/bitnami/redis:7.2.4-debian-12-r9
-        - name: redis-exporter
-          image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4
-        - name: redis-sentinel
-          image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7
-      licenses: Apache-2.0
-    apiVersion: v2
-    appVersion: 7.2.4
-    created: "2024-08-19T23:58:25.339392+08:00"
-    dependencies:
-    - name: common
-      repository: oci://registry-1.docker.io/bitnamicharts
-      tags:
-      - bitnami-common
-      version: 2.x.x
-    description: Redis(R) is an open source, advanced key-value store. It is often
-      referred to as a data structure server since keys can contain strings, hashes,
-      lists, sets and sorted sets.
-    digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984
-    home: https://bitnami.com
-    icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png
-    keywords:
-    - redis
-    - keyvalue
-    - database
-    maintainers:
-    - name: VMware, Inc.
-      url: https://github.com/bitnami/charts
-    name: redis
-    sources:
-    - https://github.com/bitnami/charts/tree/main/bitnami/redis
-    urls:
-    - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz
-    version: 18.19.1
-generated: "2024-08-19T23:58:25.322532+08:00"

litellm-js/proxy/.npmrc

@@ -1,5 +0,0 @@
-# Supply-chain hardening
-# Packages needing lifecycle scripts: npm rebuild <pkg>
-ignore-scripts=true
-# Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d

litellm-js/proxy/README.md

@@ -1,8 +0,0 @@
-```
-npm install
-npm run dev
-```
-
-```
-npm run deploy
-```

litellm-js/proxy/package.json

@@ -1,14 +0,0 @@
-{
-  "scripts": {
-    "dev": "wrangler dev src/index.ts",
-    "deploy": "wrangler deploy --minify src/index.ts"
-  },
-  "dependencies": {
-    "hono": "4.12.12",
-    "openai": "4.29.2"
-  },
-  "devDependencies": {
-    "@cloudflare/workers-types": "4.20240208.0",
-    "wrangler": "3.32.0"
-  }
-}

litellm-js/proxy/src/index.ts

@@ -1,59 +0,0 @@
-import { Hono } from 'hono'
-import { Context } from 'hono';
-import { bearerAuth } from 'hono/bearer-auth'
-import OpenAI from "openai";
-
-const openai = new OpenAI({
-  apiKey: "sk-1234",
-  baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev"
-});
-
-async function call_proxy() {
-  const completion = await openai.chat.completions.create({
-    messages: [{ role: "system", content: "You are a helpful assistant." }],
-    model: "gpt-3.5-turbo",
-  });
-
-  return completion
-}
-
-const app = new Hono()
-
-// Middleware for API Key Authentication
-const apiKeyAuth = async (c: Context, next: Function) => {
-  const apiKey = c.req.header('Authorization');
-  if (!apiKey || apiKey !== 'Bearer sk-1234') {
-    return c.text('Unauthorized', 401);
-  }
-  await next();
-};
-
-
-app.use('/*', apiKeyAuth)
-
-
-app.get('/', (c) => {
-  return c.text('Hello Hono!')
-})
-
-
-
-
-// Handler for chat completions
-const chatCompletionHandler = async (c: Context) => {
-  // Assuming your logic for handling chat completion goes here
-  // For demonstration, just returning a simple JSON response
-  const response = await call_proxy()
-  return c.json(response);
-};
-
-// Register the above handler for different POST routes with the apiKeyAuth middleware
-app.post('/v1/chat/completions', chatCompletionHandler);
-app.post('/chat/completions', chatCompletionHandler);
-
-// Example showing how you might handle dynamic segments within the URL
-// Here, using ':model*' to capture the rest of the path as a parameter 'model'
-app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler);
-
-
-export default app

litellm-js/proxy/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "strict": true,
-    "lib": [
-      "ESNext"
-    ],
-    "types": [
-      "@cloudflare/workers-types"
-    ],
-    "jsx": "react-jsx",
-    "jsxImportSource": "hono/jsx",
-    "skipLibCheck": true
-  },
-}

litellm-js/proxy/wrangler.toml

@@ -1,18 +0,0 @@
-name = "my-app"
-compatibility_date = "2023-12-01"
-
-# [vars]
-# MY_VAR = "my-variable"
-
-# [[kv_namespaces]]
-# binding = "MY_KV_NAMESPACE"
-# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-
-# [[r2_buckets]]
-# binding = "MY_BUCKET"
-# bucket_name = "my-bucket"
-
-# [[d1_databases]]
-# binding = "DB"
-# database_name = "my-database"
-# database_id = ""

litellm-js/spend-logs/.npmrc

@@ -1,5 +0,0 @@
-# Supply-chain hardening
-# Packages needing lifecycle scripts: npm rebuild <pkg>
-ignore-scripts=true
-# Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d

litellm-js/spend-logs/Dockerfile

@@ -1,26 +0,0 @@
-# Use the specific Node.js v20.11.0 image
-FROM node:20.18.1-alpine3.20
-
-# Set the working directory inside the container
-WORKDIR /app
-
-# Copy package.json and package-lock.json to the working directory
-COPY ./litellm-js/spend-logs/package*.json ./
-
-# Install dependencies
-RUN npm ci
-
-# Install Prisma globally
-RUN npm install -g prisma
-
-# Copy the rest of the application code
-COPY ./litellm-js/spend-logs .
-
-# Generate Prisma client
-RUN npx prisma generate
-
-# Expose the port that the Node.js server will run on
-EXPOSE 3000
-
-# Command to run the Node.js app with npm run dev
-CMD ["npm", "run", "dev"]

litellm-js/spend-logs/README.md

@@ -1,8 +0,0 @@
-```
-npm install
-npm run dev
-```
-
-```
-open http://localhost:3000
-```

litellm-js/spend-logs/package-lock.json

@@ -1,597 +0,0 @@
-{
-  "name": "spend-logs",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "dependencies": {
-        "@hono/node-server": "1.19.13",
-        "hono": "4.12.12"
-      },
-      "devDependencies": {
-        "@types/node": "20.19.25",
-        "tsx": "4.20.6"
-      }
-    },
-    "node_modules/@esbuild/aix-ppc64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
-      "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==",
-      "cpu": [
-        "ppc64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "aix"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-arm": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz",
-      "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz",
-      "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz",
-      "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/darwin-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz",
-      "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/darwin-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz",
-      "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/freebsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/freebsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz",
-      "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-arm": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz",
-      "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz",
-      "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-ia32": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz",
-      "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==",
-      "cpu": [
-        "ia32"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-loong64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz",
-      "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==",
-      "cpu": [
-        "loong64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-mips64el": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz",
-      "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==",
-      "cpu": [
-        "mips64el"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-ppc64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz",
-      "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==",
-      "cpu": [
-        "ppc64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-riscv64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz",
-      "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==",
-      "cpu": [
-        "riscv64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-s390x": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz",
-      "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==",
-      "cpu": [
-        "s390x"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz",
-      "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/netbsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "netbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/netbsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz",
-      "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "netbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openbsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openbsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz",
-      "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openharmony-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz",
-      "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openharmony"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/sunos-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz",
-      "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "sunos"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz",
-      "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-ia32": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz",
-      "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==",
-      "cpu": [
-        "ia32"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz",
-      "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@hono/node-server": {
-      "version": "1.19.13",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz",
-      "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=18.14.1"
-      },
-      "peerDependencies": {
-        "hono": "^4"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "20.19.25",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
-      "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "undici-types": "~6.21.0"
-      }
-    },
-    "node_modules/esbuild": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
-      "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "bin": {
-        "esbuild": "bin/esbuild"
-      },
-      "engines": {
-        "node": ">=18"
-      },
-      "optionalDependencies": {
-        "@esbuild/aix-ppc64": "0.25.12",
-        "@esbuild/android-arm": "0.25.12",
-        "@esbuild/android-arm64": "0.25.12",
-        "@esbuild/android-x64": "0.25.12",
-        "@esbuild/darwin-arm64": "0.25.12",
-        "@esbuild/darwin-x64": "0.25.12",
-        "@esbuild/freebsd-arm64": "0.25.12",
-        "@esbuild/freebsd-x64": "0.25.12",
-        "@esbuild/linux-arm": "0.25.12",
-        "@esbuild/linux-arm64": "0.25.12",
-        "@esbuild/linux-ia32": "0.25.12",
-        "@esbuild/linux-loong64": "0.25.12",
-        "@esbuild/linux-mips64el": "0.25.12",
-        "@esbuild/linux-ppc64": "0.25.12",
-        "@esbuild/linux-riscv64": "0.25.12",
-        "@esbuild/linux-s390x": "0.25.12",
-        "@esbuild/linux-x64": "0.25.12",
-        "@esbuild/netbsd-arm64": "0.25.12",
-        "@esbuild/netbsd-x64": "0.25.12",
-        "@esbuild/openbsd-arm64": "0.25.12",
-        "@esbuild/openbsd-x64": "0.25.12",
-        "@esbuild/openharmony-arm64": "0.25.12",
-        "@esbuild/sunos-x64": "0.25.12",
-        "@esbuild/win32-arm64": "0.25.12",
-        "@esbuild/win32-ia32": "0.25.12",
-        "@esbuild/win32-x64": "0.25.12"
-      }
-    },
-    "node_modules/fsevents": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
-      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-      }
-    },
-    "node_modules/get-tsconfig": {
-      "version": "4.13.0",
-      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
-      "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "resolve-pkg-maps": "^1.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
-      }
-    },
-    "node_modules/hono": {
-      "version": "4.12.12",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz",
-      "integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=16.9.0"
-      }
-    },
-    "node_modules/resolve-pkg-maps": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
-      "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
-      "dev": true,
-      "license": "MIT",
-      "funding": {
-        "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
-      }
-    },
-    "node_modules/tsx": {
-      "version": "4.20.6",
-      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
-      "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "esbuild": "~0.25.0",
-        "get-tsconfig": "^4.7.5"
-      },
-      "bin": {
-        "tsx": "dist/cli.mjs"
-      },
-      "engines": {
-        "node": ">=18.0.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.3"
-      }
-    },
-    "node_modules/undici-types": {
-      "version": "6.21.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
-      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
-      "dev": true,
-      "license": "MIT"
-    }
-  }
-}

litellm-js/spend-logs/package.json

@@ -1,13 +0,0 @@
-{
-  "scripts": {
-    "dev": "tsx watch src/index.ts"
-  },
-  "dependencies": {
-    "@hono/node-server": "1.19.13",
-    "hono": "4.12.12"
-  },
-  "devDependencies": {
-    "@types/node": "20.19.25",
-    "tsx": "4.20.6"
-  }
-}

litellm-js/spend-logs/schema.prisma

@@ -1,29 +0,0 @@
-generator client {
-  provider        = "prisma-client-js"
-}
-
-datasource client {
-  provider = "postgresql"
-  url      = env("DATABASE_URL")
-}
-
-model LiteLLM_SpendLogs {
-  request_id        String   @id
-  call_type         String
-  api_key           String   @default("")
-  spend             Float    @default(0.0)
-  total_tokens      Int      @default(0)
-  prompt_tokens     Int      @default(0)
-  completion_tokens Int      @default(0)
-  startTime         DateTime
-  endTime           DateTime
-  model             String   @default("")
-  api_base          String   @default("")
-  user              String   @default("")
-  metadata          Json     @default("{}")
-  cache_hit         String   @default("")
-  cache_key         String   @default("")
-  request_tags      Json     @default("[]")
-  team_id           String?
-  end_user          String?
-}

litellm-js/spend-logs/src/_types.ts

@@ -1,32 +0,0 @@
-export type LiteLLM_IncrementSpend = {
-    key_transactions: Array<LiteLLM_IncrementObject>, // [{"key": spend},..]
-    user_transactions: Array<LiteLLM_IncrementObject>, 
-    team_transactions: Array<LiteLLM_IncrementObject>,
-    spend_logs_transactions: Array<LiteLLM_SpendLogs>
-}
-
-export type LiteLLM_IncrementObject = {
-    key: string,
-    spend: number
-}
-
-export type LiteLLM_SpendLogs = {
-    request_id: string; // @id means it's a unique identifier
-    call_type: string;
-    api_key: string; // @default("") means it defaults to an empty string if not provided
-    spend: number; // Float in Prisma corresponds to number in TypeScript
-    total_tokens: number; // Int in Prisma corresponds to number in TypeScript
-    prompt_tokens: number;
-    completion_tokens: number;
-    startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript
-    endTime: Date;
-    model: string; // @default("") means it defaults to an empty string if not provided
-    api_base: string;
-    user: string;
-    metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known
-    cache_hit: string;
-    cache_key: string;
-    request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure
-    team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided
-    end_user?: string | null;
-};

litellm-js/spend-logs/src/index.ts

@@ -1,84 +0,0 @@
-import { serve } from '@hono/node-server'
-import { Hono } from 'hono'
-import { PrismaClient } from '@prisma/client'
-import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types'
-
-const app = new Hono()
-const prisma = new PrismaClient()
-// In-memory storage for logs
-let spend_logs: LiteLLM_SpendLogs[] = [];
-const key_logs: LiteLLM_IncrementObject[] = [];
-const user_logs: LiteLLM_IncrementObject[] = [];
-const transaction_logs: LiteLLM_IncrementObject[] = [];
-
-
-app.get('/', (c) => {
-  return c.text('Hello Hono!')
-})
-
-const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush
-const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again
-const BATCH_SIZE = 100; // Preferred size of each batch to write to the database
-const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval
-
-const flushLogsToDb = async () => {
-  if (spend_logs.length >= MIN_LOGS) {
-    // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less
-    const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL);
-  
-    for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) {
-      // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE
-      const batch = logsToProcess.slice(i, i + BATCH_SIZE);
-
-      // Convert datetime strings to Date objects
-      const batchWithDates = batch.map(entry => ({
-        ...entry,
-        startTime: new Date(entry.startTime),
-        endTime: new Date(entry.endTime),
-        // Repeat for any other DateTime fields you may have
-      }));
-
-      await prisma.liteLLM_SpendLogs.createMany({
-        data: batchWithDates,
-      });
-
-      console.log(`Flushed ${batch.length} logs to the DB.`);
-    }
-
-    // Remove the processed logs from spend_logs
-    spend_logs = spend_logs.slice(logsToProcess.length);
-    
-    console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`);
-  } else {
-    // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold.
-    if(spend_logs.length > 0) {
-      console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`);
-    } else {
-      console.log("No logs to flush.");
-    }
-  }
-};
-
-// Setup interval for attempting to flush the logs
-setInterval(flushLogsToDb, FLUSH_INTERVAL);
-
-// Route to receive log messages
-app.post('/spend/update', async (c) => {
-  const incomingLogs = await c.req.json<LiteLLM_SpendLogs[]>();
-  
-  spend_logs.push(...incomingLogs);
-
-  console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`);
-  
-  return c.json({ message: `Successfully stored ${incomingLogs.length} logs` });
-});
-
-
-
-const port = 3000
-console.log(`Server is running on port ${port}`)
-
-serve({
-  fetch: app.fetch,
-  port
-})

litellm-js/spend-logs/tsconfig.json

@@ -1,13 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "strict": true,
-    "types": [
-      "node"
-    ],
-    "jsx": "react-jsx",
-    "jsxImportSource": "hono/jsx",
-  }
-}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260421135425_add_team_membership_total_spend/migration.sql

@@ -1,3 +1,3 @@
 -- AlterTable
-ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN     "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+ALTER TABLE "LiteLLM_TeamMembership" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql

@@ -0,0 +1,2 @@
+-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores).
+ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql

@@ -0,0 +1,75 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowRun" (
+    "run_id" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "workflow_type" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "created_by" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "input" JSONB,
+    "output" JSONB,
+    "metadata" JSONB,
+
+    CONSTRAINT "LiteLLM_WorkflowRun_pkey" PRIMARY KEY ("run_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowEvent" (
+    "event_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "event_type" TEXT NOT NULL,
+    "step_name" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "data" JSONB,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowMessage" (
+    "message_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "role" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "session_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowMessage_pkey" PRIMARY KEY ("message_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowRun_session_id_key" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_workflow_type_status_idx" ON "LiteLLM_WorkflowRun"("workflow_type", "status");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_session_id_idx" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_at_idx" ON "LiteLLM_WorkflowRun"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_by_idx" ON "LiteLLM_WorkflowRun"("created_by");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowEvent_run_id_idx" ON "LiteLLM_WorkflowEvent"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowEvent_run_id_sequence_number_key" ON "LiteLLM_WorkflowEvent"("run_id", "sequence_number");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowMessage_run_id_idx" ON "LiteLLM_WorkflowMessage"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowMessage_run_id_sequence_number_key" ON "LiteLLM_WorkflowMessage"("run_id", "sequence_number");
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowEvent" ADD CONSTRAINT "LiteLLM_WorkflowEvent_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowMessage" ADD CONSTRAINT "LiteLLM_WorkflowMessage_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260501195714_managed_resource_team_owner/migration.sql

@@ -0,0 +1,20 @@
+-- Adds `team_id` to managed-resource tables so service-account API
+-- keys (no `user_id`) can be scoped by team instead of bypassing the
+-- `created_by` filter entirely. Existing rows keep `team_id = NULL`
+-- and become invisible to team-only callers — that is the intended isolation
+-- outcome; backfill manually if legacy rows must remain visible.
+--
+-- The composite indexes match the listing query: filter by team owner, sort by
+-- created_at DESC. Tables are typically small (resources per tenant, not per
+-- request); a future operator with a large table can switch to
+-- CREATE INDEX CONCURRENTLY in a follow-up migration.
+
+ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+ALTER TABLE "LiteLLM_ManagedVectorStoreTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+
+-- Index names follow Prisma's auto-generated convention so `prisma migrate diff`
+-- against the schema is clean.
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_team_id_created_at_idx" ON "LiteLLM_ManagedFileTable" ("team_id", "created_at" DESC);
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedObjectTable_team_id_created_at_idx" ON "LiteLLM_ManagedObjectTable" ("team_id", "created_at" DESC);
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable_team_id_created_at_idx" ON "LiteLLM_ManagedVectorStoreTable" ("team_id", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260513120000_add_delegate_auth_to_upstream_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "delegate_auth_to_upstream" BOOLEAN NOT NULL DEFAULT false;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
   models                String[]       @default([])
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
+  search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -322,6 +323,7 @@ model LiteLLM_MCPServerTable {
   registration_url   String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
+  delegate_auth_to_upstream Boolean @default(false)
   is_byok               Boolean  @default(false)
   byok_description      String[] @default([])
   byok_api_key_help_url String?
@@ -883,28 +885,32 @@ model LiteLLM_ManagedFileTable {
   storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
   storage_url String? // The actual storage URL where the file is stored
   created_at DateTime @default(now())
-  created_by String? 
+  created_by String?
+  team_id String? // Team that owns the resource; populated for service-account keys without a user_id so listings can isolate by team.
   updated_at DateTime @updatedAt
   updated_by String?
 
   @@index([unified_file_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
-model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the 
+model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
   id String @id @default(uuid())
   unified_object_id String @unique // The base64 encoded unified file ID
-  model_object_id String @unique // the id returned by the backend API provider 
+  model_object_id String @unique // the id returned by the backend API provider
   file_object Json // Stores the OpenAIFileObject
   file_purpose String // either 'batch' or 'fine-tune'
-  status String? // check if batch cost has been tracked  
+  status String? // check if batch cost has been tracked
   batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
   created_at DateTime @default(now())
   created_by String?
+  team_id String?
   updated_at DateTime @updatedAt
-  updated_by String? 
+  updated_by String?
 
   @@index([unified_object_id])
   @@index([model_object_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedVectorStoreTable {
@@ -917,10 +923,12 @@ model LiteLLM_ManagedVectorStoreTable {
   storage_url String? // Storage URL (if applicable)
   created_at DateTime @default(now())
   created_by String?
+  team_id String?
   updated_at DateTime @updatedAt
   updated_by String?
 
   @@index([unified_resource_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedVectorStoresTable {
@@ -1290,3 +1298,80 @@ model LiteLLM_AdaptiveRouterSession {
   @@id([session_id, router_name, model_name])
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
+
+// ---------------------------------------------------------------------------
+// Workflow Run Tracking
+//
+// Generic durable state tracking for any agent or automated workflow.
+// Design: three tables — run (header + materialized status), event (append-only
+// source of truth for state transitions), message (conversation inbox/outbox).
+//
+// Usage:
+//   - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
+//   - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
+//   - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
+//     the proxy — all spend logs for this run are automatically tagged.
+// ---------------------------------------------------------------------------
+
+// One instance of work being done. `status` is a materialized cache of the
+// latest event; the event log is the authoritative source of truth.
+model LiteLLM_WorkflowRun {
+  run_id        String   @id @default(uuid())
+  session_id    String   @unique @default(uuid())
+  workflow_type String
+  status        String   @default("pending")
+  created_by    String?  // user_id of the key that created this run; null = created by master key
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+  input         Json?
+  output        Json?
+  metadata      Json?
+
+  events   LiteLLM_WorkflowEvent[]
+  messages LiteLLM_WorkflowMessage[]
+
+  @@index([workflow_type, status])
+  @@index([session_id])
+  @@index([created_at])
+  @@index([created_by])
+}
+
+// Append-only log of state transitions. Never mutate rows here.
+// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
+// Status auto-update rules (applied by the append endpoint):
+//   step.started   → run.status = running
+//   step.failed    → run.status = failed
+//   hook.waiting   → run.status = paused
+//   hook.received  → run.status = running
+model LiteLLM_WorkflowEvent {
+  event_id        String   @id @default(uuid())
+  run_id          String
+  event_type      String
+  step_name       String
+  sequence_number Int
+  data            Json?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}
+
+// Conversation inbox/outbox — full message content, separate from the durable
+// event log. Spend logs truncate messages; this table stores them in full.
+// `session_id` here is the Claude --resume session ID (or similar).
+model LiteLLM_WorkflowMessage {
+  message_id      String   @id @default(uuid())
+  run_id          String
+  role            String
+  content         String
+  sequence_number Int
+  session_id      String?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.69"
+version = "0.4.72"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
 Documentation = "https://docs.litellm.ai"
 
 [build-system]
-requires = ["uv_build==0.10.7"]
+requires = ["uv_build==0.11.8"]
 build-backend = "uv_build"
 
 [tool.uv]
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.69"
+version = "0.4.72"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -166,7 +166,7 @@ langfuse_default_tags: Optional[List[str]] = None
 langsmith_batch_size: Optional[int] = None
 prometheus_initialize_budget_metrics: Optional[bool] = False
 prometheus_latency_buckets: Optional[List[float]] = None
-require_auth_for_metrics_endpoint: Optional[bool] = False
+require_auth_for_metrics_endpoint: Optional[bool] = True
 argilla_batch_size: Optional[int] = None
 datadog_use_v1: Optional[bool] = False  # if you want to use v1 datadog logged payload.
 gcs_pub_sub_use_v1: Optional[bool] = (
@@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = (
 )
 store_audit_logs = False  # Enterprise feature, allow users to see audit logs
 skip_system_message_in_guardrail: bool = False
+skip_tool_message_in_guardrail: bool = False
 ### end of callbacks #############
 
 email: Optional[str] = (
@@ -280,6 +281,7 @@ ssl_security_level: Optional[str] = None
 ssl_certificate: Optional[str] = None
 user_url_validation: bool = True
 user_url_allowed_hosts: List[str] = []
+provider_url_destination_allowed_hosts: List[str] = []
 ssl_ecdh_curve: Optional[str] = (
     None  # Set to 'X25519' to disable PQC and improve performance
 )
@@ -288,6 +290,7 @@ disable_token_counter: bool = False
 disable_add_transform_inline_image_block: bool = False
 disable_add_user_agent_to_request_tags: bool = False
 disable_anthropic_gemini_context_caching_transform: bool = False
+disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"
 safe_memory_mode: bool = False
@@ -330,6 +333,9 @@ enable_model_config_credential_overrides: bool = False
 enable_key_alias_format_validation: bool = (
     False  # opt-in validation of key_alias format on /key/generate and /key/update
 )
+enable_gemini_default_thinking_level_low: bool = (
+    False  # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
+)
 ####################
 logging: bool = True
 enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@@ -383,6 +389,7 @@ anthropic_beta_headers_url: str = os.getenv(
 suppress_debug_info = False
 dynamodb_table_name: Optional[str] = None
 s3_callback_params: Optional[Dict] = None
+s3_audit_callback_params: Optional[Dict] = None
 datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
 datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
 aws_sqs_callback_params: Optional[Dict] = None
@@ -409,6 +416,9 @@ custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
 prometheus_emit_stream_label: bool = False
+prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
+prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
+prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
 disable_add_prefix_to_prompt: bool = (
     False  # used by anthropic, to disable adding prefix to prompt
 )
@@ -581,6 +591,7 @@ anyscale_models: Set = set()
 cerebras_models: Set = set()
 galadriel_models: Set = set()
 nvidia_nim_models: Set = set()
+nvidia_riva_models: Set = set()
 sambanova_models: Set = set()
 sambanova_embedding_models: Set = set()
 novita_models: Set = set()
@@ -807,6 +818,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             galadriel_models.add(key)
         elif value.get("litellm_provider") == "nvidia_nim":
             nvidia_nim_models.add(key)
+        elif value.get("litellm_provider") == "nvidia_riva":
+            nvidia_riva_models.add(key)
         elif value.get("litellm_provider") == "sambanova":
             sambanova_models.add(key)
         elif value.get("litellm_provider") == "sambanova-embedding-models":
@@ -966,6 +979,7 @@ model_list = list(
     | cerebras_models
     | galadriel_models
     | nvidia_nim_models
+    | nvidia_riva_models
     | sambanova_models
     | azure_text_models
     | novita_models
@@ -1062,6 +1076,7 @@ models_by_provider: dict = {
     "cerebras": cerebras_models,
     "galadriel": galadriel_models,
     "nvidia_nim": nvidia_nim_models,
+    "nvidia_riva": nvidia_riva_models,
     "sambanova": sambanova_models | sambanova_embedding_models,
     "novita": novita_models,
     "nebius": nebius_models | nebius_embedding_models,
@@ -1411,6 +1426,12 @@ if TYPE_CHECKING:
     )
     from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig
     from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig
+    from .llms.bedrock.claude_platform.transformation import (
+        BedrockClaudePlatformConfig as BedrockClaudePlatformConfig,
+    )
+    from .llms.bedrock.claude_platform.messages_transformation import (
+        BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig,
+    )
     from .llms.anthropic.completion.transformation import (
         AnthropicTextConfig as AnthropicTextConfig,
     )
@@ -1613,6 +1634,9 @@ if TYPE_CHECKING:
     from .llms.deepgram.audio_transcription.transformation import (
         DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
     )
+    from .llms.nvidia_riva.audio_transcription.transformation import (
+        NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
+    )
     from .llms.topaz.image_variations.transformation import (
         TopazImageVariationConfig as TopazImageVariationConfig,
     )

litellm/_lazy_imports_registry.py

@@ -131,6 +131,7 @@ LLM_CONFIG_NAMES = (
     "OpenrouterConfig",
     "DataRobotConfig",
     "AnthropicConfig",
+    "BedrockClaudePlatformConfig",
     "AnthropicTextConfig",
     "GroqSTTConfig",
     "TritonConfig",
@@ -170,6 +171,7 @@ LLM_CONFIG_NAMES = (
     "SagemakerNovaConfig",
     "CohereChatConfig",
     "AnthropicMessagesConfig",
+    "BedrockClaudePlatformMessagesConfig",
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
@@ -610,6 +612,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
     "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"),
     "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"),
     "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"),
+    "BedrockClaudePlatformConfig": (
+        ".llms.bedrock.claude_platform.transformation",
+        "BedrockClaudePlatformConfig",
+    ),
     "AnthropicTextConfig": (
         ".llms.anthropic.completion.transformation",
         "AnthropicTextConfig",
@@ -712,6 +718,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
         ".llms.anthropic.experimental_pass_through.messages.transformation",
         "AnthropicMessagesConfig",
     ),
+    "BedrockClaudePlatformMessagesConfig": (
+        ".llms.bedrock.claude_platform.messages_transformation",
+        "BedrockClaudePlatformMessagesConfig",
+    ),
     "AmazonAnthropicClaudeMessagesConfig": (
         ".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation",
         "AmazonAnthropicClaudeMessagesConfig",

litellm/_logging.py

@@ -1,12 +1,12 @@
 import ast
 import logging
 import os
-import re
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 
@@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = (
     os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
 )
 
-_REDACTED = "REDACTED"
-
-
-def _build_secret_patterns() -> re.Pattern:
-    patterns: List[str] = [
-        # ── PEM private key / certificate blocks ──
-        r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
-        # ── GCP OAuth2 access tokens (ya29.*) ──
-        r"\bya29\.[A-Za-z0-9_.~+/-]+",
-        # ── Credential %s formatting (space separator, no key= prefix) ──
-        r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
-        # AWS access key IDs
-        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
-        # AWS secrets / session tokens / access key IDs (key=value)
-        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
-        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
-        # Bearer tokens (OAuth, JWT, etc.)
-        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
-        # Basic auth headers
-        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
-        # OpenAI / Anthropic sk- prefixed keys
-        r"sk-[A-Za-z0-9\-_]{20,}",
-        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
-        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
-        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
-        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Anthropic internal header keys
-        r"x-ak-[A-Za-z0-9\-_]{20,}",
-        # Google API keys
-        r"AIza[0-9A-Za-z\-_]{35}",
-        # Password / secret params (handles key=value and 'key': 'value')
-        # Word boundary prevents O(n^2) backtracking on long word-char runs.
-        r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
-        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Database connection string credentials (scheme://user:pass@host)
-        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
-        # Databricks personal access tokens
-        r"dapi[0-9a-f]{32}",
-        # ── Key-name-based redaction ──
-        # Catches secrets inside dicts/config dumps by matching on the KEY name
-        # regardless of what the value looks like.
-        # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
-        # private_key with PEM-aware value capture
-        r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
-        r"(?:master_key|database_url|db_url|connection_string|"
-        r"signing_key|encryption_key|"
-        r"auth_token|access_token|refresh_token|"
-        r"slack_webhook_url|webhook_url|"
-        r"database_connection_string|"
-        r"huggingface_token|jwt_secret)"
-        r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
-        # ── Raw JWTs (without Bearer prefix) ──
-        r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
-        # ── Azure SAS tokens in URLs ──
-        r"[?&]sig=[A-Za-z0-9%+/=]+",
-        # ── Full JSON service-account blobs (single-line and multi-line) ──
-        r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
-    ]
-    return re.compile("|".join(patterns), re.IGNORECASE)
-
-
-_SECRET_RE = _build_secret_patterns()
-
 
 def _redact_string(value: str) -> str:
     if not _ENABLE_SECRET_REDACTION:
         return value
-    return _SECRET_RE.sub(_REDACTED, value)
+    return redact_string(value)
 
 
 def redact_secrets(value: str) -> str:

litellm/_redis.py

@@ -19,6 +19,7 @@ import redis.asyncio as async_redis  # type: ignore
 
 from litellm import get_secret, get_secret_str
 from litellm._redis_credential_provider import (
+    AzureADCredentialProvider,
     GCPIAMCredentialProvider,
     _generate_gcp_iam_access_token,
 )
@@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 from ._logging import verbose_logger
 
+AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
+
 
 def _get_redis_kwargs():
     arg_spec = inspect.getfullargspec(redis.Redis)
@@ -38,14 +41,18 @@ def _get_redis_kwargs():
         "retry",
     }
 
-    include_args = [
+    include_args = {
         "url",
         "redis_connect_func",
         "gcp_service_account",
         "gcp_ssl_ca_certs",
-    ]
+        "azure_redis_ad_token",
+        "azure_client_id",
+        "azure_tenant_id",
+        "azure_client_secret",
+    }
 
-    available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
+    available_args = {x for x in arg_spec.args if x not in exclude_args} | include_args
 
     return available_args
 
@@ -77,19 +84,23 @@ def _get_redis_cluster_kwargs(client=None):
     # Only allow primitive arguments
     exclude_args = {"self", "connection_pool", "retry", "host", "port", "startup_nodes"}
 
-    available_args = [x for x in arg_spec.args if x not in exclude_args]
-    available_args.append("password")
-    available_args.append("username")
-    available_args.append("ssl")
-    available_args.append("ssl_cert_reqs")
-    available_args.append("ssl_check_hostname")
-    available_args.append("ssl_ca_certs")
-    available_args.append(
-        "redis_connect_func"
-    )  # Needed for sync clusters and IAM detection
-    available_args.append("gcp_service_account")
-    available_args.append("gcp_ssl_ca_certs")
-    available_args.append("max_connections")
+    available_args = {x for x in arg_spec.args if x not in exclude_args}
+    available_args |= {
+        "password",
+        "username",
+        "ssl",
+        "ssl_cert_reqs",
+        "ssl_check_hostname",
+        "ssl_ca_certs",
+        "redis_connect_func",  # Needed for sync clusters and IAM detection
+        "gcp_service_account",
+        "gcp_ssl_ca_certs",
+        "azure_redis_ad_token",
+        "azure_client_id",
+        "azure_tenant_id",
+        "azure_client_secret",
+        "max_connections",
+    }
 
     return available_args
 
@@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func(
     return iam_connect
 
 
+def _build_azure_credential(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+):
+    """
+    Build a long-lived Azure credential object.
+
+    Azure SDK credentials cache tokens internally and handle expiry/refresh
+    transparently, so this should be called once and the result reused.
+    """
+    try:
+        from azure.identity import (
+            ClientSecretCredential,
+            DefaultAzureCredential,
+            ManagedIdentityCredential,
+        )
+    except ImportError:
+        raise ImportError(
+            "azure-identity is required for Azure AD Redis authentication. "
+            "Install it with: pip install azure-identity"
+        )
+
+    _client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID")
+    _tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID")
+    _client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET")
+
+    if _client_id and _tenant_id and _client_secret:
+        return ClientSecretCredential(
+            client_id=_client_id,
+            tenant_id=_tenant_id,
+            client_secret=_client_secret,
+        )
+    elif _client_id:
+        return ManagedIdentityCredential(client_id=_client_id)
+    else:
+        return DefaultAzureCredential()
+
+
+def _generate_azure_ad_redis_token(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+) -> str:
+    """
+    One-shot helper that builds a credential and fetches a single Azure AD
+    access token for Redis. Each call rebuilds the credential and performs a
+    network round-trip, so it should not be used in steady-state Redis flows
+    — the sync (``create_azure_ad_redis_connect_func``) and async paths
+    (``AzureADCredentialProvider``) keep the credential alive across
+    connections so the Azure SDK's internal cache + silent refresh apply.
+    """
+    credential = _build_azure_credential(
+        azure_client_id=azure_client_id,
+        azure_tenant_id=azure_tenant_id,
+        azure_client_secret=azure_client_secret,
+    )
+    token = credential.get_token(AZURE_REDIS_SCOPE)
+    return token.token
+
+
+def create_azure_ad_redis_connect_func(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+) -> Callable:
+    """
+    Creates a custom Redis connection function for Azure AD authentication.
+
+    Used for sync Redis clients. The credential is created once (captured by the
+    closure) and reused across connections — the Azure SDK handles token caching
+    and silent renewal internally. Only ``get_token`` is called per connection.
+    """
+    credential = _build_azure_credential(
+        azure_client_id=azure_client_id,
+        azure_tenant_id=azure_tenant_id,
+        azure_client_secret=azure_client_secret,
+    )
+
+    def ad_connect(self):
+        """Initialize the connection and authenticate using Azure AD"""
+        from redis.exceptions import (
+            AuthenticationError,
+            AuthenticationWrongNumberOfArgsError,
+        )
+        from redis.utils import str_if_bytes
+
+        self._parser.on_connect(self)
+
+        access_token = credential.get_token(AZURE_REDIS_SCOPE).token
+
+        # Only include username when explicitly set — sending AUTH "" <token>
+        # is invalid for most ACL-configured Azure Redis instances.
+        username = os.environ.get("REDIS_USERNAME", "")
+        if username:
+            auth_args = (username, access_token)
+        else:
+            auth_args = (access_token,)
+
+        self.send_command("AUTH", *auth_args, check_health=False)
+
+        try:
+            auth_response = self.read_response()
+        except AuthenticationWrongNumberOfArgsError:
+            # Fallback: try with just the token (Redis < 6 / no ACL)
+            self.send_command("AUTH", access_token, check_health=False)
+            auth_response = self.read_response()
+
+        if str_if_bytes(auth_response) != "OK":
+            raise AuthenticationError("Azure AD authentication failed for Redis")
+
+    # Attach the live credential object so async paths can wrap it in
+    # AzureADCredentialProvider for refresh-aware token retrieval. The raw
+    # client_id/tenant_id/secret are intentionally NOT exposed here — the
+    # credential closure already holds them.
+    ad_connect._azure_credential = credential  # type: ignore[attr-defined]
+    return ad_connect
+
+
 def get_redis_url_from_environment():
     if "REDIS_URL" in os.environ:
         return os.environ["REDIS_URL"]
@@ -179,7 +309,7 @@ def get_redis_url_from_environment():
     return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
 
 
-def _get_redis_client_logic(**env_overrides):
+def _get_redis_client_logic(**env_overrides):  # noqa: PLR0915
     """
     Common functionality across sync + async redis client implementations
     """
@@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides):
         if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
             redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
 
+    # Handle Azure AD authentication (after GCP IAM block)
+    _azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret(
+        "REDIS_AZURE_AD_TOKEN"
+    )
+
+    _azure_ad_enabled = (
+        _azure_redis_ad_token is not None
+        and str(_azure_redis_ad_token).lower() == "true"
+    )
+
+    if _azure_ad_enabled and _gcp_service_account is not None:
+        verbose_logger.warning(
+            "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
+            "Using GCP IAM. Remove one to avoid misconfiguration."
+        )
+
+    if _azure_ad_enabled and _gcp_service_account is None:
+        _azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str(
+            "AZURE_CLIENT_ID"
+        )
+        _azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str(
+            "AZURE_TENANT_ID"
+        )
+        _azure_client_secret = redis_kwargs.get(
+            "azure_client_secret"
+        ) or get_secret_str("AZURE_CLIENT_SECRET")
+
+        verbose_logger.debug("Setting up Azure AD authentication for Redis.")
+        redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
+            azure_client_id=_azure_client_id,
+            azure_tenant_id=_azure_tenant_id,
+            azure_client_secret=_azure_client_secret,
+        )
+        # Marker for async paths to detect Azure AD auth. The live credential
+        # object is attached separately as `_azure_credential` by
+        # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
+        # are intentionally NOT exposed on the function to avoid leaking
+        # credentials via inspection or logging.
+        redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True  # type: ignore[attr-defined]
+
+    # Always remove Azure-specific kwargs that shouldn't be passed to Redis client
+    redis_kwargs.pop("azure_redis_ad_token", None)
+    redis_kwargs.pop("azure_client_id", None)
+    redis_kwargs.pop("azure_tenant_id", None)
+    redis_kwargs.pop("azure_client_secret", None)
+
     if "url" in redis_kwargs and redis_kwargs["url"] is not None:
         # Only strip host/port/db/password when not routing to a cluster.
         # When startup_nodes is also present the cluster path takes priority and
@@ -303,10 +479,24 @@ def init_redis_cluster(redis_kwargs) -> redis.RedisCluster:
     return redis.RedisCluster(startup_nodes=new_startup_nodes, **cluster_kwargs)  # type: ignore
 
 
+def _get_redis_sentinel_connection_kwargs(redis_kwargs: dict) -> dict:
+    connection_kwargs = {}
+    args = _get_redis_kwargs()
+    for arg in redis_kwargs:
+        if arg in args:
+            connection_kwargs[arg] = redis_kwargs[arg]
+
+    return connection_kwargs
+
+
 def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     sentinel_nodes = redis_kwargs.get("sentinel_nodes")
     sentinel_password = redis_kwargs.get("sentinel_password")
     service_name = redis_kwargs.get("service_name")
+    connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
+    connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
+    sentinel_kwargs = dict(connection_kwargs)
+    sentinel_kwargs["password"] = sentinel_password
 
     if not sentinel_nodes or not service_name:
         raise ValueError(
@@ -318,19 +508,22 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     # Set up the Sentinel client
     sentinel = redis.Sentinel(
         sentinel_nodes,
-        socket_timeout=REDIS_SOCKET_TIMEOUT,
-        password=sentinel_password,
+        sentinel_kwargs=sentinel_kwargs,
     )
 
     # Return the master instance for the given service
 
-    return sentinel.master_for(service_name)
+    return sentinel.master_for(service_name, **connection_kwargs)
 
 
 def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     sentinel_nodes = redis_kwargs.get("sentinel_nodes")
     sentinel_password = redis_kwargs.get("sentinel_password")
     service_name = redis_kwargs.get("service_name")
+    connection_kwargs = _get_redis_sentinel_connection_kwargs(redis_kwargs)
+    connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
+    sentinel_kwargs = dict(connection_kwargs)
+    sentinel_kwargs["password"] = sentinel_password
 
     if not sentinel_nodes or not service_name:
         raise ValueError(
@@ -342,13 +535,12 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     # Set up the Sentinel client
     sentinel = async_redis.Sentinel(
         sentinel_nodes,
-        socket_timeout=REDIS_SOCKET_TIMEOUT,
-        password=sentinel_password,
+        sentinel_kwargs=sentinel_kwargs,
     )
 
     # Return the master instance for the given service
 
-    return sentinel.master_for(service_name)
+    return sentinel.master_for(service_name, **connection_kwargs)
 
 
 def get_redis_client(**env_overrides):
@@ -373,7 +565,7 @@ def get_redis_client(**env_overrides):
     return redis.Redis(**redis_kwargs)
 
 
-def get_redis_async_client(
+def get_redis_async_client(  # noqa: PLR0915
     connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
     **env_overrides,
 ) -> Union[async_redis.Redis, async_redis.RedisCluster]:
@@ -398,6 +590,14 @@ def get_redis_async_client(
             cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
                 redis_connect_func._gcp_service_account
             )
+        # Handle Azure AD authentication for async clusters via CredentialProvider
+        # so the credential's internal cache + silent refresh runs per connection
+        # (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry).
+        elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+            cluster_kwargs["credential_provider"] = AzureADCredentialProvider(
+                redis_connect_func._azure_credential,
+                username=os.environ.get("REDIS_USERNAME") or None,
+            )
 
         new_startup_nodes: List[ClusterNode] = []
 
@@ -431,6 +631,22 @@ def get_redis_async_client(
     # Check for Redis Sentinel
     if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
         return _init_async_redis_sentinel(redis_kwargs)
+
+    # Wrap GCP / Azure AD auth in a CredentialProvider for the standard async
+    # Redis client. The async client doesn't support redis_connect_func, but it
+    # does honour credential_provider — which is called per connection, so the
+    # underlying SDK can refresh tokens silently before they expire.
+    redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
+    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
+            redis_connect_func._azure_credential,
+            username=os.environ.get("REDIS_USERNAME") or None,
+        )
+    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
+        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
+            redis_connect_func._gcp_service_account
+        )
+
     _pretty_print_redis_config(redis_kwargs=redis_kwargs)
 
     if connection_pool is not None:
@@ -464,6 +680,21 @@ def get_redis_connection_pool(
                     redis_kwargs["max_connections"],
                 )
         return async_redis.BlockingConnectionPool.from_url(**pool_kwargs)
+
+    # Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed
+    # connections re-fetch tokens via the SDK's internal cache + silent refresh
+    # rather than reusing a single token captured at pool creation.
+    redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
+    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
+            redis_connect_func._azure_credential,
+            username=os.environ.get("REDIS_USERNAME") or None,
+        )
+    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
+        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
+            redis_connect_func._gcp_service_account
+        )
+
     connection_class = async_redis.Connection
     if "ssl" in redis_kwargs:
         connection_class = async_redis.SSLConnection

litellm/_redis_credential_provider.py

@@ -1,10 +1,13 @@
 import asyncio
 import threading
 import time
-from typing import Dict, Tuple
+from typing import Any, Dict, Optional, Tuple, Union
 
 from redis.credentials import CredentialProvider  # type: ignore[attr-defined]
 
+# Azure AD scope for Redis Cache for Azure.
+AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
+
 # GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry.
 _GCP_IAM_TOKEN_TTL_SECONDS = 3300
 
@@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider):
             _get_cached_gcp_iam_token, self._gcp_service_account
         )
         return (token,)
+
+
+class AzureADCredentialProvider(CredentialProvider):
+    """
+    redis.credentials.CredentialProvider implementation that supplies Azure AD
+    tokens for Redis authentication.
+
+    Wraps an azure-identity credential object so the Azure SDK's internal token
+    cache and silent refresh are honoured on every Redis connection. This avoids
+    the static-token-baked-in-pool issue where pool-managed connections would
+    fail authentication after the initial token expired (~1 hour TTL).
+    """
+
+    def __init__(self, credential: Any, username: Optional[str] = None) -> None:
+        self._credential = credential
+        self._username = username
+
+    def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]:
+        token = self._credential.get_token(AZURE_REDIS_SCOPE).token
+        if self._username:
+            return (self._username, token)
+        return (token,)
+
+    async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]:
+        token_obj = await asyncio.to_thread(
+            self._credential.get_token, AZURE_REDIS_SCOPE
+        )
+        if self._username:
+            return (self._username, token_obj.token)
+        return (token_obj.token,)

litellm/anthropic_beta_headers_config.json

@@ -72,7 +72,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": null,
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
     "fine-grained-tool-streaming-2025-05-14": null,
@@ -103,7 +103,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": null,
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
     "fine-grained-tool-streaming-2025-05-14": null,

litellm/batches/batch_utils.py

@@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
     )
 
 
+def _get_models_from_batch_input_file_content(
+    file_content_dictionary: List[dict],
+) -> List[str]:
+    """Extract the distinct ``body.model`` values from a batch *input* file.
+
+    Used by the proxy's batch pre-call hook to enforce that the caller is
+    authorized for every model named inside the JSONL — not just the one
+    on the outer request — so the proxy's per-key model allowlist isn't
+    bypassed by smuggling expensive models into the batch file.
+    """
+    models: List[str] = []
+    seen: set = set()
+    for _item in file_content_dictionary:
+        body = _item.get("body") or {}
+        model = body.get("model")
+        if model and model not in seen:
+            seen.add(model)
+            models.append(model)
+    return models
+
+
 def _get_batch_job_input_file_usage(
     file_content_dictionary: List[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@@ -403,11 +424,25 @@ def _get_batch_job_input_file_usage(
     for _item in file_content_dictionary:
         body = _item.get("body", {})
         model = body.get("model", model_name or "")
-        messages = body.get("messages", [])
 
+        # Chat completion payloads.
+        messages = body.get("messages")
         if messages:
-            item_tokens = token_counter(model=model, messages=messages)
-            prompt_tokens += item_tokens
+            prompt_tokens += token_counter(model=model, messages=messages)
+            continue
+
+        # Text completion payloads (`prompt`).
+        prompt = body.get("prompt")
+        if prompt:
+            prompt_tokens += _count_prompt_or_input_tokens(model=model, value=prompt)
+            continue
+
+        # Embedding payloads (`input`).
+        input_data = body.get("input")
+        if input_data:
+            prompt_tokens += _count_prompt_or_input_tokens(
+                model=model, value=input_data
+            )
 
     return Usage(
         total_tokens=prompt_tokens + completion_tokens,
@@ -416,6 +451,43 @@ def _get_batch_job_input_file_usage(
     )
 
 
+def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
+    """Token-count a ``prompt`` / ``input`` field that the OpenAI batch
+    schema allows in four shapes:
+
+    - ``str``: a single text prompt.
+    - ``list[str]``: multiple text prompts.
+    - ``list[int]``: a pre-tokenized prompt (each int counts as 1 token).
+    - ``list[list[int]]``: multiple pre-tokenized prompts.
+
+    Pre-fix only the string shapes were counted, so a caller could send
+    a large ``list[list[int]]`` payload and slip past TPM rate limits
+    with a recorded cost of zero tokens.
+    """
+    if isinstance(value, str):
+        return token_counter(model=model, text=value)
+    if isinstance(value, list):
+        total = 0
+        for chunk in value:
+            if isinstance(chunk, str):
+                total += token_counter(model=model, text=chunk)
+            elif isinstance(chunk, int):
+                # Single pre-tokenized prompt at the top level: each
+                # int counts as one token.
+                total += 1
+            elif isinstance(chunk, list):
+                # Nested pre-tokenized prompt: every int contributes a
+                # token. Mixed string/int items still count.
+                total += sum(1 if isinstance(t, int) else 0 for t in chunk)
+                total += sum(
+                    token_counter(model=model, text=t)
+                    for t in chunk
+                    if isinstance(t, str)
+                )
+        return total
+    return 0
+
+
 def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
     """
     Get the tokens of a batch job from the response body

litellm/batches/main.py

@@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config(
         )
     else:
         raise litellm.exceptions.BadRequestError(
-            message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
-                custom_llm_provider
-            ),
+            message=(
+                "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. "
+                "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
+                "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
+            ).format(custom_llm_provider),
             model="n/a",
             llm_provider=custom_llm_provider,
             response=httpx.Response(
                 status_code=400,
                 content="Unsupported provider",
-                request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+                request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),  # type: ignore
             ),
         )
     return response
@@ -615,24 +617,35 @@ def retrieve_batch(
         _is_async = kwargs.pop("aretrieve_batch", False) is True
         client = kwargs.get("client", None)
 
-        # Check if this is an async invoke ARN (different from regular batch ARN)
-        # Async invoke ARNs have format: arn:aws(-[^:]+)?:bedrock:[a-z0-9-]{1,20}:[0-9]{12}:async-invoke/[a-z0-9]{12}
-        if (
-            batch_id.startswith("arn:aws")
-            and ":bedrock:" in batch_id
-            and ":async-invoke/" in batch_id
-        ):
-            # Handle async invoke status check
-            # Remove aws_region_name from kwargs to avoid duplicate parameter
-            async_kwargs = kwargs.copy()
-            async_kwargs.pop("aws_region_name", None)
+        # Bedrock has two distinct ARN families that need different APIs:
+        #   * async-invoke ARNs       (Twelve Labs Marengo embeddings)        -> bedrock-runtime data plane
+        #   * model-invocation-job ARNs (CreateModelInvocationJob batch)      -> bedrock control plane
+        # They live on different AWS service endpoints and can't share a handler.
+        # ARN shapes:
+        #   arn:aws(-[^:]+)?:bedrock:<region>:<account>:async-invoke/<id>
+        #   arn:aws(-[^:]+)?:bedrock:<region>:<account>:model-invocation-job/<id>
+        if batch_id.startswith("arn:aws") and ":bedrock:" in batch_id:
+            if ":async-invoke/" in batch_id:
+                # Remove aws_region_name from kwargs to avoid duplicate parameter
+                async_kwargs = kwargs.copy()
+                async_kwargs.pop("aws_region_name", None)
 
-            return BedrockBatchesHandler._handle_async_invoke_status(
-                batch_id=batch_id,
-                aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
-                logging_obj=litellm_logging_obj,
-                **async_kwargs,
-            )
+                return BedrockBatchesHandler._handle_async_invoke_status(
+                    batch_id=batch_id,
+                    aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
+                    logging_obj=litellm_logging_obj,
+                    **async_kwargs,
+                )
+            if ":model-invocation-job/" in batch_id:
+                mij_kwargs = kwargs.copy()
+                mij_kwargs.pop("aws_region_name", None)
+
+                return BedrockBatchesHandler._handle_model_invocation_job_status(
+                    batch_id=batch_id,
+                    aws_region_name=kwargs.get("aws_region_name"),
+                    logging_obj=litellm_logging_obj,
+                    **mij_kwargs,
+                )
 
         # Try to use provider config first (for providers like bedrock)
         model: Optional[str] = kwargs.get("model", None)

litellm/budget_manager.py

@@ -178,6 +178,18 @@ class BudgetManager:
         return list(self.user_dict.keys())
 
     def reset_cost(self, user):
+        """
+        Reset the tracked spend for a user back to zero.
+
+        Clears both the aggregate ``current_cost`` and the per-model
+        ``model_cost`` breakdown stored for the given user.
+
+        Args:
+            user: The user identifier whose cost should be reset.
+
+        Returns:
+            dict: ``{"user": <updated user record>}`` reflecting the reset state.
+        """
         self.user_dict[user]["current_cost"] = 0
         self.user_dict[user]["model_cost"] = {}
         return {"user": self.user_dict[user]}

litellm/caching/caching.py

@@ -432,9 +432,10 @@ class Cache:
             str: The final hashed cache key with the redis namespace.
         """
         dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
+        metadata = kwargs.get("metadata") or {}
         namespace = (
             dynamic_cache_control.get("namespace")
-            or kwargs.get("metadata", {}).get("redis_namespace")
+            or metadata.get("redis_namespace")
             or self.namespace
         )
         if namespace:

litellm/caching/caching_handler.py

@@ -87,6 +87,18 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
+    """
+    When stream=True, do not run success callbacks at cache-hit time.
+
+    Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
+    replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
+    handlers when the stream finishes; firing them here too would double-count
+    spend and callback records.
+    """
+    return kwargs.get("stream", False) is True
+
+
 class LLMCachingHandler:
     def __init__(
         self,
@@ -99,6 +111,7 @@ class LLMCachingHandler:
         self.async_streaming_chunks: List[ModelResponse] = []
         self.sync_streaming_chunks: List[ModelResponse] = []
         self.request_kwargs = request_kwargs
+        self.preset_cache_key: Optional[str] = None
         self.original_function = original_function
         self.start_time = start_time
         if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
@@ -206,7 +219,7 @@ class LLMCachingHandler:
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         args=args,
                     )
-                    if kwargs.get("stream", False) is False:
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
                         # LOG SUCCESS
                         self._async_log_cache_hit_on_callbacks(
                             logging_obj=logging_obj,
@@ -215,11 +228,12 @@ class LLMCachingHandler:
                             end_time=end_time,
                             cache_hit=cache_hit,
                         )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
+                    )
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
                 elif (
@@ -265,8 +279,6 @@ class LLMCachingHandler:
         kwargs: Dict[str, Any],
         args: Optional[Tuple[Any, ...]] = None,
     ) -> CachingHandlerResponse:
-        from litellm.utils import CustomStreamWrapper
-
         cached_result: Optional[Any] = None
 
         # Check if caching should be performed BEFORE doing expensive kwargs copy
@@ -282,6 +294,11 @@ class LLMCachingHandler:
                     args,
                 )
             )
+            if new_kwargs.get("metadata") is None:
+                new_kwargs.pop("metadata", None)
+            if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+                new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+            self.request_kwargs = new_kwargs
             print_verbose("Checking Sync Cache")
             cached_result = litellm.cache.get_cache(**new_kwargs)
             if cached_result is not None:
@@ -322,17 +339,19 @@ class LLMCachingHandler:
                         is_async=False,
                     )
 
-                    logging_obj.handle_sync_success_callbacks_for_async_calls(
-                        result=cached_result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        cache_hit=cache_hit,
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
+                        logging_obj.handle_sync_success_callbacks_for_async_calls(
+                            result=cached_result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            cache_hit=cache_hit,
+                        )
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
         return CachingHandlerResponse(cached_result=cached_result)
@@ -686,6 +705,11 @@ class LLMCachingHandler:
                 args,
             )
         )
+        if new_kwargs.get("metadata") is None:
+            new_kwargs.pop("metadata", None)
+        if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+            new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+        self.request_kwargs = new_kwargs
         cached_result: Optional[Any] = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):
@@ -710,14 +734,26 @@ class LLMCachingHandler:
                 if all(result is None for result in cached_result):
                     cached_result = None
         else:
+            request_kwargs = new_kwargs.copy()
+            request_cache_key = request_kwargs.pop("cache_key", None)
             if litellm.cache._supports_async() is True:
                 ## check if dual cache is supported ##
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = await litellm.cache.async_get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
             else:  # fallback for caches that don't support async
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = litellm.cache.get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
         return cached_result
 
@@ -825,8 +861,27 @@ class LLMCachingHandler:
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(
             cached_result, dict
         ):
-            # Convert cached dict back to ResponsesAPIResponse object
-            cached_result = ResponsesAPIResponse(**cached_result)
+            from litellm.responses.streaming_iterator import (
+                CachedResponsesAPIStreamingIterator,
+            )
+
+            response_obj = ResponsesAPIResponse(**cached_result)
+            if (
+                hasattr(response_obj, "_hidden_params")
+                and response_obj._hidden_params is not None
+                and isinstance(response_obj._hidden_params, dict)
+            ):
+                response_obj._hidden_params["cache_hit"] = True
+
+            if kwargs.get("stream", False) is True:
+                cached_result = CachedResponsesAPIStreamingIterator(
+                    response=response_obj,
+                    logging_obj=logging_obj,
+                    request_data=kwargs,
+                    call_type=call_type,
+                )
+            else:
+                cached_result = response_obj
 
         if (
             hasattr(cached_result, "_hidden_params")

litellm/caching/dual_cache.py

@@ -92,6 +92,25 @@ class DualCache(BaseCache):
         if default_redis_ttl is not None:
             self.default_redis_ttl = default_redis_ttl
 
+    def attach_redis_cache(
+        self,
+        redis_cache: Optional[RedisCache] = None,
+        *,
+        default_redis_ttl: Optional[float] = None,
+    ) -> None:
+        """
+        Attach a Redis backend if this DualCache does not already have one.
+
+        No-op when ``redis_cache`` is None or when Redis was already set (constructor
+        or a prior attach). Use this for lazy wiring after a shared Redis client exists.
+        Does not backfill in-memory-only keys to Redis.
+        """
+        if redis_cache is None or self.redis_cache is not None:
+            return
+        self.redis_cache = redis_cache
+        if default_redis_ttl is not None:
+            self.default_redis_ttl = default_redis_ttl
+
     def set_cache(self, key, value, local_only: bool = False, **kwargs):
         # Update both Redis and in-memory cache
         try:
@@ -392,6 +411,7 @@ class DualCache(BaseCache):
         value: float,
         parent_otel_span: Optional[Span] = None,
         local_only: bool = False,
+        refresh_ttl: bool = False,
         **kwargs,
     ) -> Optional[float]:
         """
@@ -399,6 +419,9 @@ class DualCache(BaseCache):
 
         Value - float - the value you want to increment by
 
+        Refresh_ttl - bool - if True, resets the Redis TTL on every write.
+        Default False preserves window-style semantics.
+
         Returns - the incremented value, or None if no cache backend is
         available (in_memory_cache is None and Redis failed/is absent).
         """
@@ -415,6 +438,7 @@ class DualCache(BaseCache):
                     value,
                     parent_otel_span=parent_otel_span,
                     ttl=kwargs.get("ttl", None),
+                    refresh_ttl=refresh_ttl,
                 )
 
             return result

litellm/caching/qdrant_semantic_cache.py

@@ -11,17 +11,23 @@ Has 4 methods:
 import ast
 import asyncio
 import json
-from typing import Any, cast
+import os
+from typing import Any, Dict, cast
 
 import litellm
 from litellm._logging import print_verbose
 from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    get_str_from_messages,
+)
 from litellm.types.utils import EmbeddingResponse
 
 from .base_cache import BaseCache
 
 
 class QdrantSemanticCache(BaseCache):
+    CACHE_KEY_FIELD_NAME = "litellm_cache_key"
+
     def __init__(  # noqa: PLR0915
         self,
         qdrant_api_base=None,
@@ -33,8 +39,6 @@ class QdrantSemanticCache(BaseCache):
         host_type=None,
         vector_size=None,
     ):
-        import os
-
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
             get_async_httpx_client,
@@ -115,7 +119,9 @@ class QdrantSemanticCache(BaseCache):
             print_verbose(
                 f"Collection already exists.\nCollection details:{self.collection_info}"
             )
+            self._ensure_cache_key_payload_index()
         else:
+            quantization_params: Dict[str, Any]
             if quantization_config is None or quantization_config == "binary":
                 quantization_params = {
                     "binary": {
@@ -156,6 +162,7 @@ class QdrantSemanticCache(BaseCache):
                 print_verbose(
                     f"New collection created.\nCollection details:{self.collection_info}"
                 )
+                self._ensure_cache_key_payload_index()
             else:
                 raise Exception("Error while creating new collection")
 
@@ -170,15 +177,94 @@ class QdrantSemanticCache(BaseCache):
             cached_response = ast.literal_eval(cached_response)
         return cached_response
 
+    def _get_qdrant_cache_key_filter(self, key: str) -> dict:
+        return {
+            "must": [
+                {
+                    "key": self.CACHE_KEY_FIELD_NAME,
+                    "match": {"value": str(key)},
+                }
+            ]
+        }
+
+    def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None:
+        data["filter"] = self._get_qdrant_cache_key_filter(key)
+
+    def _ensure_cache_key_payload_index(self) -> None:
+        try:
+            response = self.sync_client.put(
+                url=f"{self.qdrant_api_base}/collections/{self.collection_name}/index",
+                headers=self.headers,
+                json={
+                    "field_name": self.CACHE_KEY_FIELD_NAME,
+                    "field_schema": "keyword",
+                },
+            )
+            if response.status_code not in (200, 201):
+                print_verbose(
+                    "Qdrant semantic-cache could not create cache-key payload index: "
+                    f"{response.text}"
+                )
+        except Exception as exc:
+            print_verbose(
+                "Qdrant semantic-cache could not create cache-key payload index: "
+                f"{str(exc)}"
+            )
+
+    def _payload_matches_cache_key(self, payload: dict, key: str) -> bool:
+        # Pre-isolation points stored only prompt + response with no cache-key
+        # payload field. Reassigning them to a caller's key would risk
+        # cross-scope hits, so they're treated as misses and re-populated on
+        # the next set_cache.
+        cached_key = payload.get(self.CACHE_KEY_FIELD_NAME)
+        return cached_key is not None and str(cached_key) == str(key)
+
+    async def _get_async_embedding(self, prompt: str, **kwargs) -> Any:
+        llm_model_list = None
+        llm_router = None
+
+        try:
+            from litellm.proxy.proxy_server import (
+                llm_model_list as proxy_llm_model_list,
+                llm_router as proxy_llm_router,
+            )
+
+            llm_model_list = proxy_llm_model_list
+            llm_router = proxy_llm_router
+        except ImportError:
+            pass
+
+        router_model_names = (
+            [m["model_name"] for m in llm_model_list]
+            if llm_model_list is not None
+            else []
+        )
+        if llm_router is not None and self.embedding_model in router_model_names:
+            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
+            return await llm_router.aembedding(
+                model=self.embedding_model,
+                input=prompt,
+                cache={"no-store": True, "no-cache": True},
+                metadata={
+                    "user_api_key": user_api_key,
+                    "semantic-cache-embedding": True,
+                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
+                },
+            )
+
+        return await litellm.aembedding(
+            model=self.embedding_model,
+            input=prompt,
+            cache={"no-store": True, "no-cache": True},
+        )
+
     def set_cache(self, key, value, **kwargs):
         print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")
         from litellm._uuid import uuid
 
         # get the prompt
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
         # create an embedding for prompt
         embedding_response = cast(
@@ -202,6 +288,7 @@ class QdrantSemanticCache(BaseCache):
                     "id": str(uuid.uuid4()),
                     "vector": embedding,
                     "payload": {
+                        self.CACHE_KEY_FIELD_NAME: str(key),
                         "text": prompt,
                         "response": value,
                     },
@@ -220,9 +307,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the messages
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
         # convert to embedding
         embedding_response = cast(
@@ -249,6 +334,7 @@ class QdrantSemanticCache(BaseCache):
             "limit": 1,
             "with_payload": True,
         }
+        self._add_cache_key_filter_to_search_data(data=data, key=key)
 
         search_response = self.sync_client.post(
             url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@@ -258,21 +344,33 @@ class QdrantSemanticCache(BaseCache):
         results = search_response.json()["result"]
 
         if results is None:
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
             return None
         if isinstance(results, list):
             if len(results) == 0:
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
         similarity = results[0]["score"]
-        cached_prompt = results[0]["payload"]["text"]
+        payload = results[0]["payload"]
+        if not self._payload_matches_cache_key(payload=payload, key=key):
+            print_verbose("Qdrant semantic-cache hit did not match cache key scope")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+            return None
+
+        cached_prompt = payload["text"]
 
         # check similarity, if more than self.similarity_threshold, return results
         print_verbose(
             f"semantic cache: similarity threshold: {self.similarity_threshold}, similarity: {similarity}, prompt: {prompt}, closest_cached_prompt: {cached_prompt}"
         )
+
+        # update kwargs["metadata"] with similarity, don't rewrite the original metadata
+        kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
+
         if similarity >= self.similarity_threshold:
             # cache hit !
-            cached_value = results[0]["payload"]["response"]
+            cached_value = payload["response"]
             print_verbose(
                 f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
             )
@@ -285,40 +383,12 @@ class QdrantSemanticCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         from litellm._uuid import uuid
 
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
-
         print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}")
 
         # get the prompt
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
-        # create an embedding for prompt
-        router_model_names = (
-            [m["model_name"] for m in llm_model_list]
-            if llm_model_list is not None
-            else []
-        )
-        if llm_router is not None and self.embedding_model in router_model_names:
-            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
-            embedding_response = await llm_router.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-                metadata={
-                    "user_api_key": user_api_key,
-                    "semantic-cache-embedding": True,
-                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
-                },
-            )
-        else:
-            # convert to embedding
-            embedding_response = await litellm.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-            )
+        prompt = get_str_from_messages(messages)
+        embedding_response = await self._get_async_embedding(prompt, **kwargs)
 
         # get the embedding
         embedding = embedding_response["data"][0]["embedding"]
@@ -332,6 +402,7 @@ class QdrantSemanticCache(BaseCache):
                     "id": str(uuid.uuid4()),
                     "vector": embedding,
                     "payload": {
+                        self.CACHE_KEY_FIELD_NAME: str(key),
                         "text": prompt,
                         "response": value,
                     },
@@ -348,38 +419,12 @@ class QdrantSemanticCache(BaseCache):
 
     async def async_get_cache(self, key, **kwargs):
         print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}")
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
 
         # get the messages
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
-        router_model_names = (
-            [m["model_name"] for m in llm_model_list]
-            if llm_model_list is not None
-            else []
-        )
-        if llm_router is not None and self.embedding_model in router_model_names:
-            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
-            embedding_response = await llm_router.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-                metadata={
-                    "user_api_key": user_api_key,
-                    "semantic-cache-embedding": True,
-                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
-                },
-            )
-        else:
-            # convert to embedding
-            embedding_response = await litellm.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-            )
+        embedding_response = await self._get_async_embedding(prompt, **kwargs)
 
         # get the embedding
         embedding = embedding_response["data"][0]["embedding"]
@@ -396,6 +441,7 @@ class QdrantSemanticCache(BaseCache):
             "limit": 1,
             "with_payload": True,
         }
+        self._add_cache_key_filter_to_search_data(data=data, key=key)
 
         search_response = await self.async_client.post(
             url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@@ -414,7 +460,13 @@ class QdrantSemanticCache(BaseCache):
                 return None
 
         similarity = results[0]["score"]
-        cached_prompt = results[0]["payload"]["text"]
+        payload = results[0]["payload"]
+        if not self._payload_matches_cache_key(payload=payload, key=key):
+            print_verbose("Qdrant semantic-cache hit did not match cache key scope")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+            return None
+
+        cached_prompt = payload["text"]
 
         # check similarity, if more than self.similarity_threshold, return results
         print_verbose(
@@ -426,7 +478,7 @@ class QdrantSemanticCache(BaseCache):
 
         if similarity >= self.similarity_threshold:
             # cache hit !
-            cached_value = results[0]["payload"]["response"]
+            cached_value = payload["response"]
             print_verbose(
                 f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
             )

litellm/caching/redis_cache.py

@@ -551,6 +551,13 @@ class RedisCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         from redis.asyncio import Redis
 
+        if key is None:
+            verbose_logger.debug(
+                "LiteLLM Redis Caching: async set() skipped — key is None, value=%r",
+                value,
+            )
+            return None
+
         start_time = time.time()
         try:
             _redis_client: Redis = self.init_async_client()  # type: ignore
@@ -569,8 +576,9 @@ class RedisCache(BaseCache):
                 )
             )
             verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
+                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
                 str(e),
+                key,
                 value,
             )
             raise e
@@ -824,6 +832,7 @@ class RedisCache(BaseCache):
         value: float,
         ttl: Optional[int] = None,
         parent_otel_span: Optional[Span] = None,
+        refresh_ttl: bool = False,
     ) -> float:
         from redis.asyncio import Redis
 
@@ -834,11 +843,12 @@ class RedisCache(BaseCache):
         try:
             result = await _redis_client.incrbyfloat(name=key, amount=value)
             if _used_ttl is not None:
-                # check if key already has ttl, if not -> set ttl
-                current_ttl = await _redis_client.ttl(key)
-                if current_ttl == -1:
-                    # Key has no expiration
+                if refresh_ttl:
                     await _redis_client.expire(key, _used_ttl)
+                else:
+                    current_ttl = await _redis_client.ttl(key)
+                    if current_ttl == -1:
+                        await _redis_client.expire(key, _used_ttl)
 
             ## LOGGING ##
             end_time = time.time()

litellm/caching/redis_semantic_cache.py

@@ -35,6 +35,7 @@ class RedisSemanticCache(BaseCache):
     """
 
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
+    CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
 
     def __init__(
         self,
@@ -66,8 +67,8 @@ class RedisSemanticCache(BaseCache):
             Exception: If similarity_threshold is not provided or required Redis
                 connection information is missing
         """
-        from redisvl.extensions.llmcache import SemanticCache
-        from redisvl.utils.vectorize import CustomTextVectorizer
+        from redisvl.extensions.llmcache import SemanticCache  # type: ignore[import-not-found, import-untyped]
+        from redisvl.utils.vectorize import CustomTextVectorizer  # type: ignore[import-not-found, import-untyped]
 
         if index_name is None:
             index_name = self.DEFAULT_REDIS_INDEX_NAME
@@ -109,14 +110,94 @@ class RedisSemanticCache(BaseCache):
         # Initialize the Redis vectorizer and cache
         cache_vectorizer = CustomTextVectorizer(self._get_embedding)
 
-        self.llmcache = SemanticCache(
-            name=index_name,
+        self.llmcache = self._init_semantic_cache(
+            semantic_cache_cls=SemanticCache,
+            index_name=index_name,
             redis_url=redis_url,
-            vectorizer=cache_vectorizer,
-            distance_threshold=self.distance_threshold,
-            overwrite=False,
+            cache_vectorizer=cache_vectorizer,
         )
 
+    @classmethod
+    def _cache_key_filterable_field(cls) -> Dict[str, str]:
+        return {
+            "name": cls.CACHE_KEY_FIELD_NAME,
+            "type": "tag",
+        }
+
+    def _init_semantic_cache(
+        self,
+        semantic_cache_cls: Any,
+        index_name: str,
+        redis_url: str,
+        cache_vectorizer: Any,
+    ) -> Any:
+        def _is_schema_mismatch(exc: ValueError) -> bool:
+            error_message = str(exc).lower()
+            return any(
+                phrase in error_message
+                for phrase in ("schema does not match", "index schema")
+            )
+
+        try:
+            return semantic_cache_cls(
+                name=index_name,
+                redis_url=redis_url,
+                vectorizer=cache_vectorizer,
+                distance_threshold=self.distance_threshold,
+                filterable_fields=[self._cache_key_filterable_field()],
+                overwrite=False,
+            )
+        except ValueError as exc:
+            if not _is_schema_mismatch(exc):
+                raise
+
+            isolated_index_name = f"{index_name}_isolated"
+            print_verbose(
+                "Redis semantic-cache existing index schema is not isolated; "
+                f"using isolated index - {isolated_index_name}"
+            )
+            try:
+                return semantic_cache_cls(
+                    name=isolated_index_name,
+                    redis_url=redis_url,
+                    vectorizer=cache_vectorizer,
+                    distance_threshold=self.distance_threshold,
+                    filterable_fields=[self._cache_key_filterable_field()],
+                    overwrite=False,
+                )
+            except ValueError as isolated_exc:
+                if not _is_schema_mismatch(isolated_exc):
+                    raise
+
+                print_verbose(
+                    "Redis semantic-cache isolated index schema is stale; "
+                    f"recreating isolated index - {isolated_index_name}"
+                )
+                return semantic_cache_cls(
+                    name=isolated_index_name,
+                    redis_url=redis_url,
+                    vectorizer=cache_vectorizer,
+                    distance_threshold=self.distance_threshold,
+                    filterable_fields=[self._cache_key_filterable_field()],
+                    overwrite=True,
+                )
+
+    def _get_cache_filters(self, key: str) -> Dict[str, str]:
+        return {self.CACHE_KEY_FIELD_NAME: str(key)}
+
+    def _get_cache_key_filter_expression(self, key: str) -> Any:
+        from redisvl.query.filter import Tag  # type: ignore[import-not-found, import-untyped]
+
+        return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)
+
+    def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool:
+        # Pre-isolation entries with no ``litellm_cache_key`` field cannot be
+        # safely reassigned to a caller's scope and are treated as misses.
+        cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME)
+        if isinstance(cached_key, bytes):
+            cached_key = cached_key.decode("utf-8")
+        return cached_key is not None and str(cached_key) == str(key)
+
     def _get_ttl(self, **kwargs) -> Optional[int]:
         """
         Get the TTL (time-to-live) value for cache entries.
@@ -188,7 +269,7 @@ class RedisSemanticCache(BaseCache):
         Store a value in the semantic cache.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             value: The response value to cache
             **kwargs: Additional arguments including 'messages' for the prompt
                 and optional 'ttl' for time-to-live
@@ -206,12 +287,15 @@ class RedisSemanticCache(BaseCache):
             prompt = get_str_from_messages(messages)
             value_str = str(value)
 
+            store_kwargs: Dict[str, Any] = {
+                "filters": self._get_cache_filters(key),
+            }
+
             # Get TTL and store in Redis semantic cache
             ttl = self._get_ttl(**kwargs)
             if ttl is not None:
-                self.llmcache.store(prompt, value_str, ttl=int(ttl))
-            else:
-                self.llmcache.store(prompt, value_str)
+                store_kwargs["ttl"] = int(ttl)
+            self.llmcache.store(prompt, value_str, **store_kwargs)
         except Exception as e:
             print_verbose(
                 f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}"
@@ -222,7 +306,7 @@ class RedisSemanticCache(BaseCache):
         Retrieve a semantically similar cached response.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             **kwargs: Additional arguments including 'messages' for the prompt
 
         Returns:
@@ -235,18 +319,29 @@ class RedisSemanticCache(BaseCache):
             messages = kwargs.get("messages", [])
             if not messages:
                 print_verbose("No messages provided for semantic cache lookup")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             prompt = get_str_from_messages(messages)
-            # Check the cache for semantically similar prompts
-            results = self.llmcache.check(prompt=prompt)
+            # Check the cache for semantically similar prompts in this exact
+            # LiteLLM cache-key scope.
+            check_kwargs: Dict[str, Any] = {
+                "prompt": prompt,
+                "filter_expression": self._get_cache_key_filter_expression(key),
+            }
+            results = self.llmcache.check(**check_kwargs)
 
             # Return None if no similar prompts found
             if not results:
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             # Process the best matching result
             cache_hit = results[0]
+            if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
+                print_verbose("Redis semantic-cache hit did not match cache key scope")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+                return None
             vector_distance = float(cache_hit["vector_distance"])
 
             # Convert vector distance back to similarity score
@@ -257,6 +352,9 @@ class RedisSemanticCache(BaseCache):
             cached_prompt = cache_hit["prompt"]
             cached_response = cache_hit["response"]
 
+            # update kwargs["metadata"] with similarity, don't rewrite the original metadata
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
+
             print_verbose(
                 f"Cache hit: similarity threshold: {self.similarity_threshold}, "
                 f"actual similarity: {similarity}, "
@@ -267,6 +365,7 @@ class RedisSemanticCache(BaseCache):
             return self._get_cache_logic(cached_response=cached_response)
         except Exception as e:
             print_verbose(f"Error retrieving from Redis semantic cache: {str(e)}")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
 
     async def _get_async_embedding(self, prompt: str, **kwargs) -> List[float]:
         """
@@ -321,7 +420,7 @@ class RedisSemanticCache(BaseCache):
         Asynchronously store a value in the semantic cache.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             value: The response value to cache
             **kwargs: Additional arguments including 'messages' for the prompt
                 and optional 'ttl' for time-to-live
@@ -341,21 +440,20 @@ class RedisSemanticCache(BaseCache):
             # Generate embedding for the value (response) to cache
             prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
 
+            store_kwargs: Dict[str, Any] = {
+                "vector": prompt_embedding,
+                "filters": self._get_cache_filters(key),
+            }
+
             # Get TTL and store in Redis semantic cache
             ttl = self._get_ttl(**kwargs)
             if ttl is not None:
-                await self.llmcache.astore(
-                    prompt,
-                    value_str,
-                    vector=prompt_embedding,  # Pass through custom embedding
-                    ttl=ttl,
-                )
-            else:
-                await self.llmcache.astore(
-                    prompt,
-                    value_str,
-                    vector=prompt_embedding,  # Pass through custom embedding
-                )
+                store_kwargs["ttl"] = ttl
+            await self.llmcache.astore(
+                prompt,
+                value_str,
+                **store_kwargs,
+            )
         except Exception as e:
             print_verbose(f"Error in async_set_cache: {str(e)}")
 
@@ -364,7 +462,7 @@ class RedisSemanticCache(BaseCache):
         Asynchronously retrieve a semantically similar cached response.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             **kwargs: Additional arguments including 'messages' for the prompt
 
         Returns:
@@ -385,17 +483,25 @@ class RedisSemanticCache(BaseCache):
             # Generate embedding for the prompt
             prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
 
-            # Check the cache for semantically similar prompts
-            results = await self.llmcache.acheck(prompt=prompt, vector=prompt_embedding)
+            # Check the cache for semantically similar prompts in this exact
+            # LiteLLM cache-key scope.
+            check_kwargs: Dict[str, Any] = {
+                "prompt": prompt,
+                "vector": prompt_embedding,
+                "filter_expression": self._get_cache_key_filter_expression(key),
+            }
+            results = await self.llmcache.acheck(**check_kwargs)
 
             # handle results / cache hit
             if not results:
-                kwargs.setdefault("metadata", {})[
-                    "semantic-similarity"
-                ] = 0.0  # TODO why here but not above??
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             cache_hit = results[0]
+            if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
+                print_verbose("Redis semantic-cache hit did not match cache key scope")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+                return None
             vector_distance = float(cache_hit["vector_distance"])
 
             # Convert vector distance back to similarity

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -119,6 +119,20 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def __init__(self):
         pass
 
+    def _normalize_tool_choice_for_responses_api(self, tool_choice: Any) -> Any:
+        """Chat tool_choice uses function.name; Responses API expects top-level name."""
+        if not isinstance(tool_choice, dict) or tool_choice.get("type") != "function":
+            return tool_choice
+        if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
+            # Return only Responses shape so stray chat ``function`` key is not sent upstream.
+            return {"type": "function", "name": tool_choice["name"]}
+        fn = tool_choice.get("function")
+        if isinstance(fn, dict):
+            fn_name = fn.get("name")
+            if isinstance(fn_name, str) and fn_name:
+                return {"type": "function", "name": fn_name}
+        return tool_choice
+
     def _handle_raw_dict_response_item(
         self, item: Dict[str, Any], index: int
     ) -> Tuple[Optional[Any], int]:
@@ -309,6 +323,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 text_format = self._transform_response_format_to_text_format(value)
                 if text_format:
                     responses_api_request["text"] = text_format  # type: ignore
+            elif key == "tool_choice":
+                responses_api_request["tool_choice"] = (  # type: ignore[assignment]
+                    self._normalize_tool_choice_for_responses_api(value)
+                )
             elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
                 responses_api_request[key] = value  # type: ignore
             elif key == "previous_response_id":

litellm/constants.py

@@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
     | (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
 )
 
+# MCP OAuth2 Token Exchange (OBO) Defaults
+MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
+    os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
+)
+
 LITELLM_UI_ALLOW_HEADERS = [
     "x-litellm-semantic-filter",
     "x-litellm-semantic-filter-tools",
@@ -202,6 +207,12 @@ DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = int(
 DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096)
 )
+DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192)
+)
+DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384)
+)
 MAX_TOKEN_TRIMMING_ATTEMPTS = int(
     os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10)
 )  # Maximum number of attempts to trim the message
@@ -224,6 +235,16 @@ AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
 )
 AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
 AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
+# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs
+# whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT
+# Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing
+# during a long provider call and the NAT reaps the flow before the response
+# arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset
+# the NAT idle timer.
+AIOHTTP_SO_KEEPALIVE = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true"
+AIOHTTP_TCP_KEEPIDLE = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60))
+AIOHTTP_TCP_KEEPINTVL = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30))
+AIOHTTP_TCP_KEEPCNT = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5))
 # enable_cleanup_closed is only needed for Python versions with the SSL leak bug
 # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
 # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
@@ -389,6 +410,8 @@ BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
 BEDROCK_MIN_THINKING_BUDGET_TOKENS = int(
     os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
 )
+# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
+ANTHROPIC_MIN_THINKING_BUDGET_TOKENS = 1024
 REPLICATE_POLLING_DELAY_SECONDS = float(
     os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
 )
@@ -409,9 +432,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0
 AUDIO_SPEECH_CHUNK_SIZE = int(
     os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
 )  # chunk_size for audio speech streaming. Balance between latency and memory usage
-MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
-    os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
-)
 DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
 #### Networking settings ####
 # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
@@ -1383,6 +1403,10 @@ except (ValueError, TypeError):
 LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
 LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
 LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
+# Stable identifier substituted in place of the master key on UserAPIKeyAuth
+# objects so the master key (or its hash) never propagates to spend logs,
+# Prometheus metrics, audit trails, or any other downstream consumer.
+LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
 
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
@@ -1411,6 +1435,7 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id"
 LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
 LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
+CLI_SSO_SESSION_TTL_SECONDS = 600
 CLI_JWT_TOKEN_NAME = "cli-jwt-token"
 # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
 CLI_JWT_EXPIRATION_HOURS = int(
@@ -1437,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
 EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
+SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
+    os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
+)
+SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
+    os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
+)
 SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(
@@ -1538,6 +1569,15 @@ DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
     os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
 )
 DEFAULT_ACCESS_GROUP_CACHE_TTL = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600))
+# Short TTL for negative MCP access-group existence lookups. Keeps unauthenticated
+# callers from forcing a DB query per request for unknown names, while bounding
+# staleness so a transient DB error (which surfaces as an empty list) cannot
+# hide a real group for long.
+DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL = 10
+# Maximum number of comma-separated MCP server / access-group tokens accepted
+# in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
+# fan-out an authenticated caller can trigger by stuffing the path with tokens.
+DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS = 16
 
 # Sentry Scrubbing Configuration
 SENTRY_DENYLIST = [

litellm/cost_calculator.py

@@ -513,7 +513,10 @@ def cost_per_token(  # noqa: PLR0915
         return fireworks_ai_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "azure":
         return azure_openai_cost_per_token(
-            model=model, usage=usage_block, response_time_ms=response_time_ms
+            model=model,
+            usage=usage_block,
+            response_time_ms=response_time_ms,
+            service_tier=service_tier,
         )
     elif custom_llm_provider == "gemini":
         return gemini_cost_per_token(
@@ -539,6 +542,7 @@ def cost_per_token(  # noqa: PLR0915
             usage=usage_block,
             response_time_ms=response_time_ms,
             request_model=request_model,
+            service_tier=service_tier,
         )
     else:
         model_info = _cached_get_model_info_helper(

litellm/experimental_mcp_client/client.py

@@ -366,6 +366,8 @@ class MCPClient:
                     headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.token:
                     headers["Authorization"] = f"token {self._mcp_auth_value}"
+                elif self.auth_type == MCPAuth.oauth2_token_exchange:
+                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

litellm/files/main.py

@@ -10,6 +10,7 @@ import contextvars
 import time
 import uuid as uuid_module
 from functools import partial
+from types import MappingProxyType
 from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 
 import httpx
@@ -85,6 +86,16 @@ bedrock_files_instance = BedrockFilesHandler()
 #################################################
 
 
+def _add_trusted_model_credentials_to_litellm_params(
+    litellm_params_dict: Dict[str, Any], kwargs: Dict[str, Any]
+) -> None:
+    trusted_model_credentials = kwargs.get("_litellm_internal_model_credentials")
+    if isinstance(trusted_model_credentials, type(MappingProxyType({}))):
+        litellm_params_dict["_litellm_internal_model_credentials"] = (
+            trusted_model_credentials
+        )
+
+
 @client
 async def acreate_file(
     file: FileTypes,
@@ -373,6 +384,10 @@ def file_retrieve(
             )
             if provider_config is not None:
                 litellm_params_dict = get_litellm_params(**kwargs)
+                _add_trusted_model_credentials_to_litellm_params(
+                    litellm_params_dict=litellm_params_dict,
+                    kwargs=kwargs,
+                )
                 litellm_params_dict["api_key"] = optional_params.api_key
                 litellm_params_dict["api_base"] = optional_params.api_base
 
@@ -497,6 +512,10 @@ def file_delete(
             pass
         optional_params = GenericLiteLLMParams(**kwargs)
         litellm_params_dict = get_litellm_params(**kwargs)
+        _add_trusted_model_credentials_to_litellm_params(
+            litellm_params_dict=litellm_params_dict,
+            kwargs=kwargs,
+        )
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         # set timeout for 10 minutes by default
@@ -846,6 +865,10 @@ def file_content(
     try:
         optional_params = GenericLiteLLMParams(**kwargs)
         litellm_params_dict = get_litellm_params(**kwargs)
+        _add_trusted_model_credentials_to_litellm_params(
+            litellm_params_dict=litellm_params_dict,
+            kwargs=kwargs,
+        )
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         client = kwargs.get("client")
@@ -993,6 +1016,7 @@ def file_content(
                 vertex_location=vertex_ai_location,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "bedrock":
             response = bedrock_files_instance.file_content(

litellm/google_genai/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.success_handler import (
@@ -29,12 +29,14 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
         litellm_logging_obj: LiteLLMLoggingObj,
         request_body: dict,
         model: str,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         self.litellm_logging_obj = litellm_logging_obj
         self.request_body = request_body
         self.start_time = datetime.now()
         self.collected_chunks: List[bytes] = []
         self.model = model
+        self._hidden_params: Dict[str, Any] = hidden_params or {}
 
     async def _handle_async_streaming_logging(
         self,
@@ -76,11 +78,13 @@ class GoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model
@@ -130,11 +134,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model

litellm/integrations/arize/_utils.py

@@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
             safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role)
 
 
+def _safe_get(obj, key, default=None):
+    """Read ``key`` from a dict-like or Pydantic-model-like object.
+
+    The arize/langfuse_otel logger receives ``usage`` objects from many sources:
+    plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI
+    Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and
+    nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT
+    expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` —
+    see https://github.com/BerriAI/litellm/issues/13672.
+    """
+    if obj is None:
+        return default
+    getter = getattr(obj, "get", None)
+    if callable(getter):
+        try:
+            return getter(key, default)
+        except TypeError:
+            # Some objects expose `.get` with a different signature
+            pass
+    return getattr(obj, key, default)
+
+
 def _set_usage_outputs(span: "Span", response_obj, span_attrs):
     usage = response_obj and response_obj.get("usage")
     if not usage:
         return
 
     safe_set_attribute(
-        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")
+        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens")
+    )
+    completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get(
+        usage, "output_tokens"
     )
-    completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens")
     if completion_tokens:
         safe_set_attribute(
             span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens
         )
-    prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens")
+    prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get(
+        usage, "input_tokens"
+    )
     if prompt_tokens:
         safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens)
-    reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens")
+
+    # Reasoning tokens live in `completion_tokens_details` for Chat Completions
+    # API (Usage) and in `output_tokens_details` for Responses API
+    # (ResponseAPIUsage). Both nested objects may be plain Pydantic models
+    # without `.get`.
+    token_details = _safe_get(usage, "completion_tokens_details") or _safe_get(
+        usage, "output_tokens_details"
+    )
+    reasoning_tokens = _safe_get(token_details, "reasoning_tokens")
     if reasoning_tokens:
         safe_set_attribute(
             span,

litellm/integrations/arize/arize_phoenix_client.py

@@ -2,11 +2,23 @@
 Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 """
 
+import urllib.parse
 from typing import Any, Dict, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_id(identifier: str) -> str:
+    """Reject path traversal characters and URL-encode the identifier."""
+    if any(c in identifier for c in ("/", "\\", "#", "?")):
+        raise ValueError(
+            f"Invalid identifier {identifier!r}: contains disallowed characters"
+        )
+    if ".." in identifier:
+        raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
+    return urllib.parse.quote(identifier, safe="")
+
+
 class ArizePhoenixClient:
     """
     Client for interacting with Arize Phoenix API to fetch prompt versions.
@@ -53,7 +65,8 @@ class ArizePhoenixClient:
         Returns:
             Dictionary containing prompt version data, or None if not found
         """
-        url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
+        safe_id = _sanitize_id(prompt_version_id)
+        url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
 
         try:
             # Use the underlying httpx client directly to avoid query param extraction

litellm/integrations/arize/arize_phoenix_prompt_manager.py

@@ -5,7 +5,8 @@ Fetches prompt versions from Arize Phoenix and provides workspace-based access c
 
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import (
@@ -74,7 +75,13 @@ class ArizePhoenixTemplateManager:
             api_key=self.api_key, api_base=self.api_base
         )
 
-        self.jinja_env = Environment(
+        # Templates fetched from Arize Phoenix come from external workspace
+        # users; in a plain `Environment()` a malicious template could reach
+        # `__class__.__init__.__globals__` and execute arbitrary code on the
+        # proxy host. The sandbox blocks that attribute traversal while
+        # leaving normal `{{ var }}` substitution intact. Matches the
+        # dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             # Use Mustache/Handlebars-style delimiters

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
 
 import asyncio
 import os
+import time
 import traceback
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.types.utils import StandardLoggingPayload
+from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 
 class AzureSentinelLogger(CustomBatchLogger):
@@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         tenant_id: Optional[str] = None,
         client_id: Optional[str] = None,
         client_secret: Optional[str] = None,
+        audit_stream_name: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
                 If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
             client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
                 If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
+            audit_stream_name (str, optional): Stream name from DCR for audit logs.
+                If not provided, audit logs use the standard stream name.
         """
         self.async_httpx_client = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.LoggingCallback
         )
 
-        self.dcr_immutable_id = dcr_immutable_id or os.getenv(
+        resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
             "AZURE_SENTINEL_DCR_IMMUTABLE_ID"
         )
-        self.stream_name = stream_name or os.getenv(
-            "AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
+        resolved_stream_name = (
+            stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
         )
-        self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
-        self.tenant_id = (
+        resolved_audit_stream_name = audit_stream_name or resolved_stream_name
+        resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
+        resolved_tenant_id = (
             tenant_id
             or os.getenv("AZURE_SENTINEL_TENANT_ID")
             or os.getenv("AZURE_TENANT_ID")
         )
-        self.client_id = (
+        resolved_client_id = (
             client_id
             or os.getenv("AZURE_SENTINEL_CLIENT_ID")
             or os.getenv("AZURE_CLIENT_ID")
         )
-        self.client_secret = (
+        resolved_client_secret = (
             client_secret
             or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
             or os.getenv("AZURE_CLIENT_SECRET")
         )
 
-        if not self.dcr_immutable_id:
+        if not resolved_dcr_immutable_id:
             raise ValueError(
                 "AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
             )
-        if not self.endpoint:
+        if not resolved_endpoint:
             raise ValueError(
                 "AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
             )
-        if not self.tenant_id:
+        if not resolved_tenant_id:
             raise ValueError(
                 "AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
             )
-        if not self.client_id:
+        if not resolved_client_id:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
             )
-        if not self.client_secret:
+        if not resolved_client_secret:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
             )
 
+        self.dcr_immutable_id = resolved_dcr_immutable_id
+        self.stream_name = resolved_stream_name
+        self.audit_stream_name = resolved_audit_stream_name
+        self.endpoint = resolved_endpoint
+        self.tenant_id = resolved_tenant_id
+        self.client_id = resolved_client_id
+        self.client_secret = resolved_client_secret
+
         # Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
-        self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
+        self.api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_stream_name,
+        )
+        self.audit_api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_audit_stream_name,
+        )
 
         # OAuth2 scope for Azure Monitor
         self.oauth_scope = "https://monitor.azure.com/.default"
@@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
         super().__init__(**kwargs, flush_lock=self.flush_lock)
         asyncio.create_task(self.periodic_flush())
         self.log_queue: List[StandardLoggingPayload] = []
+        self.audit_log_queue: List[StandardAuditLogPayload] = []
+
+    @staticmethod
+    def _build_api_endpoint(
+        endpoint: str, dcr_immutable_id: str, stream_name: str
+    ) -> str:
+        return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
 
     async def _get_oauth_token(self) -> str:
         """
@@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         Returns:
             Bearer token string
         """
-        # Check if we have a valid cached token
-        import time
-
         if (
             self.oauth_token
             and self.oauth_token_expires_at
@@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         if not self.oauth_token:
             raise Exception("OAuth2 token response did not contain access_token")
 
-        # Cache token expiry time
-        import time
-
         self.oauth_token_expires_at = time.time() + expires_in
 
         return self.oauth_token
@@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
             )
             pass
 
+    async def async_log_audit_log_event(
+        self, audit_log: StandardAuditLogPayload
+    ) -> None:
+        """
+        Async log LiteLLM audit log events to Azure Sentinel.
+
+        Audit logs are queued separately from standard LLM logs so mixed callback
+        usage never sends schema-mismatched records in the same ingestion batch.
+        """
+        try:
+            verbose_logger.debug(
+                "Azure Sentinel: Logging audit event id=%s action=%s table=%s",
+                audit_log.get("id"),
+                audit_log.get("action"),
+                audit_log.get("table_name"),
+            )
+
+            self.audit_log_queue.append(audit_log)
+
+            if len(self.audit_log_queue) >= self.batch_size:
+                await self.async_send_audit_batch()
+
+        except Exception as e:
+            verbose_logger.exception(
+                f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
+            )
+            pass
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
+        await self._async_send_batch_to_api(
+            log_queue=self.log_queue,
+            api_endpoint=self.api_endpoint,
+            log_type="logs",
+        )
+
+    async def async_send_audit_batch(self):
+        """
+        Sends the batch of audit logs to Azure Monitor Logs Ingestion API
+        """
+        await self._async_send_batch_to_api(
+            log_queue=self.audit_log_queue,
+            api_endpoint=self.audit_api_endpoint,
+            log_type="audit logs",
+        )
+
+    async def _async_send_batch_to_api(
+        self,
+        log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
+        api_endpoint: str,
+        log_type: str,
+    ) -> None:
         try:
-            if not self.log_queue:
+            if not log_queue:
                 return
 
             verbose_logger.debug(
-                "Azure Sentinel - about to flush %s events", len(self.log_queue)
+                "Azure Sentinel - about to flush %s %s", len(log_queue), log_type
             )
 
-            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-
             # Get OAuth2 token
             bearer_token = await self._get_oauth_token()
 
             # Convert log queue to JSON array format expected by Logs Ingestion API
             # Each log entry should be a JSON object in the array
-            body = safe_dumps(self.log_queue)
+            body = safe_dumps(log_queue)
 
             # Set headers for Logs Ingestion API
             headers = {
@@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             # Send the request
             response = await self.async_httpx_client.post(
-                url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
+                url=api_endpoint, data=body.encode("utf-8"), headers=headers
             )
 
             if response.status_code not in [200, 204]:
@@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
                 f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
             )
         finally:
-            self.log_queue.clear()
+            log_queue.clear()
+
+    async def flush_queue(self):
+        if self.flush_lock is None:
+            return
+
+        async with self.flush_lock:
+            if self.log_queue:
+                await self.async_send_batch()
+            if self.audit_log_queue:
+                await self.async_send_audit_batch()
+            self.last_flush_time = time.time()

litellm/integrations/bitbucket/bitbucket_client.py

@@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
 """
 
 import base64
+import urllib.parse
 from typing import Any, Dict, List, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_file_path(file_path: str) -> str:
+    """Reject path traversal and URL-encode each path segment."""
+    if "#" in file_path or "?" in file_path:
+        raise ValueError(
+            f"Invalid file path {file_path!r}: contains URL special characters"
+        )
+    parts = file_path.split("/")
+    for part in parts:
+        if part == "..":
+            raise ValueError(
+                f"Invalid file path {file_path!r}: path traversal detected"
+            )
+    return "/".join(urllib.parse.quote(part, safe="") for part in parts)
+
+
 class BitBucketClient:
     """
     Client for interacting with BitBucket API to fetch .prompt files.
@@ -72,7 +88,8 @@ class BitBucketClient:
         Returns:
             File content as string, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -119,7 +136,8 @@ class BitBucketClient:
         Returns:
             List of file paths
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
+        safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -211,7 +229,8 @@ class BitBucketClient:
         Returns:
             Dictionary containing file metadata, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             # Use GET with Range header to get just the headers (HEAD equivalent)

litellm/integrations/bitbucket/bitbucket_prompt_manager.py

@@ -5,7 +5,8 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 
@@ -74,7 +75,13 @@ class BitBucketTemplateManager:
         self.prompts: Dict[str, BitBucketPromptTemplate] = {}
         self.bitbucket_client = BitBucketClient(bitbucket_config)
 
-        self.jinja_env = Environment(
+        # Templates fetched from a BitBucket repo are not trustworthy:
+        # anyone with repo write access can ship Jinja syntax that, in a
+        # plain `Environment()`, would reach `__class__.__init__.__globals__`
+        # and pivot into RCE on the proxy host. The sandbox blocks that
+        # attribute traversal while leaving normal `{{ var }}` substitution
+        # intact. Matches the dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             # Use Handlebars-style delimiters to match Dotprompt spec

litellm/integrations/custom_guardrail.py

@@ -888,6 +888,15 @@ def log_guardrail_information(func):
         - pre_call
         - during_call
         - post_call
+
+    Some guardrails (e.g. ``block_code_execution``) call
+    ``add_standard_logging_guardrail_information_to_request_data`` directly
+    from inside the wrapped function so they can record a richer payload
+    (structured detections, tracing detail) than this decorator's
+    "allow"/"mask"/raw-response default. To avoid double-recording in that
+    case (which would emit two spans, two Datadog records, two spend-log
+    entries, etc.), snapshot the entry count before invocation: if the
+    wrapped function already appended its own entry, skip the auto-record.
     """
     import functools
     import inspect
@@ -907,6 +916,16 @@ def log_guardrail_information(func):
             return GuardrailEventHooks.post_call
         return None
 
+    def _count_recorded_guardrail_entries(request_data: dict) -> int:
+        total = 0
+        for container_key in ("metadata", "litellm_metadata"):
+            container = request_data.get(container_key)
+            if isinstance(container, dict):
+                entries = container.get("standard_logging_guardrail_information")
+                if isinstance(entries, list):
+                    total += len(entries)
+        return total
+
     @functools.wraps(func)
     async def async_wrapper(*args, **kwargs):
         start_time = datetime.now()  # Move start_time inside the wrapper
@@ -919,8 +938,11 @@ def log_guardrail_information(func):
         if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
             original_inputs = kwargs.get("inputs")
 
+        entries_before = _count_recorded_guardrail_entries(request_data)
         try:
             response = await func(*args, **kwargs)
+            if _count_recorded_guardrail_entries(request_data) > entries_before:
+                return response
             return self._process_response(
                 response=response,
                 request_data=request_data,
@@ -931,6 +953,8 @@ def log_guardrail_information(func):
                 original_inputs=original_inputs,
             )
         except Exception as e:
+            if _count_recorded_guardrail_entries(request_data) > entries_before:
+                raise
             return self._process_error(
                 e=e,
                 request_data=request_data,
@@ -952,8 +976,11 @@ def log_guardrail_information(func):
         if func.__name__ == "apply_guardrail" and "inputs" in kwargs:
             original_inputs = kwargs.get("inputs")
 
+        entries_before = _count_recorded_guardrail_entries(request_data)
         try:
             response = func(*args, **kwargs)
+            if _count_recorded_guardrail_entries(request_data) > entries_before:
+                return response
             return self._process_response(
                 response=response,
                 request_data=request_data,
@@ -962,6 +989,8 @@ def log_guardrail_information(func):
                 original_inputs=original_inputs,
             )
         except Exception as e:
+            if _count_recorded_guardrail_entries(request_data) > entries_before:
+                raise
             return self._process_error(
                 e=e,
                 request_data=request_data,

litellm/integrations/custom_sso_handler.py

@@ -18,6 +18,17 @@ class CustomSSOLoginHandler(CustomLogger):
         self,
         request: Request,
     ) -> OpenID:
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
+        from litellm.proxy.proxy_server import general_settings
+
+        require_trusted_proxy_request(
+            request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
+        )
+
         request_headers_dict = dict(request.headers)
         return OpenID(
             id=request_headers_dict.get("x-litellm-user-id"),

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -6,12 +6,14 @@ import time
 from litellm._uuid import uuid
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
-from urllib.parse import quote
 
 from litellm._logging import verbose_logger
 from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
 from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
 from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
+from litellm.litellm_core_utils.cloud_storage_security import (
+    sanitize_cloud_object_component,
+)
 from litellm.proxy._types import CommonProxyErrors
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
@@ -335,7 +337,11 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         _litellm_params = kwargs.get("litellm_params", None) or {}
         _metadata = _litellm_params.get("metadata", None) or {}
         if "gcs_log_id" in _metadata:
-            object_name = _metadata["gcs_log_id"]
+            safe_log_id = sanitize_cloud_object_component(
+                _metadata.get("gcs_log_id"), fallback=""
+            )
+            if safe_log_id:
+                object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
 
         return object_name
 
@@ -367,8 +373,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
                     request_date_str=date_str,
                     response_id=request_id,
                 )
-                encoded_object_name = quote(object_name, safe="")
-                response = await self.download_gcs_object(encoded_object_name)
+                response = await self.download_gcs_object(object_name)
 
                 if response is not None:
                     loaded_response = json.loads(response)

litellm/integrations/gcs_bucket/gcs_bucket_base.py

@@ -11,6 +11,10 @@ from litellm.integrations.gcs_bucket.gcs_bucket_mock_client import (
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.cloud_storage_security import (
+    encode_gcs_object_name_for_url,
+    split_configured_cloud_bucket_name,
+)
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -133,8 +137,8 @@ class GCSBucketBase(CustomBatchLogger):
             - Returns: bucket_name="my-bucket", object_name="my-folder/dev/my-object"
 
         """
-        if "/" in bucket_name:
-            bucket_name, prefix = bucket_name.split("/", 1)
+        bucket_name, prefix = split_configured_cloud_bucket_name(bucket_name)
+        if prefix:
             object_name = f"{prefix}/{object_name}"
             return bucket_name, object_name
         return bucket_name, object_name
@@ -248,6 +252,7 @@ class GCSBucketBase(CustomBatchLogger):
                 bucket_name=bucket_name,
                 object_name=object_name,
             )
+            object_name = encode_gcs_object_name_for_url(object_name)
 
             url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media"
 
@@ -288,6 +293,7 @@ class GCSBucketBase(CustomBatchLogger):
                 bucket_name=bucket_name,
                 object_name=object_name,
             )
+            object_name = encode_gcs_object_name_for_url(object_name)
 
             url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}"
 
@@ -334,10 +340,11 @@ class GCSBucketBase(CustomBatchLogger):
             bucket_name=bucket_name,
             object_name=object_name,
         )
+        encoded_object_name = encode_gcs_object_name_for_url(object_name)
 
         response = await self.async_httpx_client.post(
             headers=headers,
-            url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}",
+            url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={encoded_object_name}",
             data=json_logged_payload,
         )
 

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -4,7 +4,8 @@ GitLab prompt manager with configurable prompts folder.
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 
@@ -90,7 +91,13 @@ class GitLabTemplateManager:
             or ""
         ).strip("/")
 
-        self.jinja_env = Environment(
+        # Templates fetched from a GitLab repo are not trustworthy:
+        # anyone with repo write access can ship Jinja syntax that, in a
+        # plain `Environment()`, would reach `__class__.__init__.__globals__`
+        # and pivot into RCE on the proxy host. The sandbox blocks that
+        # attribute traversal while leaving normal `{{ var }}` substitution
+        # intact. Matches the dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             variable_start_string="{{",

litellm/integrations/langfuse/langfuse.py

@@ -90,6 +90,29 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
     return cache_read_input_tokens
 
 
+def resolve_langfuse_credentials(
+    langfuse_public_key=None,
+    langfuse_secret=None,
+    langfuse_secret_key=None,
+    langfuse_host=None,
+    allow_env_credentials: bool = True,
+):
+    if allow_env_credentials is False and langfuse_host is not None:
+        secret_key = langfuse_secret or langfuse_secret_key
+        public_key = langfuse_public_key
+    else:
+        secret_key = (
+            langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
+        )
+        public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
+
+    resolved_host = langfuse_host or os.getenv(
+        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    )
+
+    return public_key, secret_key, resolved_host
+
+
 class LangFuseLogger:
     # Class variables or attributes
     def __init__(
@@ -98,6 +121,7 @@ class LangFuseLogger:
         langfuse_secret=None,
         langfuse_host=None,
         flush_interval=1,
+        allow_env_credentials: bool = True,
     ):
         try:
             import langfuse
@@ -106,11 +130,13 @@ class LangFuseLogger:
             raise Exception(
                 f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n{traceback.format_exc()}\033[0m"
             )
-        # Instance variables
-        self.secret_key = langfuse_secret or os.getenv("LANGFUSE_SECRET_KEY")
-        self.public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-        self.langfuse_host = langfuse_host or os.getenv(
-            "LANGFUSE_HOST", "https://cloud.langfuse.com"
+        self.public_key, self.secret_key, self.langfuse_host = (
+            resolve_langfuse_credentials(
+                langfuse_public_key=langfuse_public_key,
+                langfuse_secret=langfuse_secret,
+                langfuse_host=langfuse_host,
+                allow_env_credentials=allow_env_credentials,
+            )
         )
         if not (
             self.langfuse_host.startswith("http://")
@@ -160,9 +186,10 @@ class LangFuseLogger:
                 project_id = None
 
         if os.getenv("UPSTREAM_LANGFUSE_SECRET_KEY") is not None:
+            upstream_langfuse_debug_env = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
             upstream_langfuse_debug = (
-                str_to_bool(self.upstream_langfuse_debug)
-                if self.upstream_langfuse_debug is not None
+                str_to_bool(upstream_langfuse_debug_env)
+                if upstream_langfuse_debug_env is not None
                 else None
             )
             self.upstream_langfuse_secret_key = os.getenv(
@@ -173,7 +200,7 @@ class LangFuseLogger:
             )
             self.upstream_langfuse_host = os.getenv("UPSTREAM_LANGFUSE_HOST")
             self.upstream_langfuse_release = os.getenv("UPSTREAM_LANGFUSE_RELEASE")
-            self.upstream_langfuse_debug = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
+            self.upstream_langfuse_debug = upstream_langfuse_debug_env
             self.upstream_langfuse = Langfuse(
                 public_key=self.upstream_langfuse_public_key,
                 secret_key=self.upstream_langfuse_secret_key,

litellm/integrations/langfuse/langfuse_handler.py

@@ -115,8 +115,10 @@ class LangFuseHandler:
 
         langfuse_logger = LangFuseLogger(
             langfuse_public_key=credentials.get("langfuse_public_key"),
-            langfuse_secret=credentials.get("langfuse_secret"),
+            langfuse_secret=credentials.get("langfuse_secret")
+            or credentials.get("langfuse_secret_key"),
             langfuse_host=credentials.get("langfuse_host"),
+            allow_env_credentials=credentials.get("langfuse_host") is None,
         )
         in_memory_dynamic_logger_cache.set_cache(
             credentials=credentials,

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -20,7 +20,7 @@ from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import (
     DynamicLoggingCache,
 )
 from ..prompt_management_base import PromptManagementBase
-from .langfuse import LangFuseLogger
+from .langfuse import LangFuseLogger, resolve_langfuse_credentials
 from .langfuse_handler import LangFuseHandler
 
 if TYPE_CHECKING:
@@ -46,6 +46,7 @@ def langfuse_client_init(
     langfuse_secret_key=None,
     langfuse_host=None,
     flush_interval=1,
+    allow_env_credentials: bool = True,
 ) -> LangfuseClass:
     """
     Initialize Langfuse client with caching to prevent multiple initializations.
@@ -70,14 +71,12 @@ def langfuse_client_init(
             f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n\033[0m"
         )
 
-    # Instance variables
-
-    secret_key = (
-        langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
-    )
-    public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-    langfuse_host = langfuse_host or os.getenv(
-        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    public_key, secret_key, langfuse_host = resolve_langfuse_credentials(
+        langfuse_public_key=langfuse_public_key,
+        langfuse_secret=langfuse_secret,
+        langfuse_secret_key=langfuse_secret_key,
+        langfuse_host=langfuse_host,
+        allow_env_credentials=allow_env_credentials,
     )
 
     if not (
@@ -222,6 +221,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,
@@ -246,6 +246,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,

litellm/integrations/langsmith.py

@@ -19,6 +19,7 @@ from litellm.integrations.langsmith_mock_client import (
     create_mock_langsmith_client,
     should_use_langsmith_mock,
 )
+from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -112,17 +113,28 @@ class LangsmithLogger(CustomBatchLogger):
         langsmith_project: Optional[str] = None,
         langsmith_base_url: Optional[str] = None,
         langsmith_tenant_id: Optional[str] = None,
+        allow_env_credentials: bool = True,
     ) -> LangsmithCredentialsObject:
-        _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
-        _credentials_project = (
-            langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
-        )
-        _credentials_base_url = (
-            langsmith_base_url
-            or os.getenv("LANGSMITH_BASE_URL")
-            or "https://api.smith.langchain.com"
-        )
-        _credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
+        if allow_env_credentials is False and langsmith_base_url is not None:
+            _credentials_api_key = langsmith_api_key
+            _credentials_project = langsmith_project or "litellm-completion"
+            _credentials_base_url = langsmith_base_url
+            _credentials_tenant_id = langsmith_tenant_id
+        else:
+            _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
+            _credentials_project = (
+                langsmith_project
+                or os.getenv("LANGSMITH_PROJECT")
+                or "litellm-completion"
+            )
+            _credentials_base_url = (
+                langsmith_base_url
+                or os.getenv("LANGSMITH_BASE_URL")
+                or "https://api.smith.langchain.com"
+            )
+            _credentials_tenant_id = langsmith_tenant_id or os.getenv(
+                "LANGSMITH_TENANT_ID"
+            )
 
         return LangsmithCredentialsObject(
             LANGSMITH_API_KEY=_credentials_api_key,
@@ -153,6 +165,15 @@ class LangsmithLogger(CustomBatchLogger):
             for key in ("session_id", "thread_id", "conversation_id"):
                 if key in requester_metadata and key not in extra_metadata:
                     extra_metadata[key] = requester_metadata[key]
+
+        # helper is shallow; also scrub nested requester_metadata since
+        # LangSmith forwards the whole dict into `extra`
+        extra_metadata = redact_user_api_key_info(metadata=extra_metadata)
+        nested = extra_metadata.get("requester_metadata")
+        if isinstance(nested, dict):
+            extra_metadata["requester_metadata"] = redact_user_api_key_info(
+                metadata=nested
+            )
         return extra_metadata
 
     def _build_outputs_with_usage(
@@ -540,6 +561,10 @@ class LangsmithLogger(CustomBatchLogger):
                 langsmith_tenant_id=standard_callback_dynamic_params.get(
                     "langsmith_tenant_id", None
                 ),
+                allow_env_credentials=standard_callback_dynamic_params.get(
+                    "langsmith_base_url", None
+                )
+                is None,
             )
         else:
             credentials = self.default_credentials

litellm/integrations/opentelemetry.py

@@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
 
+CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
+CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
+CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
+CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
+_VALID_CAPTURE_MODES = {
+    CAPTURE_MODE_NO_CONTENT,
+    CAPTURE_MODE_SPAN_ONLY,
+    CAPTURE_MODE_EVENT_ONLY,
+    CAPTURE_MODE_SPAN_AND_EVENT,
+}
+
 
 @dataclass
 class OpenTelemetryConfig:
@@ -69,6 +80,11 @@ class OpenTelemetryConfig:
     deployment_environment: Optional[str] = None
     model_id: Optional[str] = None
     ignore_context_propagation: Optional[bool] = None
+    # When True, create a private TracerProvider instead of reusing or setting the global one.
+    skip_set_global: bool = False
+    # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
+    # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
+    capture_message_content: Optional[str] = None
 
     def __post_init__(self) -> None:
         # If endpoint is specified but exporter is still the default "console",
@@ -180,6 +196,9 @@ class OpenTelemetry(CustomLogger):
         super().__init__(**kwargs)
         self._init_metrics(meter_provider)
         self._init_logs(logger_provider)
+        # Sample env-var / config / message_logging at init so subsequent
+        # _capture_in_span / _capture_in_event calls are deterministic.
+        self._capture_mode_cached = self._compute_capture_mode_from_init_state()
         self._init_otel_logger_on_litellm_proxy()
 
     @staticmethod
@@ -218,7 +237,14 @@ class OpenTelemetry(CustomLogger):
             not isinstance(cb, OpenTelemetry) for cb in litellm.service_callback
         ):
             litellm.service_callback.append(self)
-        setattr(proxy_server, "open_telemetry_logger", self)
+        # avoid proxy logger ownership being overwritten by later
+        # handlers. Multiple integrations (default OTEL, Langfuse OTEL,
+        # Arize OTEL, etc.) may initialize in sequence; without this guard,
+        # the last one silently replaces the first and breaks expected
+        # routing for proxy_server.open_telemetry_logger consumers.
+        # Behavior: first-registered wins.
+        if getattr(proxy_server, "open_telemetry_logger", None) is None:
+            setattr(proxy_server, "open_telemetry_logger", self)
 
     def _get_or_create_provider(
         self,
@@ -259,16 +285,21 @@ class OpenTelemetry(CustomLogger):
         try:
             existing_provider = get_existing_provider_fn()
 
-            # If a real SDK provider exists (set by another SDK like Langfuse), use it
-            # This uses a positive check for SDK providers instead of a negative check for proxy providers
             if isinstance(existing_provider, sdk_provider_class):
-                verbose_logger.debug(
-                    "OpenTelemetry: Using existing %s: %s",
-                    provider_name,
-                    type(existing_provider).__name__,
-                )
-                provider = existing_provider
-                # Don't call set_provider to preserve existing context
+                if skip_set_global:
+                    verbose_logger.debug(
+                        "OpenTelemetry: existing %s found but skip_set_global=True; creating private %s for isolation",
+                        provider_name,
+                        provider_name,
+                    )
+                    provider = create_new_provider_fn()
+                else:
+                    verbose_logger.debug(
+                        "OpenTelemetry: Using existing %s: %s",
+                        provider_name,
+                        type(existing_provider).__name__,
+                    )
+                    provider = existing_provider
             else:
                 # Default proxy provider or unknown type, create our own
                 verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
@@ -293,6 +324,68 @@ class OpenTelemetry(CustomLogger):
 
         return provider
 
+    def _skip_set_global(self) -> bool:
+        # langfuse_otel relies on the Langfuse SDK's providers; don't overwrite them.
+        return self.config.skip_set_global or (
+            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
+        )
+
+    def _compute_capture_mode_from_init_state(self) -> Optional[str]:
+        """Sample explicit settings at init. Returns the resolved mode or
+        None if nothing explicit is set (in which case the legacy
+        ``self.message_logging`` flag is consulted dynamically per request).
+
+        ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
+        ``"false"``/``"0"`` map to ``NO_CONTENT``.
+        Unknown values are ignored.
+        """
+        explicit = self.config.capture_message_content or os.getenv(
+            "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+        )
+        if not explicit:
+            return None
+        normalized = explicit.upper()
+        if normalized in ("TRUE", "1"):
+            return CAPTURE_MODE_EVENT_ONLY
+        if normalized in ("FALSE", "0"):
+            return CAPTURE_MODE_NO_CONTENT
+        if normalized in _VALID_CAPTURE_MODES:
+            return normalized
+        return None
+
+    def _resolve_capture_mode(self) -> str:
+        """Return the active capture mode for this request.
+
+        Precedence:
+          1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
+             (kill-switch checked dynamically).
+          2. Explicit setting sampled at init from
+             ``OpenTelemetryConfig.capture_message_content`` or
+             ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
+          3. Legacy ``self.message_logging`` (checked dynamically).
+        """
+        if litellm.turn_off_message_logging:
+            return CAPTURE_MODE_NO_CONTENT
+        if self._capture_mode_cached is not None:
+            return self._capture_mode_cached
+        return (
+            CAPTURE_MODE_SPAN_AND_EVENT
+            if self.message_logging
+            else CAPTURE_MODE_NO_CONTENT
+        )
+
+    def _capture_in_span(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_SPAN_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
+    def _capture_in_event(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_EVENT_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
     def _init_tracing(self, tracer_provider):
         from opentelemetry import trace
         from opentelemetry.sdk.trace import TracerProvider
@@ -303,11 +396,6 @@ class OpenTelemetry(CustomLogger):
             provider.add_span_processor(self._get_span_processor())
             return provider
 
-        # CRITICAL FIX: For Langfuse OTEL, skip setting global provider to prevent interference
-        skip_global = (
-            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
-        )
-
         tracer_provider = self._get_or_create_provider(
             provider=tracer_provider,
             provider_name="TracerProvider",
@@ -315,16 +403,18 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=TracerProvider,
             create_new_provider_fn=create_tracer_provider,
             set_provider_fn=trace.set_tracer_provider,
-            skip_set_global=skip_global,
+            skip_set_global=self._skip_set_global(),
         )
 
         # Grab our tracer from the TracerProvider (not from global context)
         # This ensures we use the provided TracerProvider (e.g., for testing)
         self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
+        self._tracer_provider = tracer_provider
         self.span_kind = SpanKind
 
     def _init_metrics(self, meter_provider):
         if not self.config.enable_metrics:
+            self._meter_provider = None
             self._operation_duration_histogram = None
             self._token_usage_histogram = None
             self._cost_histogram = None
@@ -350,7 +440,9 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=MeterProvider,
             create_new_provider_fn=create_meter_provider,
             set_provider_fn=metrics.set_meter_provider,
+            skip_set_global=self._skip_set_global(),
         )
+        self._meter_provider = meter_provider
 
         meter = meter_provider.get_meter(__name__)
 
@@ -388,6 +480,7 @@ class OpenTelemetry(CustomLogger):
     def _init_logs(self, logger_provider):
         # nothing to do if events disabled
         if not self.config.enable_events:
+            self._logger_provider = None
             return
 
         from opentelemetry._logs import get_logger_provider, set_logger_provider
@@ -404,13 +497,14 @@ class OpenTelemetry(CustomLogger):
             )
             return provider
 
-        self._get_or_create_provider(
+        self._logger_provider = self._get_or_create_provider(
             provider=logger_provider,
             provider_name="LoggerProvider",
             get_existing_provider_fn=get_logger_provider,
             sdk_provider_class=OTLoggerProvider,
             create_new_provider_fn=create_logger_provider,
             set_provider_fn=set_logger_provider,
+            skip_set_global=self._skip_set_global(),
         )
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
@@ -707,12 +801,100 @@ class OpenTelemetry(CustomLogger):
     # End of Team/Key Based Logging Control Flow
     #########################################################
 
+    def _emit_once(self, kwargs: dict, *scope: object) -> bool:
+        """Return True the first time this handler is asked to emit a span
+        for the given (handler, scope) on this kwargs; False on repeats.
+
+        Used to suppress duplicate span emission for two distinct patterns:
+
+        1. **Handler-level dual-fire**: streaming code paths trigger both
+           the sync and async callback for one request, so ``_handle_success``
+           / ``_handle_failure`` would otherwise produce two
+           ``litellm_request`` spans. Scope: ``("success",)`` / ``("failure",)``.
+        2. **Payload-driven multi-entrypoint emission**: a span loop that
+           reads entries from ``standard_logging_payload`` (currently only
+           guardrails) is invoked from multiple lifecycle points
+           (post-call hooks, success callback, failure callback). The list
+           can be re-read with mutated entries between calls, so dedupe
+           must be at entry granularity. Scope: the entry's stable identity.
+
+        ``scope`` parts can be any hashable identity. The marker is stored
+        in ``kwargs["litellm_params"]["metadata"]["_otel_internal"]`` so it
+        is request-local (kwargs is shared across the sync/async callbacks
+        and lifecycle hooks for one request).
+        """
+        litellm_params = kwargs.get("litellm_params")
+        if not isinstance(litellm_params, dict):
+            litellm_params = {}
+            kwargs["litellm_params"] = litellm_params
+
+        _metadata = litellm_params.get("metadata")
+        if not isinstance(_metadata, dict):
+            _metadata = {}
+            litellm_params["metadata"] = _metadata
+
+        _otel_internal = _metadata.get("_otel_internal")
+        if not isinstance(_otel_internal, dict):
+            _otel_internal = {}
+            _metadata["_otel_internal"] = _otel_internal
+
+        spans_logged = _otel_internal.get("spans_logged")
+        if not isinstance(spans_logged, dict):
+            spans_logged = {}
+            _otel_internal["spans_logged"] = spans_logged
+
+        dedupe_key = (self.__class__.__name__, id(self), *scope)
+        if spans_logged.get(dedupe_key) is True:
+            return False
+
+        spans_logged[dedupe_key] = True
+        return True
+
+    def _end_proxy_span_from_kwargs(self, kwargs: dict, end_time) -> None:
+        """Close the proxy-level parent span if it is still recording.
+
+        This helper retrieves the proxy span directly from kwargs metadata
+        and closes it after all child spans have been recorded.
+
+        Only called from the success path. The failure path deliberately
+        leaves the proxy span open so ``async_post_call_failure_hook`` can
+        append the ``"Failed Proxy Server Request"`` child span before
+        closing it.
+
+        Only spans named ``LITELLM_PROXY_REQUEST_SPAN_NAME`` are closed —
+        externally provided spans must not be closed by LiteLLM.
+        """
+        litellm_params = kwargs.get("litellm_params", {}) or {}
+        _metadata = litellm_params.get("metadata", {}) or {}
+        proxy_span = _metadata.get("litellm_parent_otel_span", None)
+        if (
+            proxy_span is not None
+            and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
+            and hasattr(proxy_span, "is_recording")
+            and proxy_span.is_recording()
+        ):
+            proxy_span.end(end_time=self._to_ns(end_time))
+
     def _handle_success(self, kwargs, response_obj, start_time, end_time):
+        """Create the litellm_request span then close the proxy span."""
         verbose_logger.debug(
             "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
             kwargs,
             self.config,
         )
+
+        # sync + async success handlers can both fire for one
+        # request (notably in streaming code paths). Guard against duplicate
+        # span writes — but still close the proxy span on the skip path so
+        # the trace doesn't leak an open root span.
+        if not self._emit_once(kwargs, "success"):
+            verbose_logger.debug(
+                "OpenTelemetry: skipping duplicate success span for handler=%s",
+                self.__class__.__name__,
+            )
+            self._end_proxy_span_from_kwargs(kwargs, end_time)
+            return
+
         ctx, parent_span = self._get_span_context(kwargs)
 
         if self.config.ignore_context_propagation:
@@ -772,7 +954,7 @@ class OpenTelemetry(CustomLogger):
 
         # 6. Do NOT end parent span - it should be managed by its creator
         # External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
-        # However, proxy-created spans should be closed here
+        # However, proxy-created spans should be closed here.
         if (
             parent_span is not None
             and hasattr(parent_span, "name")
@@ -780,6 +962,11 @@ class OpenTelemetry(CustomLogger):
         ):
             parent_span.end(end_time=self._to_ns(end_time))
 
+        # close the proxy span explicitly from kwargs metadata
+        # after all child spans (litellm_request, guardrail, raw_request)
+        # have been fully recorded and exported.
+        self._end_proxy_span_from_kwargs(kwargs, end_time)
+
     def _start_primary_span(
         self,
         kwargs,
@@ -811,8 +998,7 @@ class OpenTelemetry(CustomLogger):
         from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
-        # only log raw LLM request/response if message_logging is on and not globally turned off
-        if litellm.turn_off_message_logging or not self.message_logging:
+        if not self._capture_in_span():
             return
 
         litellm_params = kwargs.get("litellm_params", {})
@@ -1073,7 +1259,7 @@ class OpenTelemetry(CustomLogger):
         # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
         # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
 
-        from opentelemetry._logs import SeverityNumber, get_logger
+        from opentelemetry._logs import SeverityNumber
 
         try:
             from opentelemetry.sdk._logs import (  # type: ignore[attr-defined]  # OTEL < 1.39.0
@@ -1084,7 +1270,10 @@ class OpenTelemetry(CustomLogger):
                 LogRecord as SdkLogRecord,  # type: ignore[attr-defined]  # OTEL >= 1.39.0
             )
 
-        otel_logger = get_logger(LITELLM_LOGGER_NAME)
+        # Resolve through the handler's own LoggerProvider (which may be a
+        # private one when skip_set_global=True) rather than the module-level
+        # get_logger() which always goes through the global provider.
+        otel_logger = self._logger_provider.get_logger(LITELLM_LOGGER_NAME)
 
         parent_ctx = span.get_span_context()
         provider = (kwargs.get("litellm_params") or {}).get(
@@ -1100,9 +1289,14 @@ class OpenTelemetry(CustomLogger):
             }
             if role == "tool" and msg.get("id"):
                 attrs["id"] = msg["id"]
-            if self.message_logging and msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and msg.get("content"):
                 attrs["gen_ai.prompt"] = msg["content"]
 
+            body = msg.copy()
+            if not capture_event_content:
+                body.pop("content", None)
+
             log_record = SdkLogRecord(
                 timestamp=self._to_ns(datetime.now()),
                 trace_id=parent_ctx.trace_id,
@@ -1110,7 +1304,7 @@ class OpenTelemetry(CustomLogger):
                 trace_flags=parent_ctx.trace_flags,
                 severity_number=SeverityNumber.INFO,
                 severity_text="INFO",
-                body=msg.copy(),
+                body=body,
                 attributes=attrs,
             )
             otel_logger.emit(log_record)
@@ -1124,14 +1318,15 @@ class OpenTelemetry(CustomLogger):
                 "finish_reason": choice.get("finish_reason"),
             }
             body_msg = choice.get("message", {})
-            if self.message_logging and body_msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and body_msg.get("content"):
                 attrs["message.content"] = body_msg["content"]
             body = {
                 "index": idx,
                 "finish_reason": choice.get("finish_reason"),
                 "message": {"role": body_msg.get("role", "assistant")},
             }
-            if self.message_logging and body_msg.get("content"):
+            if capture_event_content and body_msg.get("content"):
                 body["message"]["content"] = body_msg["content"]
 
             log_record = SdkLogRecord(
@@ -1201,6 +1396,21 @@ class OpenTelemetry(CustomLogger):
         for guardrail_information in guardrail_information_list:
             start_time_float = guardrail_information.get("start_time")
             end_time_float = guardrail_information.get("end_time")
+
+            # ``_create_guardrail_span`` is called from three lifecycle
+            # points (``async_post_call_success_hook``, ``_handle_success``,
+            # ``_handle_failure``) and re-reads the (mutating) entry list
+            # each time. Dedupe at entry granularity so a single real
+            # guardrail invocation produces exactly one span per handler.
+            if not self._emit_once(
+                kwargs,
+                "guardrail",
+                guardrail_information.get("guardrail_name"),
+                start_time_float,
+                guardrail_information.get("guardrail_mode"),
+            ):
+                continue
+
             start_time_datetime = datetime.now()
             if start_time_float is not None:
                 start_time_datetime = datetime.fromtimestamp(start_time_float)
@@ -1254,6 +1464,21 @@ class OpenTelemetry(CustomLogger):
             kwargs,
             self.config,
         )
+
+        # sync + async failure handlers can both fire for one
+        # request (notably in streaming code paths), producing two
+        # semantically identical ERROR spans. Unlike the success path, the
+        # proxy span is intentionally left open here so that
+        # ``async_post_call_failure_hook`` can append the
+        # "Failed Proxy Server Request" child span before closing it —
+        # there is no proxy-span side-effect to preserve on the skip path.
+        if not self._emit_once(kwargs, "failure"):
+            verbose_logger.debug(
+                "OpenTelemetry: skipping duplicate failure span for handler=%s",
+                self.__class__.__name__,
+            )
+            return
+
         _parent_context, parent_otel_span = self._get_span_context(kwargs)
 
         if self.config.ignore_context_propagation:
@@ -1657,9 +1882,7 @@ class OpenTelemetry(CustomLogger):
             ########## LLM Request Medssages / tools / content Attributes ###########
             #########################################################################
 
-            if litellm.turn_off_message_logging is True:
-                return
-            if self.message_logging is not True:
+            if not self._capture_in_span():
                 return
 
             if optional_params.get("tools"):
@@ -1678,17 +1901,41 @@ class OpenTelemetry(CustomLogger):
                     value=safe_dumps(transformed_messages),
                 )
 
-            if kwargs.get("system_instructions"):
-                transformed_system_instructions = (
-                    self._transform_messages_to_otel_semantic_conventions(
-                        kwargs.get("system_instructions")
+            # Coalesce the different kwarg names that carry the system
+            # prompt depending on the call path:
+            #   - "system_instructions" — Vertex AI Gemini chat-completion
+            #   - "instructions"        — OpenAI Responses API
+            #   - "system"              — Anthropic Messages API
+            # Use `is not None` rather than truthiness to avoid falsy
+            # values (e.g. []) falling through to the wrong kwarg.
+            system_instructions = (
+                kwargs.get("system_instructions")
+                if kwargs.get("system_instructions") is not None
+                else (
+                    kwargs.get("instructions")
+                    if kwargs.get("instructions") is not None
+                    else kwargs.get("system")
+                )
+            )
+            if system_instructions:
+                if isinstance(system_instructions, str):
+                    # Plain text system prompt — no transformation needed
+                    self.safe_set_attribute(
+                        span=span,
+                        key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
+                        value=system_instructions,
+                    )
+                else:
+                    transformed_system_instructions = (
+                        self._transform_messages_to_otel_semantic_conventions(
+                            system_instructions
+                        )
+                    )
+                    self.safe_set_attribute(
+                        span=span,
+                        key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
+                        value=safe_dumps(transformed_system_instructions),
                     )
-                )
-                self.safe_set_attribute(
-                    span=span,
-                    key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
-                    value=safe_dumps(transformed_system_instructions),
-                )
 
             self.safe_set_attribute(
                 span=span,
@@ -1747,6 +1994,57 @@ class OpenTelemetry(CustomLogger):
                                         value=value,
                                     )
 
+                elif response_obj.get("output"):
+                    # Responses API: ResponsesAPIResponse has an "output"
+                    # list instead of "choices".  Each item with
+                    # type="message" contains a "content" list of
+                    # OutputText objects (type="output_text").
+                    output_items = response_obj.get("output")
+                    output_messages = self._transform_responses_api_output_to_otel(
+                        output_items
+                    )
+                    if output_messages:
+                        self.safe_set_attribute(
+                            span=span,
+                            key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value,
+                            value=safe_dumps(output_messages),
+                        )
+
+                    # Emit per-tool-call span attributes (parity with
+                    # the choices branch that calls _tool_calls_kv_pair).
+                    # Convert Responses API function_call items to the
+                    # ChatCompletionMessageToolCall format expected by
+                    # _tool_calls_kv_pair.
+                    tool_calls = []
+                    for out_item in output_items:
+                        item_d = self._to_dict(out_item)
+                        if item_d and item_d.get("type") == "function_call":
+                            tool_calls.append(
+                                {
+                                    "function": {
+                                        "name": item_d.get("name", ""),
+                                        "arguments": item_d.get("arguments", ""),
+                                    }
+                                }
+                            )
+                    if tool_calls:
+                        kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)  # type: ignore
+                        for key, value in kv_pairs.items():
+                            self.safe_set_attribute(
+                                span=span,
+                                key=key,
+                                value=value,
+                            )
+
+                    # Extract finish reason from ResponsesAPIResponse.status
+                    status = response_obj.get("status")
+                    if status:
+                        self.safe_set_attribute(
+                            span=span,
+                            key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value,
+                            value=safe_dumps([status]),
+                        )
+
         except Exception as e:
             self.handle_callback_failure(
                 callback_name=self.callback_name or "opentelemetry"
@@ -1842,6 +2140,78 @@ class OpenTelemetry(CustomLogger):
             transformed.append(transformed_msg)
         return transformed
 
+    @staticmethod
+    def _to_dict(obj) -> Optional[dict]:
+        """Normalize an object to a plain dict.
+
+        Handles three forms that appear in practice:
+
+        1. Plain ``dict`` — returned as-is.
+        2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a
+           ``.get()`` method that delegates to ``__dict__``.
+        3. Raw Pydantic v2 models from the ``openai`` SDK (e.g.
+           ``ResponseOutputMessage``, ``ResponseOutputText``) — these do
+           **not** have ``.get()`` but do have ``.model_dump()``.
+
+        Returns ``None`` for anything else so callers can skip it.
+        """
+        if isinstance(obj, dict):
+            return obj
+        if hasattr(obj, "get"):
+            # BaseLiteLLMOpenAIResponseObject duck-type
+            return obj  # type: ignore[return-value]
+        if hasattr(obj, "model_dump"):
+            # Raw Pydantic v2 model (e.g. openai SDK types)
+            return obj.model_dump()  # type: ignore[union-attr]
+        return None
+
+    def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]:
+        """
+        Transform Responses API output items into OTEL GenAI 1.38 format.
+
+        The Responses API returns output as a list of items, each with a
+        ``type`` field.  Message items (``type="message"``) contain a
+        ``content`` list of ``OutputText`` objects with ``type="output_text"``
+        and ``text`` fields.
+
+        Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``),
+        or raw Pydantic v2 models from the ``openai`` SDK (with
+        ``.model_dump()``).  We normalize each item to a dict via
+        ``_to_dict`` before processing.
+
+        This method converts them to the same ``{"role": ..., "parts": [...]}``
+        format used by ``_transform_choices_to_otel_semantic_conventions``.
+        """
+        transformed = []
+        for raw_item in output:
+            item = self._to_dict(raw_item)
+            if item is None:
+                continue
+            if item.get("type") == "message":
+                role = item.get("role", "assistant")
+                parts = []
+                for raw_content in item.get("content", []):
+                    content = self._to_dict(raw_content)
+                    if content is None:
+                        continue
+                    if content.get("type") == "output_text":
+                        text = content.get("text", "")
+                        if text:
+                            parts.append({"type": "text", "content": text})
+                if parts:
+                    transformed.append({"role": role, "parts": parts})
+            elif item.get("type") == "function_call":
+                # Surface tool calls from Responses API output
+                part: dict = {
+                    "type": "tool_call",
+                    "name": item.get("name", ""),
+                    "arguments": item.get("arguments", ""),
+                }
+                if item.get("call_id"):
+                    part["id"] = item["call_id"]
+                transformed.append({"role": "assistant", "parts": [part]})
+        return transformed
+
     def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
         try:
             # Only set provider-specific raw payload attributes on this span.
@@ -1948,7 +2318,7 @@ class OpenTelemetry(CustomLogger):
             verbose_logger.debug(
                 "OpenTelemetry: Using explicit parent span from metadata"
             )
-            return trace.set_span_in_context(parent_otel_span), parent_otel_span
+            return trace.set_span_in_context(parent_otel_span), None
 
         # Priority 2: HTTP traceparent header
         if traceparent is not None: