Merge branch 'litellm_internal_staging' into fix/credential-list-stale-on-add-model

# Conflicts:
#	ui/litellm-dashboard/src/components/model_add/credentials.tsx

.circleci/config.yml

@@ -226,7 +226,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="xargs uv run --no-sync python -m pytest \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
                 --cov=litellm \
                 --cov-report=xml \
@@ -291,7 +291,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="xargs uv run --no-sync python -m pytest \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
                 --cov=litellm \
                 --cov-report=xml \
@@ -350,7 +350,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse"
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -k \"langfuse\""
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -395,7 +403,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
 
       # Store test results
@@ -433,7 +449,7 @@ jobs:
             echo "$TEST_FILES" | circleci tests run \
               --split-by=timings \
               --verbose \
-              --command="xargs uv run --no-sync python -m pytest \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v \
                 -k 'router' \
                 -n 4 \
@@ -471,7 +487,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -495,7 +519,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -k \"assistants\""
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -528,14 +560,19 @@ jobs:
             # Add --timeout to kill hanging tests after 120s (2 min)
             # Add --durations=20 to show 20 slowest tests for debugging
             # Subdirectories with dedicated jobs (maintain this list as new jobs are added)
-            IGNORE_DIRS=(
-              "tests/llm_translation/realtime"
-            )
-            IGNORE_ARGS=""
-            for dir in "${IGNORE_DIRS[@]}"; do
-              IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
-            done
-            uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
+            mkdir -p test-results
+            # Glob excludes the realtime/ subdirectory since it has its own job
+            TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v \
+                --junitxml=test-results/junit.xml \
+                --durations=20 \
+                -n 4 \
+                --timeout=120 --timeout_method=thread \
+                --retries 2 --retry-delay 5 \
+                --max-worker-restart=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -560,7 +597,17 @@ jobs:
           command: |
             # Add --timeout to kill hanging tests after 120s (2 min)
             # Add --durations=20 to show 20 slowest tests for debugging
-            uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=20 \
+                -n 4 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -593,7 +640,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -626,7 +681,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            export LITELLM_LOG=WARNING
+            TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -660,7 +726,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                --retries 3 --retry-delay 5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -702,7 +777,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 8"
           no_output_timeout: 15m
 
       # Store test results
@@ -725,7 +808,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -758,7 +850,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -793,7 +894,15 @@ jobs:
           name: Run enterprise tests
           command: |
             uv run --no-sync python -m prisma generate
-            uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-enterprise.xml \
+                --durations=10 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -815,7 +924,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -848,7 +966,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 2"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -882,7 +1009,16 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -916,7 +1052,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                -n 4"
           no_output_timeout: 15m
       # Store test results
       - store_test_results:
@@ -939,7 +1083,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread
+            mkdir -p test-results
+            export LITELLM_LOG=WARNING
+            TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv \
+                --cov=litellm --cov-report=xml \
+                -n 4 \
+                --junitxml=test-results/junit.xml \
+                --durations=5 \
+                --timeout=120 --timeout_method=thread"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -972,7 +1127,15 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Rename the coverage files
@@ -1012,14 +1175,19 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv \
+            mkdir -p test-results
+            TEST_FILES=$(printf "%s\n" \
               tests/local_testing/test_dual_cache.py \
               tests/local_testing/test_redis_batch_optimizations.py \
-              tests/local_testing/test_router_utils.py \
-              --cov=litellm --cov-report=xml \
-              -x -s -v --junitxml=test-results/junit.xml \
-              --durations=5 -n 2 \
-              --reruns 2 --reruns-delay 1
+              tests/local_testing/test_router_utils.py)
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --cov=litellm --cov-report=xml \
+                --junitxml=test-results/junit.xml \
+                --durations=5 -n 2 \
+                --reruns 2 --reruns-delay 1"
           no_output_timeout: 20m
       - run:
           name: Rename the coverage files
@@ -1260,8 +1428,17 @@ jobs:
       - run:
           name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion)
           command: |
-            uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
+      - store_test_results:
+          path: test-results
 
   build_and_test:
     machine:
@@ -1331,7 +1508,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
+            mkdir -p test-results
+            # Original used `tests/*.py` (top-level only); the `--ignore=...`
+            # flags were vestigial since shell globbing did not descend into
+            # subdirectories. Replicate by globbing only top-level test files.
+            TEST_FILES=$(circleci tests glob "tests/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -s -v -x \
+                --junitxml=test-results/junit.xml \
+                -n 4 \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1406,7 +1594,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -s -vv \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1475,7 +1670,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
       - run:
@@ -1518,7 +1720,14 @@ jobs:
       - run:
           name: Run second round of tests
           command: |
-            uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1587,8 +1796,17 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
+      - store_test_results:
+          path: test-results
       - run:
           name: Stop and remove first container
           when: always
@@ -1676,7 +1894,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
       # Store test results
@@ -1732,7 +1957,14 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
       - run:
           name: Stop and remove containers
@@ -1805,9 +2037,18 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x \
+                --junitxml=test-results/junit-2.xml \
+                --durations=5"
           no_output_timeout: 15m
             # Clean up first container
+      - store_test_results:
+          path: test-results
       - run:
           name: Stop and remove first container
           command: |
@@ -1935,12 +2176,19 @@ jobs:
           name: Run Vertex AI, Google AI Studio Node.js tests
           command: |
             cd tests/pass_through_tests
-            npx jest . --verbose
+            NODE_OPTIONS=--experimental-vm-modules npx jest . --verbose
           no_output_timeout: 30m
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
+            mkdir -p test-results
+            TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -v -x \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -1997,9 +2245,16 @@ jobs:
       - run:
           name: Run Claude Agent SDK E2E Tests
           command: |
+            mkdir -p test-results
             export LITELLM_PROXY_URL="http://localhost:4000"
             export LITELLM_API_KEY="sk-1234"
-            uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5
+            TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
+            echo "$TEST_FILES" | circleci tests run \
+              --verbose \
+              --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
+                -vv -x -s \
+                --junitxml=test-results/junit.xml \
+                --durations=5"
           no_output_timeout: 15m
 
       # Store test results
@@ -2138,17 +2393,23 @@ jobs:
             - ~/.cache/uv
       - restore_cache:
           keys:
-            - ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
       - run:
           name: Install Node dependencies and Playwright
+          # The cimg/python:3.12-browsers image already ships the Chromium system
+          # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.).
+          # `--with-deps` triggers a redundant apt-get update + install that adds
+          # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors,
+          # so we install just the browser binary.
           command: |
             cd ui/litellm-dashboard
             npm ci
-            npx playwright install chromium --with-deps
+            npx playwright install chromium
       - save_cache:
-          key: ui-e2e-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
           paths:
             - ui/litellm-dashboard/node_modules
+            - ~/.cache/ms-playwright
       - run:
           name: Build UI from source
           # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.

.github/pull_request_template.md

@@ -2,6 +2,10 @@
 
 <!-- e.g. "Fixes #000" -->
 
+## Linear ticket
+
+<!-- if you are an internal contributor, add the Linear ticket e.g. "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
+
 ## Pre-Submission checklist
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**

.github/workflows/create-release-branch.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
         required: true
         type: string
       commit_hash:
@@ -14,7 +14,7 @@ on:
   workflow_call:
     inputs:
       tag:
-        description: "Release tag"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -40,8 +40,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
@@ -45,6 +45,13 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
+            // like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +96,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });

.github/workflows/test-unit-proxy-db.yml

@@ -141,6 +141,7 @@ jobs:
               tests/proxy_unit_tests/test_server_root_path.py
               tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
+              tests/proxy_unit_tests/test_request_size_limit_middleware.py
             workers: 4
             dist: loadscope
             timeout: 15

.gitignore

@@ -90,7 +90,6 @@ test.py
 litellm_config.yaml
 !.github/observatory/litellm_config.yaml
 .cursor
-.vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -100,4 +99,6 @@ STABILIZATION_TODO.md
 **/test-results
 **/playwright-report
 **/*.storageState.json
-**/coverage
+**/coverage
+test-config
+.vscode

.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

Dockerfile

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
+    npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
     GLOBAL="$(npm root -g)" && \
     for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
         name="${pkg##*/}"; \
@@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
 # Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-COPY --from=builder /root/.cache /root/.cache
+# which is /root/.cache here. Copy only the Prisma subdirs — copying the
+# whole /root/.cache drags in the uv build cache (~660 MB, includes a
+# setuptools wheel that surfaces as a CVE finding even though it's not
+# on the runtime sys.path).
+COPY --from=builder /root/.cache/prisma /root/.cache/prisma
+COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

Makefile

@@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
 	$(UV_RUN) pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
 		-v --tb=short --maxfail=100 --timeout=300
+
+test-llm-translation-flush-vcr-cache:
+	$(UV_RUN) python tests/_flush_vcr_cache.py

README.md

@@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
     <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
     <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
     <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
-    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
     <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
     <td><h2>Netflix</h2></td>
     <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>

codecov.yaml

@@ -1,3 +1,8 @@
+codecov:
+  require_ci_to_pass: false # post coverage status even if CI has unrelated failures
+  notify:
+    wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
+
 component_management:
   individual_components:
     - component_id: "Router"
@@ -28,7 +33,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 1% # at maximum allow project coverage to drop by 1%
+        threshold: 0% # do not allow project coverage to drop
     patch:
       default:
         target: auto

cookbook/litellm-ollama-docker-image/requirements.txt

@@ -1 +1 @@
-litellm==1.83.5
+litellm==1.83.14

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -100,6 +100,16 @@ spec:
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
             {{- end }}
+            {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+            - name: DATABASE_URL_READ_REPLICA
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.readReplicaUrlKey }}
+            {{- else if .Values.db.readReplicaUrl }}
+            - name: DATABASE_URL_READ_REPLICA
+              value: {{ .Values.db.readReplicaUrl | quote }}
+            {{- end }}
             - name: PROXY_MASTER_KEY
               valueFrom:
                 secretKeyRef:
@@ -116,21 +126,32 @@ spec:
                   name: {{ include "redis.secretName" .Subcharts.redis }}
                   key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
             {{- end }}
+            {{- /*
+              Inject LITELLM_LOG only when envVars does not already define it.
+            */}}
+            {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
+            - name: LITELLM_LOG
+              value: {{ .Values.logLevel | quote }}
+            {{- end }}
             {{- if .Values.envVars }}
             {{- range $key, $val := .Values.envVars }}
             - name: {{ $key }}
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
-            {{- if .Values.separateHealthApp }}
-            - name: SEPARATE_HEALTH_APP
-              value: "1"
-            - name: SEPARATE_HEALTH_PORT
-              value: {{ .Values.separateHealthPort | default "8081" | quote }}
-            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.migrationJob.enabled }}
+            # Schema updates are owned by the dedicated migrations Job; skip
+            # the proxy's startup `prisma db push` so N replicas don't race
+            # one DB on every rollout. Placed last (after envVars and
+            # extraEnvVars) so this override can't be silently shadowed by a
+            # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
+            # semantics — same pattern the migrations Job uses.
+            - name: DISABLE_SCHEMA_UPDATE
+              value: "true"
+            {{- end }}
           envFrom:
           {{- range .Values.environmentSecrets }}
             - secretRef:
@@ -158,15 +179,10 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
-            {{- if .Values.separateHealthApp }}
-            - name: health
-              containerPort: {{ .Values.separateHealthPort | default 8081 }}
-              protocol: TCP
-            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@@ -175,7 +191,7 @@ spec:
           readinessProbe:
             httpGet:
               path: {{ .Values.readinessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@@ -184,7 +200,7 @@ spec:
           startupProbe:
             httpGet:
               path: {{ .Values.startupProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.startupProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -257,16 +257,16 @@ tests:
           value: 0
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.periodSeconds
-          value: 10
+          value: 15
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.successThreshold
           value: 1
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.failureThreshold
-          value: 3
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.httpGet.path
           value: /health/readiness
@@ -278,7 +278,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.successThreshold
           value: 1
@@ -296,7 +296,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].startupProbe.successThreshold
           value: 1

deploy/charts/litellm-helm/values.yaml

@@ -88,26 +88,20 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
-# Separate health app configuration
-# When enabled, health checks will use a separate port and the application
-# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
-separateHealthApp: false
-separateHealthPort: 8081
-
-# Probe tuning for proxy container
+# Probes for LiteLLM gateway container
 livenessProbe:
   path: /health/liveliness
   initialDelaySeconds: 0
-  periodSeconds: 10
-  timeoutSeconds: 1
+  periodSeconds: 15
+  timeoutSeconds: 5
   successThreshold: 1
-  failureThreshold: 3
+  failureThreshold: 5
 
 readinessProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 3
 
@@ -115,7 +109,7 @@ startupProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 30
 
@@ -258,6 +252,26 @@ db:
     passwordKey: password
     # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
     endpointKey: ""
+    # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
+    # secret key instead of db.readReplicaUrl. Prefer this over the plain
+    # value: read-replica URLs typically embed credentials, and a value
+    # written to db.readReplicaUrl ends up visible in the rendered pod spec
+    # and the Helm release secret.
+    readReplicaUrlKey: ""
+
+  # Optional read-replica routing. When set, the proxy sends read-only
+  # queries (find_*, count, group_by, query_raw/_first) to this URL while
+  # writes continue to go to db.url. Useful for Aurora-style clusters with
+  # separate reader/writer endpoints. Leave empty to keep single-DB behavior.
+  # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
+  # alongside the writer (host/port/user/db are parsed from this URL once
+  # at startup; only the IAM token rotates).
+  #
+  # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
+  # this field — the plain value is rendered into the pod spec and the
+  # Helm release secret. This field is intended for credential-less URLs
+  # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
+  readReplicaUrl: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
@@ -329,6 +343,17 @@ migrationJob:
     helm:
       enabled: false
 
+# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
+# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
+# any `envFrom:` source. If you currently source LITELLM_LOG from an
+# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
+# disable injection — otherwise this value silently overrides your secret /
+# configmap entry.
+#
+# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
+# this injection entirely when envVars already defines LITELLM_LOG.
+logLevel: INFO
+
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {}
 

docker-compose.yml

@@ -16,6 +16,11 @@ services:
       - "4000:4000" # Map the container port to the host, change the host port if necessary
     environment:
       DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      # Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
+      # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
+      # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
+      # is auto-refreshed alongside the writer.
+      # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
       STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
     env_file:
       - .env # Load local .env file

docker/Dockerfile.alpine

@@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b
 
 # Runtime image
 ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 

docker/Dockerfile.database

@@ -1,9 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
     npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/Dockerfile.dev

@@ -3,7 +3,7 @@ ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973a
 
 # Runtime image
 ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 

docker/Dockerfile.health_check

@@ -1,4 +1,4 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 FROM $UV_IMAGE AS uvbin
 
 FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:f26d42a15d09d9a643b231df929fa3cf609bedc58a728eb445be89a9d8d1da9f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 ARG PROXY_EXTRAS_SOURCE=published
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:733b4042187702f832f7fdecb3aff14a61b288c4ca37af188bb5715c1caebaf8
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
 
@@ -32,7 +32,6 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
     XDG_CACHE_HOME=/app/.cache
 
 # Copy dependency metadata first for layer caching
@@ -104,17 +103,15 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
-COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
 
 ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
     HOME=/app \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache \

docker/prod_entrypoint.sh

@@ -1,14 +1,8 @@
 #!/bin/sh
 
-if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
-    export LITELLM_ARGS="$@"
-    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
-    exec supervisord -c /etc/supervisord.conf
-fi
-
 if [ "$USE_DDTRACE" = "true" ]; then
     export DD_TRACE_OPENAI_ENABLED="False"
     exec ddtrace-run litellm "$@"
 else
     exec litellm "$@"
-fi
+fi

docker/supervisord.conf

@@ -1,46 +0,0 @@
-[supervisord]
-nodaemon=true
-loglevel=info
-logfile=/tmp/supervisord.log
-pidfile=/tmp/supervisord.pid
-
-[group:litellm]
-programs=main,health
-
-[program:main]
-command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
-autostart=true
-autorestart=true
-startretries=3
-priority=1
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[program:health]
-command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
-autostart=true
-autorestart=true
-startretries=3
-priority=2
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[eventlistener:process_monitor]
-command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
-events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
-autostart=true
-autorestart=true

docs/my-website/docs/providers/crusoe.md

@@ -0,0 +1,196 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Crusoe
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
+| Provider Route on LiteLLM | `crusoe/` |
+| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
+| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+<br />
+<br />
+
+**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
+
+## Available Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
+| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
+| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
+| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
+| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
+| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
+| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Crusoe Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Crusoe call
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Crusoe Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Write a short story about AI", "role": "user"}]
+
+# Crusoe call with streaming
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+### Function Calling
+
+```python showLineNumbers title="Crusoe Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the current weather in a location",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state, e.g. San Francisco, CA"
+                }
+            },
+            "required": ["location"]
+        }
+    }
+}]
+
+messages = [{"role": "user", "content": "What's the weather in Boston?"}]
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy Server
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: llama-3.3-70b
+    litellm_params:
+      model: crusoe/meta-llama/Llama-3.3-70B-Instruct
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-r1
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-R1-0528
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-v3
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-V3-0324
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: qwen3-235b
+    litellm_params:
+      model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: kimi-k2
+    litellm_params:
+      model: crusoe/moonshotai/Kimi-K2-Thinking
+      api_key: os.environ/CRUSOE_API_KEY
+```
+
+## Custom API Base
+
+**Option 1: Environment variable**
+
+```python showLineNumbers title="Custom API Base via env var"
+import os
+from litellm import completion
+
+os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
+os.environ["CRUSOE_API_KEY"] = ""  # your API key
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+)
+```
+
+**Option 2: Pass directly**
+
+```python showLineNumbers title="Custom API Base via parameter"
+from litellm import completion
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+    api_base="https://custom.crusoecloud.com/v1",
+    api_key="your-api-key",
+)
+```
+
+## Supported OpenAI Parameters
+
+- `temperature`
+- `max_tokens`
+- `max_completion_tokens`
+- `top_p`
+- `frequency_penalty`
+- `presence_penalty`
+- `stop`
+- `n`
+- `stream`
+- `tools`
+- `tool_choice`
+- `response_format`
+- `seed`
+- `user`
+- `logit_bias`
+- `logprobs`
+- `top_logprobs`

enterprise/enterprise_hooks/banned_keywords.py

@@ -11,6 +11,10 @@ from typing import Literal
 import litellm
 from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import (
+    is_text_content_call_type,
+    iter_message_text,
+)
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger
 from fastapi import HTTPException
@@ -73,10 +77,9 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
             - check if user id part of blocked list
             """
             self.print_verbose("Inside Banned Keyword List Pre-Call Hook")
-            if call_type == "completion" and "messages" in data:
-                for m in data["messages"]:
-                    if "content" in m and isinstance(m["content"], str):
-                        self.test_violation(test_str=m["content"])
+            if is_text_content_call_type(call_type):
+                for text in iter_message_text(data):
+                    self.test_violation(test_str=text)
 
         except HTTPException as e:
             raise e
@@ -93,11 +96,16 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
         user_api_key_dict: UserAPIKeyAuth,
         response,
     ):
-        if isinstance(response, litellm.ModelResponse) and isinstance(
-            response.choices[0], litellm.utils.Choices
-        ):
-            for word in self.banned_keywords_list:
-                self.test_violation(test_str=response.choices[0].message.content or "")
+        if not isinstance(response, litellm.ModelResponse):
+            return
+
+        for choice in response.choices:
+            if not isinstance(choice, litellm.utils.Choices):
+                continue
+            message = getattr(choice, "message", None)
+            content = getattr(message, "content", None)
+            if isinstance(content, str):
+                self.test_violation(test_str=content)
 
     async def async_post_call_streaming_hook(
         self,

enterprise/enterprise_hooks/google_text_moderation.py

@@ -12,6 +12,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
@@ -94,11 +95,9 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
         - Calls Google's Text Moderation API
         - Rejects request if it fails safety check
         """
-        if "messages" in data and isinstance(data["messages"], list):
-            text = ""
-            for m in data["messages"]:  # assume messages is a list
-                if "content" in m and isinstance(m["content"], str):
-                    text += m["content"]
+        # Covers multimodal list content + Responses-API input.
+        text = "".join(iter_message_text(data))
+        if text:
             document = self.language_document(content=text, type_=self.document_type)
 
             request = self.moderate_text_request(

enterprise/enterprise_hooks/openai_moderation.py

@@ -19,6 +19,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import iter_message_text
 from litellm.types.utils import CallTypesLiteral
 
 
@@ -37,11 +38,8 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
     ):
-        text = ""
-        if "messages" in data and isinstance(data["messages"], list):
-            for m in data["messages"]:  # assume messages is a list
-                if "content" in m and isinstance(m["content"], str):
-                    text += m["content"]
+        # Covers multimodal list content + Responses-API input.
+        text = "".join(iter_message_text(data))
 
         from litellm.proxy.proxy_server import llm_router
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -18,6 +18,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails._content_utils import walk_user_text
 
 GUARDRAIL_NAME = "hide_secrets"
 
@@ -473,23 +474,19 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         if await self.should_run_check(user_api_key_dict) is False:
             return
 
-        if "messages" in data and isinstance(data["messages"], list):
-            for message in data["messages"]:
-                if "content" in message and isinstance(message["content"], str):
-                    detected_secrets = self.scan_message_for_secrets(message["content"])
+        # Covers multimodal list content + Responses-API input.
+        def _redact_message_text(text: str) -> str:
+            detected_secrets = self.scan_message_for_secrets(text)
+            for secret in detected_secrets:
+                text = text.replace(secret["value"], "[REDACTED]")
+            if detected_secrets:
+                secret_types = [secret["type"] for secret in detected_secrets]
+                verbose_proxy_logger.warning(
+                    f"Detected and redacted secrets in message: {secret_types}"
+                )
+            return text
 
-                    for secret in detected_secrets:
-                        message["content"] = message["content"].replace(
-                            secret["value"], "[REDACTED]"
-                        )
-
-                    if len(detected_secrets) > 0:
-                        secret_types = [secret["type"] for secret in detected_secrets]
-                        verbose_proxy_logger.warning(
-                            f"Detected and redacted secrets in message: {secret_types}"
-                        )
-                    else:
-                        verbose_proxy_logger.debug("No secrets detected on input.")
+        walk_user_text(data, _redact_message_text)
 
         if "prompt" in data:
             if isinstance(data["prompt"], str):
@@ -504,11 +501,15 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
                         f"Detected and redacted secrets in prompt: {secret_types}"
                     )
             elif isinstance(data["prompt"], list):
-                for item in data["prompt"]:
+                # Index back into the list — assigning to ``item`` would only
+                # rebind the loop variable and leave ``data["prompt"]``
+                # carrying the unredacted secret.
+                for idx, item in enumerate(data["prompt"]):
                     if isinstance(item, str):
                         detected_secrets = self.scan_message_for_secrets(item)
                         for secret in detected_secrets:
                             item = item.replace(secret["value"], "[REDACTED]")
+                        data["prompt"][idx] = item
                         if len(detected_secrets) > 0:
                             secret_types = [
                                 secret["type"] for secret in detected_secrets
@@ -517,31 +518,6 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
                                 f"Detected and redacted secrets in prompt: {secret_types}"
                             )
 
-        if "input" in data:
-            if isinstance(data["input"], str):
-                detected_secrets = self.scan_message_for_secrets(data["input"])
-                for secret in detected_secrets:
-                    data["input"] = data["input"].replace(secret["value"], "[REDACTED]")
-                if len(detected_secrets) > 0:
-                    secret_types = [secret["type"] for secret in detected_secrets]
-                    verbose_proxy_logger.warning(
-                        f"Detected and redacted secrets in input: {secret_types}"
-                    )
-            elif isinstance(data["input"], list):
-                _input_in_request = data["input"]
-                for idx, item in enumerate(_input_in_request):
-                    if isinstance(item, str):
-                        detected_secrets = self.scan_message_for_secrets(item)
-                        for secret in detected_secrets:
-                            _input_in_request[idx] = item.replace(
-                                secret["value"], "[REDACTED]"
-                            )
-                        if len(detected_secrets) > 0:
-                            secret_types = [
-                                secret["type"] for secret in detected_secrets
-                            ]
-                            verbose_proxy_logger.warning(
-                                f"Detected and redacted secrets in input: {secret_types}"
-                            )
-                verbose_proxy_logger.debug("Data after redacting input %s", data)
+        # ``data["input"]`` (Responses API and embeddings/moderation) is
+        # already covered by ``walk_user_text`` above.
         return

enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py

@@ -10,28 +10,21 @@ has already authenticated the user) and you need to extract user information fro
 custom headers or other request attributes.
 """
 
-from typing import TYPE_CHECKING, Dict, Optional, Union, cast
+from typing import cast
 
 from fastapi import Request
 from fastapi.responses import RedirectResponse
 
-if TYPE_CHECKING:
-    from fastapi_sso.sso.base import OpenID
-else:
-    from typing import Any as OpenID
-
-from litellm.proxy.management_endpoints.types import CustomOpenID
-
 
 class EnterpriseCustomSSOHandler:
     """
     Enterprise Custom SSO Handler for LiteLLM Proxy
-    
+
     This class provides methods for handling custom SSO authentication flows
     where users can implement their own authentication logic by processing
     request headers and returning user information in OpenID format.
     """
-    
+
     @staticmethod
     async def handle_custom_ui_sso_sign_in(
         request: Request,
@@ -40,16 +33,16 @@ class EnterpriseCustomSSOHandler:
         Allow a user to execute their custom code to parse incoming request headers and return a OpenID object
 
         Use this when you have an OAuth proxy in front of LiteLLM (where the OAuth proxy has already authenticated the user)
-        
+
         Args:
             request: The FastAPI request object containing headers and other request data
-            
+
         Returns:
             RedirectResponse: Redirect response that sends the user to the LiteLLM UI with authentication token
-            
+
         Raises:
             ValueError: If custom_ui_sso_sign_in_handler is not configured
-            
+
         Example:
             This method is typically called when a user has already been authenticated by an
             external OAuth proxy and the proxy has added custom headers containing user information.
@@ -60,27 +53,44 @@ class EnterpriseCustomSSOHandler:
         from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
         from litellm.proxy.proxy_server import (
             CommonProxyErrors,
+            general_settings,
             premium_user,
             user_custom_ui_sso_sign_in_handler,
         )
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
+
         if premium_user is not True:
             raise ValueError(CommonProxyErrors.not_premium_user.value)
-        
+
         if user_custom_ui_sso_sign_in_handler is None:
-            raise ValueError("custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings.")
-        
-        custom_sso_login_handler = cast(CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler)
-        openid_response: OpenID = await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
+            raise ValueError(
+                "custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings."
+            )
+
+        require_trusted_proxy_request(
             request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
         )
-        
+
+        custom_sso_login_handler = cast(
+            CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler
+        )
+        openid_response: OpenID = (
+            await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
+                request=request,
+            )
+        )
+
         # Import here to avoid circular imports
         from litellm.proxy.management_endpoints.ui_sso import SSOAuthenticationHandler
-        
+
         return await SSOAuthenticationHandler.get_redirect_response_from_openid(
             result=openid_response,
             request=request,
             received_response=None,
             generic_client_id=None,
             ui_access_mode=None,
-        ) 
+        )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -15,6 +15,11 @@ from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.managed_resources.isolation import (
+    build_list_page,
+    build_owner_filter,
+    can_access_resource,
+)
 from litellm.proxy._types import (
     CallTypes,
     LiteLLM_ManagedFileTable,
@@ -99,6 +104,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 model_mappings=model_mappings,
                 flat_model_file_ids=list(model_mappings.values()),
                 created_by=user_api_key_dict.user_id,
+                team_id=user_api_key_dict.team_id,
                 updated_by=user_api_key_dict.user_id,
             )
             await self.internal_usage_cache.async_set_cache(
@@ -114,6 +120,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             "model_mappings": json.dumps(model_mappings),
             "flat_model_file_ids": list(model_mappings.values()),
             "created_by": user_api_key_dict.user_id,
+            "team_id": user_api_key_dict.team_id,
             "updated_by": user_api_key_dict.user_id,
         }
 
@@ -125,7 +132,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 db_data["storage_backend"] = hidden_params["storage_backend"]
             if "storage_url" in hidden_params:
                 db_data["storage_url"] = hidden_params["storage_url"]
-            
+
             verbose_logger.debug(
                 f"Storage metadata: storage_backend={db_data.get('storage_backend')}, "
                 f"storage_url={db_data.get('storage_url')}"
@@ -171,6 +178,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "model_object_id": model_object_id,
                     "file_purpose": file_purpose,
                     "created_by": user_api_key_dict.user_id,
+                    "team_id": user_api_key_dict.team_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
                 },
@@ -229,15 +237,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     async def can_user_call_unified_file_id(
         self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth
     ) -> bool:
-        ## check if the user has access to the unified file id
-
-        user_id = user_api_key_dict.user_id
         managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
             where={"unified_file_id": unified_file_id}
         )
 
         if managed_file:
-            return managed_file.created_by == user_id
+            return can_access_resource(
+                user_api_key_dict=user_api_key_dict,
+                created_by=managed_file.created_by,
+                resource_team_id=managed_file.team_id,
+            )
         raise HTTPException(
             status_code=404,
             detail=f"File not found: {unified_file_id}",
@@ -246,8 +255,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     async def can_user_call_unified_object_id(
         self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth
     ) -> bool:
-        ## check if the user has access to the unified object id
-        user_id = user_api_key_dict.user_id
         managed_object = (
             await self.prisma_client.db.litellm_managedobjecttable.find_first(
                 where={"unified_object_id": unified_object_id}
@@ -255,7 +262,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
 
         if managed_object:
-            return managed_object.created_by == user_id
+            return can_access_resource(
+                user_api_key_dict=user_api_key_dict,
+                created_by=managed_object.created_by,
+                resource_team_id=managed_object.team_id,
+            )
         raise HTTPException(
             status_code=404,
             detail=f"Object not found: {unified_object_id}",
@@ -285,28 +296,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             raise Exception(
                 "Filtering by 'target_model_names' is not supported when using managed batches."
             )
-        
-        where_clause: Dict[str, Any] = {"file_purpose": "batch"}
-        
-        # Filter by user who created the batch
-        if user_api_key_dict.user_id:
-            where_clause["created_by"] = user_api_key_dict.user_id
-        
+
+        owner_filter = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return build_list_page([])
+
+        where_clause: Dict[str, Any] = {"file_purpose": "batch", **owner_filter}
+
         if after:
             where_clause["id"] = {"gt": after}
-        
-        # Fetch more than needed to allow for post-fetch filtering
+
         fetch_limit = limit or 20
         if target_model_names:
-            # Fetch extra to account for filtering
+            # Oversample so post-fetch model-name filtering still has enough rows.
             fetch_limit = max(fetch_limit * 3, 100)
-        
+
         batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
             where=where_clause,
             take=fetch_limit,
             order={"created_at": "desc"},
         )
-                
+
         batch_objects: List[LiteLLMBatch] = []
         for batch in batches:
             try:
@@ -314,7 +324,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 if len(batch_objects) >= (limit or 20):
                     break
 
-                batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
+                batch_data = (
+                    json.loads(batch.file_object)
+                    if isinstance(batch.file_object, str)
+                    else batch.file_object
+                )
                 batch_obj = LiteLLMBatch(**batch_data)
                 batch_obj.id = batch.unified_object_id
                 batch_objects.append(batch_obj)
@@ -324,27 +338,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     f"Failed to parse batch object {batch.unified_object_id}: {e}"
                 )
                 continue
-        
-        return {
-            "object": "list",
-            "data": batch_objects,
-            "first_id": batch_objects[0].id if batch_objects else None,
-            "last_id": batch_objects[-1].id if batch_objects else None,
-            "has_more": len(batch_objects) == (limit or 20),
-        }
+
+        return build_list_page(
+            batch_objects, has_more=len(batch_objects) == (limit or 20)
+        )
 
     async def get_user_created_file_ids(
         self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
     ) -> List[OpenAIFileObject]:
         """
-        Get all file ids created by the user for a list of model object ids
+        Get all file ids the caller is allowed to see for a list of model
+        object ids. Service-account keys (no user_id) are scoped to their
+        team via ``team_id``; admins see all matches.
 
         Returns:
          - List of OpenAIFileObject's
         """
+        owner_filter = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return []
+
         file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many(
             where={
-                "created_by": user_api_key_dict.user_id,
+                **owner_filter,
                 "flat_model_file_ids": {"hasSome": model_object_ids},
             }
         )
@@ -377,11 +393,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         """
         Check if the user has access to a list of file IDs.
         Only checks managed (unified) file IDs.
-        
+
         Args:
             file_ids: List of file IDs to check access for
             user_api_key_dict: User API key authentication details
-            
+
         Raises:
             HTTPException: If user doesn't have access to any of the files
         """
@@ -419,10 +435,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         ### HANDLE TRANSFORMATIONS ###
         # Check both completion and acompletion call types
         is_completion_call = (
-            call_type == CallTypes.completion.value 
+            call_type == CallTypes.completion.value
             or call_type == CallTypes.acompletion.value
         )
-        
+
         if is_completion_call:
             messages = data.get("messages")
             model = data.get("model", "")
@@ -431,22 +447,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 if file_ids:
                     # Check user has access to all managed files
                     await self.check_file_ids_access(file_ids, user_api_key_dict)
-                    
+
                     # Check if any files are stored in storage backends and need base64 conversion
                     # This is needed for Vertex AI/Gemini which requires base64 content
-                    is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower())
+                    is_vertex_ai = model and (
+                        "vertex_ai" in model or "gemini" in model.lower()
+                    )
                     if is_vertex_ai:
                         await self._convert_storage_files_to_base64(
                             messages=messages,
                             file_ids=file_ids,
                             litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
                         )
-                    
+
                     model_file_id_mapping = await self.get_model_file_id_mapping(
                         file_ids, user_api_key_dict.parent_otel_span
                     )
                     data["model_file_id_mapping"] = model_file_id_mapping
-        elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
+        elif (
+            call_type == CallTypes.aresponses.value
+            or call_type == CallTypes.responses.value
+        ):
             # Handle managed files in responses API input and tools
             file_ids = []
 
@@ -611,7 +632,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             if model_id is None:
                 model_id = cast(
                     Optional[str],
-                    kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
+                    kwargs.get("litellm_metadata", {})
+                    .get("model_info", {})
+                    .get("id", None),
                 )
             mapped_file_id: Optional[str] = None
             if input_file_id and model_file_id_mapping and model_id:
@@ -648,7 +671,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     ) -> List[str]:
         """
         Gets file ids from responses API input.
-        
+
         The input can be:
         - A string (no files)
         - A list of input items, where each item can have:
@@ -656,32 +679,35 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
           - content: a list that can contain items with type: "input_file" and file_id
         """
         file_ids: List[str] = []
-        
+
         if isinstance(input, str):
             return file_ids
-        
+
         if not isinstance(input, list):
             return file_ids
-        
+
         for item in input:
             if not isinstance(item, dict):
                 continue
-            
+
             # Check for direct input_file type
             if item.get("type") == "input_file":
                 file_id = item.get("file_id")
                 if file_id:
                     file_ids.append(file_id)
-            
+
             # Check for input_file in content array
             content = item.get("content")
             if isinstance(content, list):
                 for content_item in content:
-                    if isinstance(content_item, dict) and content_item.get("type") == "input_file":
+                    if (
+                        isinstance(content_item, dict)
+                        and content_item.get("type") == "input_file"
+                    ):
                         file_id = content_item.get("file_id")
                         if file_id:
                             file_ids.append(file_id)
-        
+
         return file_ids
 
     def get_file_ids_from_responses_tools(
@@ -689,7 +715,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     ) -> List[str]:
         """
         Gets file ids from responses API tools parameter.
-        
+
         The tools can contain code_interpreter with container.file_ids:
         [
             {
@@ -699,14 +725,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         ]
         """
         file_ids: List[str] = []
-        
+
         if not isinstance(tools, list):
             return file_ids
-        
+
         for tool in tools:
             if not isinstance(tool, dict):
                 continue
-            
+
             # Check for code_interpreter with container file_ids
             if tool.get("type") == "code_interpreter":
                 container = tool.get("container")
@@ -716,7 +742,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         for file_id in container_file_ids:
                             if isinstance(file_id, str):
                                 file_ids.append(file_id)
-        
+
         return file_ids
 
     def get_vector_store_ids_from_file_search_tools(
@@ -916,10 +942,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # Emit Prometheus metrics for managed file creation
         prom_logger = self._get_prometheus_logger()
         if prom_logger:
-            first_model = target_model_names_list[0] if target_model_names_list else None
+            first_model = (
+                target_model_names_list[0] if target_model_names_list else None
+            )
             first_provider = ""
             if responses:
-                first_provider = getattr(responses[0], "_hidden_params", {}).get("custom_llm_provider") or ""
+                first_provider = (
+                    getattr(responses[0], "_hidden_params", {}).get(
+                        "custom_llm_provider"
+                    )
+                    or ""
+                )
             prom_logger.record_managed_file_created(
                 model=first_model or "",
                 api_provider=first_provider,
@@ -1073,16 +1106,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                             model_name=resolved_model_name,
                         )
                         setattr(response, file_attr, unified_file_id)
-                        
+
                         # Use llm_router credentials when available. Without credentials,
                         # Azure and other auth-required providers return 500/401.
                         file_object = None
                         try:
                             # Import module and use getattr for better testability with mocks
                             import litellm.proxy.proxy_server as proxy_server_module
-                            _llm_router = getattr(proxy_server_module, 'llm_router', None)
+
+                            _llm_router = getattr(
+                                proxy_server_module, "llm_router", None
+                            )
                             if _llm_router is not None and model_id:
-                                _creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
+                                _creds = (
+                                    _llm_router.get_deployment_credentials_with_provider(
+                                        model_id
+                                    )
+                                    or {}
+                                )
                                 file_object = await litellm.afile_retrieve(
                                     file_id=original_file_id,
                                     **_creds,
@@ -1099,7 +1140,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                             verbose_logger.warning(
                                 f"Failed to retrieve file object for {file_attr}={original_file_id}: {str(e)}. Storing with None and will fetch on-demand."
                             )
-                        
+
                         await self.store_unified_file_id(
                             file_id=unified_file_id,
                             file_object=file_object,
@@ -1128,6 +1169,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                             from litellm.litellm_core_utils.get_llm_provider_logic import (
                                 get_llm_provider,
                             )
+
                             _, batch_provider, _, _ = get_llm_provider(model=model_name)
                         except Exception:
                             if "/" in model_name:
@@ -1199,7 +1241,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # Case 1 : This is not a managed file
         if not stored_file_object:
             raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
-        
+
         # Case 2: Managed file and the file object exists in the database
         # The stored file_object has the raw provider ID. Replace with the unified ID
         # so callers see a consistent ID (matching Case 3 which does response.id = file_id).
@@ -1217,13 +1259,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )
 
         try:
-            model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
-            credentials = llm_router.get_deployment_credentials_with_provider(model_id) or {}
-            response = await litellm.afile_retrieve(file_id=model_file_id, **credentials)
+            model_id, model_file_id = next(
+                iter(stored_file_object.model_mappings.items())
+            )
+            credentials = (
+                llm_router.get_deployment_credentials_with_provider(model_id) or {}
+            )
+            response = await litellm.afile_retrieve(
+                file_id=model_file_id, **credentials
+            )
             response.id = file_id  # Replace with unified ID
             return response
         except Exception as e:
-            raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e
+            raise Exception(
+                f"Failed to retrieve file {file_id} from provider: {str(e)}"
+            ) from e
 
     async def afile_list(
         self,
@@ -1245,19 +1295,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             import litellm.proxy.proxy_server as proxy_server_module
 
             # Check if the scheduler has the batch cost checking job registered
-            scheduler = getattr(proxy_server_module, 'scheduler', None)
+            scheduler = getattr(proxy_server_module, "scheduler", None)
             if scheduler is None:
                 return False
-            
+
             # Check if the check_batch_cost_job exists in the scheduler
             try:
-                job = scheduler.get_job('check_batch_cost_job')
+                job = scheduler.get_job("check_batch_cost_job")
                 if job is not None:
                     return True
             except Exception:
                 # Job not found or scheduler doesn't support get_job
                 pass
-            
+
             return False
         except Exception as e:
             verbose_logger.warning(
@@ -1265,28 +1315,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             )
             return False
 
-    async def _get_batches_referencing_file(
-        self, file_id: str
-    ) -> List[Dict[str, Any]]:
+    async def _get_batches_referencing_file(self, file_id: str) -> List[Dict[str, Any]]:
         """
         Find batches that reference this file and still need cost tracking.
         Find batches that are in non-terminal state and have not yet been processed by CheckBatchCost.
         Args:
             file_id: The unified file ID to check
-            
+
         Returns:
             List of batch objects referencing this file in non-terminal state
             (max 10 for error message display)
         """
         # Prepare list of file IDs to check (both unified and provider IDs)
         file_ids_to_check = [file_id]
-        
+
         # Get model-specific file IDs for this unified file ID if it's a managed file
         try:
             model_file_id_mapping = await self.get_model_file_id_mapping(
                 [file_id], litellm_parent_otel_span=None
             )
-            
+
             if model_file_id_mapping and file_id in model_file_id_mapping:
                 # Add all provider file IDs for this unified file
                 provider_file_ids = list(model_file_id_mapping[file_id].values())
@@ -1296,59 +1344,67 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 f"Could not get model file ID mapping for {file_id}: {e}. "
                 f"Will only check unified file ID."
             )
-        MAX_MATCHES_TO_RETURN = 10 
-        
+        MAX_MATCHES_TO_RETURN = 10
+
         batches = await self.prisma_client.db.litellm_managedobjecttable.find_many(
             where={
                 "file_purpose": "batch",
                 "batch_processed": False,
-                "status": {"not_in": ["failed", "expired", "cancelled"]}
+                "status": {"not_in": ["failed", "expired", "cancelled"]},
             },
             take=MAX_MATCHES_TO_RETURN,
             order={"created_at": "desc"},
         )
-        
+
         referencing_batches = []
         for batch in batches:
             try:
                 # Parse the batch file_object to check for file references
-                batch_data = json.loads(batch.file_object) if isinstance(batch.file_object, str) else batch.file_object
-                
+                batch_data = (
+                    json.loads(batch.file_object)
+                    if isinstance(batch.file_object, str)
+                    else batch.file_object
+                )
+
                 # Extract file IDs from batch
                 # Batches typically reference the unified file ID in input_file_id
                 # Output and error files are generated by the provider
                 input_file_id = batch_data.get("input_file_id")
                 output_file_id = batch_data.get("output_file_id")
                 error_file_id = batch_data.get("error_file_id")
-                
-                referenced_file_ids = [fid for fid in [input_file_id, output_file_id, error_file_id] if fid]
-                
+
+                referenced_file_ids = [
+                    fid for fid in [input_file_id, output_file_id, error_file_id] if fid
+                ]
+
                 # Check if any referenced file ID matches the file we're trying to delete
                 if any(ref_id in file_ids_to_check for ref_id in referenced_file_ids):
-                    referencing_batches.append({
-                        "batch_id": batch.unified_object_id,
-                        "status": batch.status,
-                        "created_at": batch.created_at,
-                    })
+                    referencing_batches.append(
+                        {
+                            "batch_id": batch.unified_object_id,
+                            "status": batch.status,
+                            "created_at": batch.created_at,
+                        }
+                    )
             except Exception as e:
                 verbose_logger.warning(
                     f"Error parsing batch object {batch.unified_object_id}: {e}"
                 )
                 continue
-        
+
         return referencing_batches
 
     async def _check_file_deletion_allowed(self, file_id: str) -> None:
         """
         Check if file deletion should be blocked due to batch references.
-        
+
         Blocks deletion if:
         1. File is referenced by any batch in non-terminal state, AND
         2. Batch polling is configured (user wants cost tracking)
-        
+
         Args:
             file_id: The unified file ID to check
-            
+
         Raises:
             HTTPException: If file deletion should be blocked
         """
@@ -1356,39 +1412,45 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if not self._is_batch_polling_enabled():
             # Batch polling not configured, allow deletion
             return
-        
+
         # Check if file is referenced by any non-terminal batches
         referencing_batches = await self._get_batches_referencing_file(file_id)
-        
+
         if referencing_batches:
             # File is referenced by non-terminal batches and polling is enabled
-            MAX_BATCHES_IN_ERROR = 5  # Limit batches shown in error message for readability
-            
+            MAX_BATCHES_IN_ERROR = (
+                5  # Limit batches shown in error message for readability
+            )
+
             # Show up to MAX_BATCHES_IN_ERROR in the error message
             batches_to_show = referencing_batches[:MAX_BATCHES_IN_ERROR]
-            batch_statuses = [f"{b['batch_id']}: {b['status']}" for b in batches_to_show]
-            
+            batch_statuses = [
+                f"{b['batch_id']}: {b['status']}" for b in batches_to_show
+            ]
+
             # Determine the count message
             count_message = f"{len(referencing_batches)}"
-            if len(referencing_batches) >= 10:  # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
+            if (
+                len(referencing_batches) >= 10
+            ):  # MAX_MATCHES_TO_RETURN from _get_batches_referencing_file
                 count_message = "10+"
-            
+
             error_message = (
                 f"Cannot delete file {file_id}. "
                 f"The file is referenced by {count_message} batch(es) in non-terminal state"
             )
-            
+
             # Add specific batch details if not too many
             if len(referencing_batches) <= MAX_BATCHES_IN_ERROR:
                 error_message += f": {', '.join(batch_statuses)}. "
             else:
                 error_message += f" (showing {MAX_BATCHES_IN_ERROR} most recent): {', '.join(batch_statuses)}. "
-            
+
             error_message += (
                 f"To delete this file before complete cost tracking, please delete or cancel the referencing batch(es) first. "
                 f"Alternatively, wait for all batches to complete and for cost to be computed (batch_processed=true)."
             )
-            
+
             # Record blocked deletion metric
             prom_logger = self._get_prometheus_logger()
             if prom_logger:
@@ -1419,7 +1481,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
         if specific_model_file_id_mapping:
             # Remove conflicting keys from data to avoid duplicate keyword arguments
-            filtered_data = {k: v for k, v in data.items() if k not in ("model", "file_id")}
+            filtered_data = {
+                k: v for k, v in data.items() if k not in ("model", "file_id")
+            }
             for model_id, model_file_id in specific_model_file_id_mapping.items():
                 delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **filtered_data)  # type: ignore
 
@@ -1480,7 +1544,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     ) -> None:
         """
         Convert files stored in storage backends to base64 format for Vertex AI/Gemini.
-        
+
         This method checks if any managed files are stored in storage backends,
         downloads them, and converts them to base64 format in the messages.
         """
@@ -1488,29 +1552,29 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         for file_id in file_ids:
             # Check if this is a base64 encoded unified file ID
             decoded_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
-            
+
             if not decoded_unified_file_id:
                 continue
-            
+
             # Check database for storage backend info
             # IMPORTANT: The database stores the base64 encoded unified_file_id (not the decoded version)
             # So we query with the original file_id (which is base64 encoded)
             db_file = await self.prisma_client.db.litellm_managedfiletable.find_first(
                 where={"unified_file_id": file_id}
             )
-            
+
             if not db_file or not db_file.storage_backend or not db_file.storage_url:
                 continue
-            
+
             # File is stored in a storage backend, download and convert to base64
             try:
                 from litellm.llms.base_llm.files.storage_backend_factory import (
                     get_storage_backend,
                 )
-                
+
                 storage_backend_name = db_file.storage_backend
                 storage_url = db_file.storage_url
-                
+
                 # Get storage backend (uses same env vars as callback)
                 try:
                     storage_backend = get_storage_backend(storage_backend_name)
@@ -1519,18 +1583,22 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}"
                     )
                     continue
-                
+
                 file_content = await storage_backend.download_file(storage_url)
-                
+
                 # Determine content type from file object
-                content_type = self._get_content_type_from_file_object(db_file.file_object)
-                
+                content_type = self._get_content_type_from_file_object(
+                    db_file.file_object
+                )
+
                 # Convert to base64
                 base64_data = base64.b64encode(file_content).decode("utf-8")
                 base64_data_uri = f"data:{content_type};base64,{base64_data}"
-                
+
                 # Update messages to use base64 instead of file_id
-                self._update_messages_with_base64_data(messages, file_id, base64_data_uri, content_type)
+                self._update_messages_with_base64_data(
+                    messages, file_id, base64_data_uri, content_type
+                )
             except Exception as e:
                 verbose_logger.exception(
                     f"Error converting file {file_id} from storage backend to base64: {str(e)}"
@@ -1541,21 +1609,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
         """
         Determine content type from file object.
-        
+
         Uses the MIME type utility for consistent detection and normalization.
-        
+
         Args:
             file_object: The file object from the database (can be dict, JSON string, or None)
-        
+
         Returns:
             str: MIME type (defaults to "application/octet-stream" if cannot be determined)
         """
         # Use utility function for detection
         content_type = get_content_type_from_file_object(file_object)
-        
+
         # Normalize for Gemini/Vertex AI (requires image/jpeg, not image/jpg)
         content_type = normalize_mime_type_for_provider(content_type, provider="gemini")
-        
+
         return content_type
 
     def _update_messages_with_base64_data(
@@ -1567,7 +1635,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     ) -> None:
         """
         Update messages to replace file_id with base64 data URI.
-        
+
         Args:
             messages: List of messages to update
             file_id: The file ID to replace
@@ -1582,7 +1650,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         if element.get("type") == "file":
                             file_element = cast(ChatCompletionFileObject, element)
                             file_element_file = file_element.get("file", {})
-                            
+
                             if file_element_file.get("file_id") == file_id:
                                 # Replace file_id with base64 data
                                 file_element_file["file_data"] = base64_data_uri
@@ -1590,7 +1658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                                 file_element_file["format"] = content_type
                                 # Remove file_id to ensure only file_data is used
                                 file_element_file.pop("file_id", None)
-                                
+
                                 verbose_logger.debug(
                                     f"Converted file {file_id} from storage backend to base64 with format {content_type}"
                                 )

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -588,24 +588,21 @@ async def update_project(  # noqa: PLR0915
                 param="project_id",
             )
 
-        # Validate team exists and get team object for limit + permission checks
-        team_id_to_check = data.team_id or existing_project.team_id
-        team_obj_for_checks = None
-        if team_id_to_check is not None:
-            team_obj_for_checks = await _validate_team_exists(
-                team_id=team_id_to_check, prisma_client=prisma_client
+        # Permission to *edit* the project must be evaluated against the
+        # project's CURRENT team. Sourcing the team from `data.team_id`
+        # would let an admin of any team pass the check by supplying their
+        # own team_id, hijacking the project (VERIA-55).
+        target_team_id = data.team_id or existing_project.team_id
+        target_team_obj = None
+        if target_team_id is not None:
+            target_team_obj = await _validate_team_exists(
+                team_id=target_team_id, prisma_client=prisma_client
             )
 
-        # Check if user has permission to update this project
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
             team_id=existing_project.team_id,
             prisma_client=prisma_client,
-            team_object=(
-                LiteLLM_TeamTable(**team_obj_for_checks.model_dump())
-                if team_obj_for_checks
-                else None
-            ),
         )
 
         if not has_permission:
@@ -614,10 +611,32 @@ async def update_project(  # noqa: PLR0915
                 detail={"error": "Only admins or team admins can update projects"},
             )
 
+        # Reassigning to a different team also requires admin rights on the
+        # destination team — otherwise a team admin could shed projects into
+        # an unsuspecting team's namespace.
+        if data.team_id is not None and data.team_id != existing_project.team_id:
+            can_assign_to_target = await _check_user_permission_for_project(
+                user_api_key_dict=user_api_key_dict,
+                team_id=data.team_id,
+                prisma_client=prisma_client,
+                team_object=(
+                    LiteLLM_TeamTable(**target_team_obj.model_dump())
+                    if target_team_obj
+                    else None
+                ),
+            )
+            if not can_assign_to_target:
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "Cannot reassign project to a team you are not an admin of"
+                    },
+                )
+
         # Validate project limits against team limits
-        if team_obj_for_checks is not None:
+        if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**team_obj_for_checks.model_dump()),
+                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
                 data=data,
             )
 
@@ -857,10 +876,16 @@ async def project_info(
                 where={"team_id": project.team_id}
             )
             if team:
-                is_team_member = (
-                    user_api_key_dict.user_id in team.admins
-                    or user_api_key_dict.user_id in team.members
-                )
+                caller_user_id = user_api_key_dict.user_id
+                for m in team.members_with_roles or []:
+                    m_user_id = (
+                        m.get("user_id")
+                        if isinstance(m, dict)
+                        else getattr(m, "user_id", None)
+                    )
+                    if m_user_id == caller_user_id:
+                        is_team_member = True
+                        break
 
         if not (is_admin or is_team_member):
             raise HTTPException(
@@ -911,20 +936,20 @@ async def list_projects(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
-            # Get projects for teams the user belongs to
-            user_teams = await prisma_client.db.litellm_teamtable.find_many(
-                where={
-                    "OR": [
-                        {"members": {"has": user_api_key_dict.user_id}},
-                        {"admins": {"has": user_api_key_dict.user_id}},
-                    ]
-                }
+            # Look up the user's team memberships via the reverse-index on
+            # LiteLLM_UserTable.teams (maintained by team_member_add alongside
+            # members_with_roles). This avoids a full scan of all team rows.
+            user_record = await prisma_client.db.litellm_usertable.find_unique(
+                where={"user_id": user_api_key_dict.user_id},
+            )
+            user_team_ids = (
+                user_record.teams
+                if user_record is not None and user_record.teams
+                else []
             )
 
-            team_ids = [team.team_id for team in user_teams]
-
             projects = await prisma_client.db.litellm_projecttable.find_many(
-                where={"team_id": {"in": team_ids}},
+                where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )
 

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.39"
+version = "0.1.40"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
 Documentation = "https://docs.litellm.ai"
 
 [build-system]
-requires = ["uv_build==0.10.7"]
+requires = ["uv_build==0.11.8"]
 build-backend = "uv_build"
 
 [tool.uv]
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.39"
+version = "0.1.40"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-js/proxy/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

litellm-js/proxy/package.json

@@ -4,11 +4,11 @@
     "deploy": "wrangler deploy --minify src/index.ts"
   },
   "dependencies": {
-    "hono": "4.12.12",
+    "hono": "4.12.16",
     "openai": "4.29.2"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "4.20240208.0",
-    "wrangler": "3.32.0"
+    "@cloudflare/workers-types": "4.20260501.1",
+    "wrangler": "4.87.0"
   }
 }

litellm-js/spend-logs/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

litellm-js/spend-logs/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "dependencies": {
         "@hono/node-server": "1.19.13",
-        "hono": "4.12.12"
+        "hono": "4.12.16"
       },
       "devDependencies": {
         "@types/node": "20.19.25",
@@ -535,9 +535,9 @@
       }
     },
     "node_modules/get-tsconfig": {
-      "version": "4.13.0",
-      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
-      "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
+      "version": "4.14.0",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz",
+      "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -548,9 +548,9 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.12.12",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz",
-      "integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==",
+      "version": "4.12.16",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
+      "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
       "license": "MIT",
       "engines": {
         "node": ">=16.9.0"

litellm-js/spend-logs/package.json

@@ -4,7 +4,7 @@
   },
   "dependencies": {
     "@hono/node-server": "1.19.13",
-    "hono": "4.12.12"
+    "hono": "4.12.16"
   },
   "devDependencies": {
     "@types/node": "20.19.25",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql

@@ -0,0 +1,2 @@
+-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores).
+ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql

@@ -0,0 +1,75 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowRun" (
+    "run_id" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "workflow_type" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "created_by" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "input" JSONB,
+    "output" JSONB,
+    "metadata" JSONB,
+
+    CONSTRAINT "LiteLLM_WorkflowRun_pkey" PRIMARY KEY ("run_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowEvent" (
+    "event_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "event_type" TEXT NOT NULL,
+    "step_name" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "data" JSONB,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowMessage" (
+    "message_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "role" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "session_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowMessage_pkey" PRIMARY KEY ("message_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowRun_session_id_key" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_workflow_type_status_idx" ON "LiteLLM_WorkflowRun"("workflow_type", "status");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_session_id_idx" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_at_idx" ON "LiteLLM_WorkflowRun"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_by_idx" ON "LiteLLM_WorkflowRun"("created_by");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowEvent_run_id_idx" ON "LiteLLM_WorkflowEvent"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowEvent_run_id_sequence_number_key" ON "LiteLLM_WorkflowEvent"("run_id", "sequence_number");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowMessage_run_id_idx" ON "LiteLLM_WorkflowMessage"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowMessage_run_id_sequence_number_key" ON "LiteLLM_WorkflowMessage"("run_id", "sequence_number");
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowEvent" ADD CONSTRAINT "LiteLLM_WorkflowEvent_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowMessage" ADD CONSTRAINT "LiteLLM_WorkflowMessage_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260501195714_managed_resource_team_owner/migration.sql

@@ -0,0 +1,20 @@
+-- Adds `team_id` to managed-resource tables so service-account API
+-- keys (no `user_id`) can be scoped by team instead of bypassing the
+-- `created_by` filter entirely. Existing rows keep `team_id = NULL`
+-- and become invisible to team-only callers — that is the intended isolation
+-- outcome; backfill manually if legacy rows must remain visible.
+--
+-- The composite indexes match the listing query: filter by team owner, sort by
+-- created_at DESC. Tables are typically small (resources per tenant, not per
+-- request); a future operator with a large table can switch to
+-- CREATE INDEX CONCURRENTLY in a follow-up migration.
+
+ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+ALTER TABLE "LiteLLM_ManagedVectorStoreTable" ADD COLUMN IF NOT EXISTS "team_id" TEXT;
+
+-- Index names follow Prisma's auto-generated convention so `prisma migrate diff`
+-- against the schema is clean.
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedFileTable_team_id_created_at_idx" ON "LiteLLM_ManagedFileTable" ("team_id", "created_at" DESC);
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedObjectTable_team_id_created_at_idx" ON "LiteLLM_ManagedObjectTable" ("team_id", "created_at" DESC);
+CREATE INDEX IF NOT EXISTS "LiteLLM_ManagedVectorStoreTable_team_id_created_at_idx" ON "LiteLLM_ManagedVectorStoreTable" ("team_id", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
   models                String[]       @default([])
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
+  search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -883,28 +884,32 @@ model LiteLLM_ManagedFileTable {
   storage_backend String? // Storage backend name (e.g., "azure_storage", "gcs", "default")
   storage_url String? // The actual storage URL where the file is stored
   created_at DateTime @default(now())
-  created_by String? 
+  created_by String?
+  team_id String? // Team that owns the resource; populated for service-account keys without a user_id so listings can isolate by team.
   updated_at DateTime @updatedAt
   updated_by String?
 
   @@index([unified_file_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
-model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the 
+model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
   id String @id @default(uuid())
   unified_object_id String @unique // The base64 encoded unified file ID
-  model_object_id String @unique // the id returned by the backend API provider 
+  model_object_id String @unique // the id returned by the backend API provider
   file_object Json // Stores the OpenAIFileObject
   file_purpose String // either 'batch' or 'fine-tune'
-  status String? // check if batch cost has been tracked  
+  status String? // check if batch cost has been tracked
   batch_processed Boolean @default(false) // set to true by CheckBatchCost after cost is computed
   created_at DateTime @default(now())
   created_by String?
+  team_id String?
   updated_at DateTime @updatedAt
-  updated_by String? 
+  updated_by String?
 
   @@index([unified_object_id])
   @@index([model_object_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedVectorStoreTable {
@@ -917,10 +922,12 @@ model LiteLLM_ManagedVectorStoreTable {
   storage_url String? // Storage URL (if applicable)
   created_at DateTime @default(now())
   created_by String?
+  team_id String?
   updated_at DateTime @updatedAt
   updated_by String?
 
   @@index([unified_resource_id])
+  @@index([team_id, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedVectorStoresTable {
@@ -1290,3 +1297,80 @@ model LiteLLM_AdaptiveRouterSession {
   @@id([session_id, router_name, model_name])
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
+
+// ---------------------------------------------------------------------------
+// Workflow Run Tracking
+//
+// Generic durable state tracking for any agent or automated workflow.
+// Design: three tables — run (header + materialized status), event (append-only
+// source of truth for state transitions), message (conversation inbox/outbox).
+//
+// Usage:
+//   - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
+//   - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
+//   - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
+//     the proxy — all spend logs for this run are automatically tagged.
+// ---------------------------------------------------------------------------
+
+// One instance of work being done. `status` is a materialized cache of the
+// latest event; the event log is the authoritative source of truth.
+model LiteLLM_WorkflowRun {
+  run_id        String   @id @default(uuid())
+  session_id    String   @unique @default(uuid())
+  workflow_type String
+  status        String   @default("pending")
+  created_by    String?  // user_id of the key that created this run; null = created by master key
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+  input         Json?
+  output        Json?
+  metadata      Json?
+
+  events   LiteLLM_WorkflowEvent[]
+  messages LiteLLM_WorkflowMessage[]
+
+  @@index([workflow_type, status])
+  @@index([session_id])
+  @@index([created_at])
+  @@index([created_by])
+}
+
+// Append-only log of state transitions. Never mutate rows here.
+// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
+// Status auto-update rules (applied by the append endpoint):
+//   step.started   → run.status = running
+//   step.failed    → run.status = failed
+//   hook.waiting   → run.status = paused
+//   hook.received  → run.status = running
+model LiteLLM_WorkflowEvent {
+  event_id        String   @id @default(uuid())
+  run_id          String
+  event_type      String
+  step_name       String
+  sequence_number Int
+  data            Json?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}
+
+// Conversation inbox/outbox — full message content, separate from the durable
+// event log. Spend logs truncate messages; this table stores them in full.
+// `session_id` here is the Claude --resume session ID (or similar).
+model LiteLLM_WorkflowMessage {
+  message_id      String   @id @default(uuid())
+  run_id          String
+  role            String
+  content         String
+  sequence_number Int
+  session_id      String?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.69"
+version = "0.4.71"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -16,7 +16,7 @@ Repository = "https://github.com/BerriAI/litellm"
 Documentation = "https://docs.litellm.ai"
 
 [build-system]
-requires = ["uv_build==0.10.7"]
+requires = ["uv_build==0.11.8"]
 build-backend = "uv_build"
 
 [tool.uv]
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.69"
+version = "0.4.71"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -166,7 +166,7 @@ langfuse_default_tags: Optional[List[str]] = None
 langsmith_batch_size: Optional[int] = None
 prometheus_initialize_budget_metrics: Optional[bool] = False
 prometheus_latency_buckets: Optional[List[float]] = None
-require_auth_for_metrics_endpoint: Optional[bool] = False
+require_auth_for_metrics_endpoint: Optional[bool] = True
 argilla_batch_size: Optional[int] = None
 datadog_use_v1: Optional[bool] = False  # if you want to use v1 datadog logged payload.
 gcs_pub_sub_use_v1: Optional[bool] = (
@@ -280,6 +280,7 @@ ssl_security_level: Optional[str] = None
 ssl_certificate: Optional[str] = None
 user_url_validation: bool = True
 user_url_allowed_hosts: List[str] = []
+provider_url_destination_allowed_hosts: List[str] = []
 ssl_ecdh_curve: Optional[str] = (
     None  # Set to 'X25519' to disable PQC and improve performance
 )
@@ -288,6 +289,7 @@ disable_token_counter: bool = False
 disable_add_transform_inline_image_block: bool = False
 disable_add_user_agent_to_request_tags: bool = False
 disable_anthropic_gemini_context_caching_transform: bool = False
+disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"
 safe_memory_mode: bool = False
@@ -330,6 +332,9 @@ enable_model_config_credential_overrides: bool = False
 enable_key_alias_format_validation: bool = (
     False  # opt-in validation of key_alias format on /key/generate and /key/update
 )
+enable_gemini_default_thinking_level_low: bool = (
+    False  # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
+)
 ####################
 logging: bool = True
 enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@@ -383,6 +388,7 @@ anthropic_beta_headers_url: str = os.getenv(
 suppress_debug_info = False
 dynamodb_table_name: Optional[str] = None
 s3_callback_params: Optional[Dict] = None
+s3_audit_callback_params: Optional[Dict] = None
 datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
 datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
 aws_sqs_callback_params: Optional[Dict] = None
@@ -409,6 +415,9 @@ custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
 prometheus_emit_stream_label: bool = False
+prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
+prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
+prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
 disable_add_prefix_to_prompt: bool = (
     False  # used by anthropic, to disable adding prefix to prompt
 )
@@ -581,6 +590,7 @@ anyscale_models: Set = set()
 cerebras_models: Set = set()
 galadriel_models: Set = set()
 nvidia_nim_models: Set = set()
+nvidia_riva_models: Set = set()
 sambanova_models: Set = set()
 sambanova_embedding_models: Set = set()
 novita_models: Set = set()
@@ -807,6 +817,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             galadriel_models.add(key)
         elif value.get("litellm_provider") == "nvidia_nim":
             nvidia_nim_models.add(key)
+        elif value.get("litellm_provider") == "nvidia_riva":
+            nvidia_riva_models.add(key)
         elif value.get("litellm_provider") == "sambanova":
             sambanova_models.add(key)
         elif value.get("litellm_provider") == "sambanova-embedding-models":
@@ -966,6 +978,7 @@ model_list = list(
     | cerebras_models
     | galadriel_models
     | nvidia_nim_models
+    | nvidia_riva_models
     | sambanova_models
     | azure_text_models
     | novita_models
@@ -1062,6 +1075,7 @@ models_by_provider: dict = {
     "cerebras": cerebras_models,
     "galadriel": galadriel_models,
     "nvidia_nim": nvidia_nim_models,
+    "nvidia_riva": nvidia_riva_models,
     "sambanova": sambanova_models | sambanova_embedding_models,
     "novita": novita_models,
     "nebius": nebius_models | nebius_embedding_models,
@@ -1613,6 +1627,9 @@ if TYPE_CHECKING:
     from .llms.deepgram.audio_transcription.transformation import (
         DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
     )
+    from .llms.nvidia_riva.audio_transcription.transformation import (
+        NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
+    )
     from .llms.topaz.image_variations.transformation import (
         TopazImageVariationConfig as TopazImageVariationConfig,
     )

litellm/_logging.py

@@ -1,12 +1,12 @@
 import ast
 import logging
 import os
-import re
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 
@@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = (
     os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
 )
 
-_REDACTED = "REDACTED"
-
-
-def _build_secret_patterns() -> re.Pattern:
-    patterns: List[str] = [
-        # ── PEM private key / certificate blocks ──
-        r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
-        # ── GCP OAuth2 access tokens (ya29.*) ──
-        r"\bya29\.[A-Za-z0-9_.~+/-]+",
-        # ── Credential %s formatting (space separator, no key= prefix) ──
-        r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
-        # AWS access key IDs
-        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
-        # AWS secrets / session tokens / access key IDs (key=value)
-        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
-        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
-        # Bearer tokens (OAuth, JWT, etc.)
-        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
-        # Basic auth headers
-        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
-        # OpenAI / Anthropic sk- prefixed keys
-        r"sk-[A-Za-z0-9\-_]{20,}",
-        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
-        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
-        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
-        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Anthropic internal header keys
-        r"x-ak-[A-Za-z0-9\-_]{20,}",
-        # Google API keys
-        r"AIza[0-9A-Za-z\-_]{35}",
-        # Password / secret params (handles key=value and 'key': 'value')
-        # Word boundary prevents O(n^2) backtracking on long word-char runs.
-        r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
-        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Database connection string credentials (scheme://user:pass@host)
-        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
-        # Databricks personal access tokens
-        r"dapi[0-9a-f]{32}",
-        # ── Key-name-based redaction ──
-        # Catches secrets inside dicts/config dumps by matching on the KEY name
-        # regardless of what the value looks like.
-        # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
-        # private_key with PEM-aware value capture
-        r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
-        r"(?:master_key|database_url|db_url|connection_string|"
-        r"signing_key|encryption_key|"
-        r"auth_token|access_token|refresh_token|"
-        r"slack_webhook_url|webhook_url|"
-        r"database_connection_string|"
-        r"huggingface_token|jwt_secret)"
-        r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
-        # ── Raw JWTs (without Bearer prefix) ──
-        r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
-        # ── Azure SAS tokens in URLs ──
-        r"[?&]sig=[A-Za-z0-9%+/=]+",
-        # ── Full JSON service-account blobs (single-line and multi-line) ──
-        r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
-    ]
-    return re.compile("|".join(patterns), re.IGNORECASE)
-
-
-_SECRET_RE = _build_secret_patterns()
-
 
 def _redact_string(value: str) -> str:
     if not _ENABLE_SECRET_REDACTION:
         return value
-    return _SECRET_RE.sub(_REDACTED, value)
+    return redact_string(value)
 
 
 def redact_secrets(value: str) -> str:

litellm/anthropic_beta_headers_config.json

@@ -72,7 +72,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": null,
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
     "fine-grained-tool-streaming-2025-05-14": null,
@@ -103,7 +103,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": null,
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
     "fine-grained-tool-streaming-2025-05-14": null,

litellm/batches/batch_utils.py

@@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
     )
 
 
+def _get_models_from_batch_input_file_content(
+    file_content_dictionary: List[dict],
+) -> List[str]:
+    """Extract the distinct ``body.model`` values from a batch *input* file.
+
+    Used by the proxy's batch pre-call hook to enforce that the caller is
+    authorized for every model named inside the JSONL — not just the one
+    on the outer request — so the proxy's per-key model allowlist isn't
+    bypassed by smuggling expensive models into the batch file.
+    """
+    models: List[str] = []
+    seen: set = set()
+    for _item in file_content_dictionary:
+        body = _item.get("body") or {}
+        model = body.get("model")
+        if model and model not in seen:
+            seen.add(model)
+            models.append(model)
+    return models
+
+
 def _get_batch_job_input_file_usage(
     file_content_dictionary: List[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@@ -403,11 +424,25 @@ def _get_batch_job_input_file_usage(
     for _item in file_content_dictionary:
         body = _item.get("body", {})
         model = body.get("model", model_name or "")
-        messages = body.get("messages", [])
 
+        # Chat completion payloads.
+        messages = body.get("messages")
         if messages:
-            item_tokens = token_counter(model=model, messages=messages)
-            prompt_tokens += item_tokens
+            prompt_tokens += token_counter(model=model, messages=messages)
+            continue
+
+        # Text completion payloads (`prompt`).
+        prompt = body.get("prompt")
+        if prompt:
+            prompt_tokens += _count_prompt_or_input_tokens(model=model, value=prompt)
+            continue
+
+        # Embedding payloads (`input`).
+        input_data = body.get("input")
+        if input_data:
+            prompt_tokens += _count_prompt_or_input_tokens(
+                model=model, value=input_data
+            )
 
     return Usage(
         total_tokens=prompt_tokens + completion_tokens,
@@ -416,6 +451,43 @@ def _get_batch_job_input_file_usage(
     )
 
 
+def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
+    """Token-count a ``prompt`` / ``input`` field that the OpenAI batch
+    schema allows in four shapes:
+
+    - ``str``: a single text prompt.
+    - ``list[str]``: multiple text prompts.
+    - ``list[int]``: a pre-tokenized prompt (each int counts as 1 token).
+    - ``list[list[int]]``: multiple pre-tokenized prompts.
+
+    Pre-fix only the string shapes were counted, so a caller could send
+    a large ``list[list[int]]`` payload and slip past TPM rate limits
+    with a recorded cost of zero tokens.
+    """
+    if isinstance(value, str):
+        return token_counter(model=model, text=value)
+    if isinstance(value, list):
+        total = 0
+        for chunk in value:
+            if isinstance(chunk, str):
+                total += token_counter(model=model, text=chunk)
+            elif isinstance(chunk, int):
+                # Single pre-tokenized prompt at the top level: each
+                # int counts as one token.
+                total += 1
+            elif isinstance(chunk, list):
+                # Nested pre-tokenized prompt: every int contributes a
+                # token. Mixed string/int items still count.
+                total += sum(1 if isinstance(t, int) else 0 for t in chunk)
+                total += sum(
+                    token_counter(model=model, text=t)
+                    for t in chunk
+                    if isinstance(t, str)
+                )
+        return total
+    return 0
+
+
 def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
     """
     Get the tokens of a batch job from the response body

litellm/batches/main.py

@@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config(
         )
     else:
         raise litellm.exceptions.BadRequestError(
-            message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
-                custom_llm_provider
-            ),
+            message=(
+                "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. "
+                "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
+                "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
+            ).format(custom_llm_provider),
             model="n/a",
             llm_provider=custom_llm_provider,
             response=httpx.Response(
                 status_code=400,
                 content="Unsupported provider",
-                request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+                request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),  # type: ignore
             ),
         )
     return response

litellm/caching/caching.py

@@ -432,9 +432,10 @@ class Cache:
             str: The final hashed cache key with the redis namespace.
         """
         dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
+        metadata = kwargs.get("metadata") or {}
         namespace = (
             dynamic_cache_control.get("namespace")
-            or kwargs.get("metadata", {}).get("redis_namespace")
+            or metadata.get("redis_namespace")
             or self.namespace
         )
         if namespace:
@@ -650,7 +651,10 @@ class Cache:
             verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
 
     def _convert_to_cached_embedding(
-        self, embedding_response: Any, model: Optional[str]
+        self,
+        embedding_response: Any,
+        model: Optional[str],
+        prompt_tokens_details: Optional[dict] = None,
     ) -> CachedEmbedding:
         """
         Convert any embedding response into the standardized CachedEmbedding TypedDict format.
@@ -662,6 +666,7 @@ class Cache:
                     "index": embedding_response.get("index"),
                     "object": embedding_response.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
             elif hasattr(embedding_response, "model_dump"):
                 data = embedding_response.model_dump()
@@ -670,6 +675,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
             else:
                 data = vars(embedding_response)
@@ -678,10 +684,54 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
         except KeyError as e:
             raise ValueError(f"Missing expected key in embedding response: {e}")
 
+    def _get_per_item_prompt_tokens_details(
+        self,
+        result: EmbeddingResponse,
+        idx_in_result_data: int,
+    ) -> Optional[dict]:
+        """
+        Extract per-item prompt_tokens_details from a response for caching.
+
+        For single-item responses (common for multimodal providers like Bedrock Titan,
+        Nova, Vertex AI), returns the full prompt_tokens_details.
+        For multi-item responses, distributes integer fields evenly across items
+        so that summing all per-item details reconstructs the original totals.
+        """
+        if result.usage is None or result.usage.prompt_tokens_details is None:
+            return None
+
+        details = result.usage.prompt_tokens_details
+        if hasattr(details, "model_dump"):
+            details_dict = details.model_dump(exclude_none=True)
+        elif isinstance(details, dict):
+            details_dict = {k: v for k, v in details.items() if v is not None}
+        else:
+            return None
+
+        if not details_dict:
+            return None
+
+        num_items = len(result.data)
+        if num_items <= 1:
+            return details_dict
+
+        # Distribute integer/float fields evenly across items
+        per_item: dict = {}
+        for key, value in details_dict.items():
+            if isinstance(value, int):
+                quotient, remainder = divmod(value, num_items)
+                per_item[key] = quotient + (1 if idx_in_result_data < remainder else 0)
+            elif isinstance(value, float):
+                per_item[key] = value / num_items
+            else:
+                per_item[key] = value
+        return per_item if per_item else None
+
     def add_embedding_response_to_cache(
         self,
         result: EmbeddingResponse,
@@ -693,10 +743,18 @@ class Cache:
         kwargs["cache_key"] = preset_cache_key
         embedding_response = result.data[idx_in_result_data]
 
+        # Extract per-item prompt_tokens_details from response usage
+        prompt_tokens_details = self._get_per_item_prompt_tokens_details(
+            result=result,
+            idx_in_result_data=idx_in_result_data,
+        )
+
         # Always convert to properly typed CachedEmbedding
         model_name = result.model
         embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
-            embedding_response, model_name
+            embedding_response,
+            model_name,
+            prompt_tokens_details=prompt_tokens_details,
         )
 
         cache_key, cached_data, kwargs = self._add_cache_logic(

litellm/caching/caching_handler.py

@@ -59,6 +59,7 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.utils import PromptTokensDetailsWrapper
 else:
     LiteLLMLoggingObj = Any
 
@@ -86,6 +87,18 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
+    """
+    When stream=True, do not run success callbacks at cache-hit time.
+
+    Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
+    replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
+    handlers when the stream finishes; firing them here too would double-count
+    spend and callback records.
+    """
+    return kwargs.get("stream", False) is True
+
+
 class LLMCachingHandler:
     def __init__(
         self,
@@ -98,6 +111,7 @@ class LLMCachingHandler:
         self.async_streaming_chunks: List[ModelResponse] = []
         self.sync_streaming_chunks: List[ModelResponse] = []
         self.request_kwargs = request_kwargs
+        self.preset_cache_key: Optional[str] = None
         self.original_function = original_function
         self.start_time = start_time
         if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
@@ -205,7 +219,7 @@ class LLMCachingHandler:
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         args=args,
                     )
-                    if kwargs.get("stream", False) is False:
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
                         # LOG SUCCESS
                         self._async_log_cache_hit_on_callbacks(
                             logging_obj=logging_obj,
@@ -214,11 +228,12 @@ class LLMCachingHandler:
                             end_time=end_time,
                             cache_hit=cache_hit,
                         )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
+                    )
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
                 elif (
@@ -264,8 +279,6 @@ class LLMCachingHandler:
         kwargs: Dict[str, Any],
         args: Optional[Tuple[Any, ...]] = None,
     ) -> CachingHandlerResponse:
-        from litellm.utils import CustomStreamWrapper
-
         cached_result: Optional[Any] = None
 
         # Check if caching should be performed BEFORE doing expensive kwargs copy
@@ -281,6 +294,11 @@ class LLMCachingHandler:
                     args,
                 )
             )
+            if new_kwargs.get("metadata") is None:
+                new_kwargs.pop("metadata", None)
+            if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+                new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+            self.request_kwargs = new_kwargs
             print_verbose("Checking Sync Cache")
             cached_result = litellm.cache.get_cache(**new_kwargs)
             if cached_result is not None:
@@ -321,17 +339,19 @@ class LLMCachingHandler:
                         is_async=False,
                     )
 
-                    logging_obj.handle_sync_success_callbacks_for_async_calls(
-                        result=cached_result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        cache_hit=cache_hit,
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
+                        logging_obj.handle_sync_success_callbacks_for_async_calls(
+                            result=cached_result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            cache_hit=cache_hit,
+                        )
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
         return CachingHandlerResponse(cached_result=cached_result)
@@ -415,6 +435,7 @@ class LLMCachingHandler:
             final_embedding_cached_response._hidden_params["cache_hit"] = True
 
             prompt_tokens = 0
+            aggregated_details: Optional[dict] = None
             for val in non_null_list:
                 idx, cr = val  # (idx, cr) tuple
                 if cr is not None:
@@ -431,11 +452,35 @@ class LLMCachingHandler:
                         prompt_tokens += token_counter(
                             text=kwargs_input_as_list[idx], count_response_tokens=True
                         )
+                    # Aggregate prompt_tokens_details from cached items
+                    item_details = cr.get("prompt_tokens_details")
+                    if item_details:
+                        if aggregated_details is None:
+                            aggregated_details = {}
+                        for key, value in item_details.items():
+                            if isinstance(value, (int, float)):
+                                aggregated_details[key] = (
+                                    aggregated_details.get(key, 0) + value
+                                )
+                            else:
+                                aggregated_details[key] = value
+
             ## USAGE
+            prompt_tokens_details: Optional["PromptTokensDetailsWrapper"] = None
+            if aggregated_details:
+                from litellm.types.utils import PromptTokensDetailsWrapper
+
+                try:
+                    prompt_tokens_details = PromptTokensDetailsWrapper(
+                        **aggregated_details
+                    )
+                except Exception:
+                    prompt_tokens_details = None
             usage = Usage(
                 prompt_tokens=prompt_tokens,
                 completion_tokens=0,
                 total_tokens=prompt_tokens,
+                prompt_tokens_details=prompt_tokens_details,
             )
             final_embedding_cached_response.usage = usage
         if len(remaining_list) == 0:
@@ -478,8 +523,70 @@ class LLMCachingHandler:
             prompt_tokens=usage1.prompt_tokens + usage2.prompt_tokens,
             completion_tokens=usage1.completion_tokens + usage2.completion_tokens,
             total_tokens=usage1.total_tokens + usage2.total_tokens,
+            prompt_tokens_details=self._merge_prompt_tokens_details(
+                usage1.prompt_tokens_details,
+                usage2.prompt_tokens_details,
+            ),
         )
 
+    def _merge_prompt_tokens_details(
+        self,
+        details1: Optional["PromptTokensDetailsWrapper"],
+        details2: Optional["PromptTokensDetailsWrapper"],
+    ) -> Optional["PromptTokensDetailsWrapper"]:
+        """Merge two PromptTokensDetailsWrapper objects by summing numeric fields."""
+        if details1 is None and details2 is None:
+            return None
+        if details1 is None:
+            return details2
+        if details2 is None:
+            return details1
+
+        dict1 = (
+            details1.model_dump(exclude_none=True)
+            if hasattr(details1, "model_dump")
+            else {}
+        )
+        dict2 = (
+            details2.model_dump(exclude_none=True)
+            if hasattr(details2, "model_dump")
+            else {}
+        )
+
+        merged: dict = {}
+        for key in set(dict1.keys()) | set(dict2.keys()):
+            v1 = dict1.get(key, 0)
+            v2 = dict2.get(key, 0)
+            if isinstance(v1, (int, float)) and isinstance(v2, (int, float)):
+                merged[key] = v1 + v2
+            elif isinstance(v1, dict) and isinstance(v2, dict):
+                # Recursively merge nested dicts (e.g. cache_creation_token_details)
+                nested: dict = {}
+                for nk in set(v1.keys()) | set(v2.keys()):
+                    nv1 = v1.get(nk, 0)
+                    nv2 = v2.get(nk, 0)
+                    if isinstance(nv1, (int, float)) and isinstance(nv2, (int, float)):
+                        nested[nk] = nv1 + nv2
+                    elif nv1:
+                        nested[nk] = nv1
+                    else:
+                        nested[nk] = nv2
+                merged[key] = nested
+            elif v1:
+                merged[key] = v1
+            else:
+                merged[key] = v2
+
+        if not merged:
+            return None
+
+        from litellm.types.utils import PromptTokensDetailsWrapper
+
+        try:
+            return PromptTokensDetailsWrapper(**merged)
+        except Exception:
+            return None
+
     def _combine_cached_embedding_response_with_api_result(
         self,
         _caching_handler_response: CachingHandlerResponse,
@@ -598,6 +705,11 @@ class LLMCachingHandler:
                 args,
             )
         )
+        if new_kwargs.get("metadata") is None:
+            new_kwargs.pop("metadata", None)
+        if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+            new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+        self.request_kwargs = new_kwargs
         cached_result: Optional[Any] = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):
@@ -622,14 +734,26 @@ class LLMCachingHandler:
                 if all(result is None for result in cached_result):
                     cached_result = None
         else:
+            request_kwargs = new_kwargs.copy()
+            request_cache_key = request_kwargs.pop("cache_key", None)
             if litellm.cache._supports_async() is True:
                 ## check if dual cache is supported ##
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = await litellm.cache.async_get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
             else:  # fallback for caches that don't support async
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = litellm.cache.get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
         return cached_result
 
@@ -737,8 +861,27 @@ class LLMCachingHandler:
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(
             cached_result, dict
         ):
-            # Convert cached dict back to ResponsesAPIResponse object
-            cached_result = ResponsesAPIResponse(**cached_result)
+            from litellm.responses.streaming_iterator import (
+                CachedResponsesAPIStreamingIterator,
+            )
+
+            response_obj = ResponsesAPIResponse(**cached_result)
+            if (
+                hasattr(response_obj, "_hidden_params")
+                and response_obj._hidden_params is not None
+                and isinstance(response_obj._hidden_params, dict)
+            ):
+                response_obj._hidden_params["cache_hit"] = True
+
+            if kwargs.get("stream", False) is True:
+                cached_result = CachedResponsesAPIStreamingIterator(
+                    response=response_obj,
+                    logging_obj=logging_obj,
+                    request_data=kwargs,
+                    call_type=call_type,
+                )
+            else:
+                cached_result = response_obj
 
         if (
             hasattr(cached_result, "_hidden_params")

litellm/caching/dual_cache.py

@@ -92,6 +92,25 @@ class DualCache(BaseCache):
         if default_redis_ttl is not None:
             self.default_redis_ttl = default_redis_ttl
 
+    def attach_redis_cache(
+        self,
+        redis_cache: Optional[RedisCache] = None,
+        *,
+        default_redis_ttl: Optional[float] = None,
+    ) -> None:
+        """
+        Attach a Redis backend if this DualCache does not already have one.
+
+        No-op when ``redis_cache`` is None or when Redis was already set (constructor
+        or a prior attach). Use this for lazy wiring after a shared Redis client exists.
+        Does not backfill in-memory-only keys to Redis.
+        """
+        if redis_cache is None or self.redis_cache is not None:
+            return
+        self.redis_cache = redis_cache
+        if default_redis_ttl is not None:
+            self.default_redis_ttl = default_redis_ttl
+
     def set_cache(self, key, value, local_only: bool = False, **kwargs):
         # Update both Redis and in-memory cache
         try:
@@ -392,6 +411,7 @@ class DualCache(BaseCache):
         value: float,
         parent_otel_span: Optional[Span] = None,
         local_only: bool = False,
+        refresh_ttl: bool = False,
         **kwargs,
     ) -> Optional[float]:
         """
@@ -399,6 +419,9 @@ class DualCache(BaseCache):
 
         Value - float - the value you want to increment by
 
+        Refresh_ttl - bool - if True, resets the Redis TTL on every write.
+        Default False preserves window-style semantics.
+
         Returns - the incremented value, or None if no cache backend is
         available (in_memory_cache is None and Redis failed/is absent).
         """
@@ -415,6 +438,7 @@ class DualCache(BaseCache):
                     value,
                     parent_otel_span=parent_otel_span,
                     ttl=kwargs.get("ttl", None),
+                    refresh_ttl=refresh_ttl,
                 )
 
             return result

litellm/caching/qdrant_semantic_cache.py

@@ -11,17 +11,23 @@ Has 4 methods:
 import ast
 import asyncio
 import json
-from typing import Any, cast
+import os
+from typing import Any, Dict, cast
 
 import litellm
 from litellm._logging import print_verbose
 from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    get_str_from_messages,
+)
 from litellm.types.utils import EmbeddingResponse
 
 from .base_cache import BaseCache
 
 
 class QdrantSemanticCache(BaseCache):
+    CACHE_KEY_FIELD_NAME = "litellm_cache_key"
+
     def __init__(  # noqa: PLR0915
         self,
         qdrant_api_base=None,
@@ -33,8 +39,6 @@ class QdrantSemanticCache(BaseCache):
         host_type=None,
         vector_size=None,
     ):
-        import os
-
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
             get_async_httpx_client,
@@ -115,7 +119,9 @@ class QdrantSemanticCache(BaseCache):
             print_verbose(
                 f"Collection already exists.\nCollection details:{self.collection_info}"
             )
+            self._ensure_cache_key_payload_index()
         else:
+            quantization_params: Dict[str, Any]
             if quantization_config is None or quantization_config == "binary":
                 quantization_params = {
                     "binary": {
@@ -156,6 +162,7 @@ class QdrantSemanticCache(BaseCache):
                 print_verbose(
                     f"New collection created.\nCollection details:{self.collection_info}"
                 )
+                self._ensure_cache_key_payload_index()
             else:
                 raise Exception("Error while creating new collection")
 
@@ -170,15 +177,94 @@ class QdrantSemanticCache(BaseCache):
             cached_response = ast.literal_eval(cached_response)
         return cached_response
 
+    def _get_qdrant_cache_key_filter(self, key: str) -> dict:
+        return {
+            "must": [
+                {
+                    "key": self.CACHE_KEY_FIELD_NAME,
+                    "match": {"value": str(key)},
+                }
+            ]
+        }
+
+    def _add_cache_key_filter_to_search_data(self, data: dict, key: str) -> None:
+        data["filter"] = self._get_qdrant_cache_key_filter(key)
+
+    def _ensure_cache_key_payload_index(self) -> None:
+        try:
+            response = self.sync_client.put(
+                url=f"{self.qdrant_api_base}/collections/{self.collection_name}/index",
+                headers=self.headers,
+                json={
+                    "field_name": self.CACHE_KEY_FIELD_NAME,
+                    "field_schema": "keyword",
+                },
+            )
+            if response.status_code not in (200, 201):
+                print_verbose(
+                    "Qdrant semantic-cache could not create cache-key payload index: "
+                    f"{response.text}"
+                )
+        except Exception as exc:
+            print_verbose(
+                "Qdrant semantic-cache could not create cache-key payload index: "
+                f"{str(exc)}"
+            )
+
+    def _payload_matches_cache_key(self, payload: dict, key: str) -> bool:
+        # Pre-isolation points stored only prompt + response with no cache-key
+        # payload field. Reassigning them to a caller's key would risk
+        # cross-scope hits, so they're treated as misses and re-populated on
+        # the next set_cache.
+        cached_key = payload.get(self.CACHE_KEY_FIELD_NAME)
+        return cached_key is not None and str(cached_key) == str(key)
+
+    async def _get_async_embedding(self, prompt: str, **kwargs) -> Any:
+        llm_model_list = None
+        llm_router = None
+
+        try:
+            from litellm.proxy.proxy_server import (
+                llm_model_list as proxy_llm_model_list,
+                llm_router as proxy_llm_router,
+            )
+
+            llm_model_list = proxy_llm_model_list
+            llm_router = proxy_llm_router
+        except ImportError:
+            pass
+
+        router_model_names = (
+            [m["model_name"] for m in llm_model_list]
+            if llm_model_list is not None
+            else []
+        )
+        if llm_router is not None and self.embedding_model in router_model_names:
+            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
+            return await llm_router.aembedding(
+                model=self.embedding_model,
+                input=prompt,
+                cache={"no-store": True, "no-cache": True},
+                metadata={
+                    "user_api_key": user_api_key,
+                    "semantic-cache-embedding": True,
+                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
+                },
+            )
+
+        return await litellm.aembedding(
+            model=self.embedding_model,
+            input=prompt,
+            cache={"no-store": True, "no-cache": True},
+        )
+
     def set_cache(self, key, value, **kwargs):
         print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")
         from litellm._uuid import uuid
 
         # get the prompt
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
         # create an embedding for prompt
         embedding_response = cast(
@@ -202,6 +288,7 @@ class QdrantSemanticCache(BaseCache):
                     "id": str(uuid.uuid4()),
                     "vector": embedding,
                     "payload": {
+                        self.CACHE_KEY_FIELD_NAME: str(key),
                         "text": prompt,
                         "response": value,
                     },
@@ -220,9 +307,7 @@ class QdrantSemanticCache(BaseCache):
 
         # get the messages
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
         # convert to embedding
         embedding_response = cast(
@@ -249,6 +334,7 @@ class QdrantSemanticCache(BaseCache):
             "limit": 1,
             "with_payload": True,
         }
+        self._add_cache_key_filter_to_search_data(data=data, key=key)
 
         search_response = self.sync_client.post(
             url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@@ -258,21 +344,33 @@ class QdrantSemanticCache(BaseCache):
         results = search_response.json()["result"]
 
         if results is None:
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
             return None
         if isinstance(results, list):
             if len(results) == 0:
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
         similarity = results[0]["score"]
-        cached_prompt = results[0]["payload"]["text"]
+        payload = results[0]["payload"]
+        if not self._payload_matches_cache_key(payload=payload, key=key):
+            print_verbose("Qdrant semantic-cache hit did not match cache key scope")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+            return None
+
+        cached_prompt = payload["text"]
 
         # check similarity, if more than self.similarity_threshold, return results
         print_verbose(
             f"semantic cache: similarity threshold: {self.similarity_threshold}, similarity: {similarity}, prompt: {prompt}, closest_cached_prompt: {cached_prompt}"
         )
+
+        # update kwargs["metadata"] with similarity, don't rewrite the original metadata
+        kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
+
         if similarity >= self.similarity_threshold:
             # cache hit !
-            cached_value = results[0]["payload"]["response"]
+            cached_value = payload["response"]
             print_verbose(
                 f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
             )
@@ -285,40 +383,12 @@ class QdrantSemanticCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         from litellm._uuid import uuid
 
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
-
         print_verbose(f"async qdrant semantic-cache set_cache, kwargs: {kwargs}")
 
         # get the prompt
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
-        # create an embedding for prompt
-        router_model_names = (
-            [m["model_name"] for m in llm_model_list]
-            if llm_model_list is not None
-            else []
-        )
-        if llm_router is not None and self.embedding_model in router_model_names:
-            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
-            embedding_response = await llm_router.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-                metadata={
-                    "user_api_key": user_api_key,
-                    "semantic-cache-embedding": True,
-                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
-                },
-            )
-        else:
-            # convert to embedding
-            embedding_response = await litellm.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-            )
+        prompt = get_str_from_messages(messages)
+        embedding_response = await self._get_async_embedding(prompt, **kwargs)
 
         # get the embedding
         embedding = embedding_response["data"][0]["embedding"]
@@ -332,6 +402,7 @@ class QdrantSemanticCache(BaseCache):
                     "id": str(uuid.uuid4()),
                     "vector": embedding,
                     "payload": {
+                        self.CACHE_KEY_FIELD_NAME: str(key),
                         "text": prompt,
                         "response": value,
                     },
@@ -348,38 +419,12 @@ class QdrantSemanticCache(BaseCache):
 
     async def async_get_cache(self, key, **kwargs):
         print_verbose(f"async qdrant semantic-cache get_cache, kwargs: {kwargs}")
-        from litellm.proxy.proxy_server import llm_model_list, llm_router
 
         # get the messages
         messages = kwargs["messages"]
-        prompt = ""
-        for message in messages:
-            prompt += message["content"]
+        prompt = get_str_from_messages(messages)
 
-        router_model_names = (
-            [m["model_name"] for m in llm_model_list]
-            if llm_model_list is not None
-            else []
-        )
-        if llm_router is not None and self.embedding_model in router_model_names:
-            user_api_key = kwargs.get("metadata", {}).get("user_api_key", "")
-            embedding_response = await llm_router.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-                metadata={
-                    "user_api_key": user_api_key,
-                    "semantic-cache-embedding": True,
-                    "trace_id": kwargs.get("metadata", {}).get("trace_id", None),
-                },
-            )
-        else:
-            # convert to embedding
-            embedding_response = await litellm.aembedding(
-                model=self.embedding_model,
-                input=prompt,
-                cache={"no-store": True, "no-cache": True},
-            )
+        embedding_response = await self._get_async_embedding(prompt, **kwargs)
 
         # get the embedding
         embedding = embedding_response["data"][0]["embedding"]
@@ -396,6 +441,7 @@ class QdrantSemanticCache(BaseCache):
             "limit": 1,
             "with_payload": True,
         }
+        self._add_cache_key_filter_to_search_data(data=data, key=key)
 
         search_response = await self.async_client.post(
             url=f"{self.qdrant_api_base}/collections/{self.collection_name}/points/search",
@@ -414,7 +460,13 @@ class QdrantSemanticCache(BaseCache):
                 return None
 
         similarity = results[0]["score"]
-        cached_prompt = results[0]["payload"]["text"]
+        payload = results[0]["payload"]
+        if not self._payload_matches_cache_key(payload=payload, key=key):
+            print_verbose("Qdrant semantic-cache hit did not match cache key scope")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+            return None
+
+        cached_prompt = payload["text"]
 
         # check similarity, if more than self.similarity_threshold, return results
         print_verbose(
@@ -426,7 +478,7 @@ class QdrantSemanticCache(BaseCache):
 
         if similarity >= self.similarity_threshold:
             # cache hit !
-            cached_value = results[0]["payload"]["response"]
+            cached_value = payload["response"]
             print_verbose(
                 f"got a cache hit, similarity: {similarity}, Current prompt: {prompt}, cached_prompt: {cached_prompt}"
             )

litellm/caching/redis_cache.py

@@ -551,6 +551,13 @@ class RedisCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         from redis.asyncio import Redis
 
+        if key is None:
+            verbose_logger.debug(
+                "LiteLLM Redis Caching: async set() skipped — key is None, value=%r",
+                value,
+            )
+            return None
+
         start_time = time.time()
         try:
             _redis_client: Redis = self.init_async_client()  # type: ignore
@@ -569,8 +576,9 @@ class RedisCache(BaseCache):
                 )
             )
             verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
+                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
                 str(e),
+                key,
                 value,
             )
             raise e
@@ -824,6 +832,7 @@ class RedisCache(BaseCache):
         value: float,
         ttl: Optional[int] = None,
         parent_otel_span: Optional[Span] = None,
+        refresh_ttl: bool = False,
     ) -> float:
         from redis.asyncio import Redis
 
@@ -834,11 +843,12 @@ class RedisCache(BaseCache):
         try:
             result = await _redis_client.incrbyfloat(name=key, amount=value)
             if _used_ttl is not None:
-                # check if key already has ttl, if not -> set ttl
-                current_ttl = await _redis_client.ttl(key)
-                if current_ttl == -1:
-                    # Key has no expiration
+                if refresh_ttl:
                     await _redis_client.expire(key, _used_ttl)
+                else:
+                    current_ttl = await _redis_client.ttl(key)
+                    if current_ttl == -1:
+                        await _redis_client.expire(key, _used_ttl)
 
             ## LOGGING ##
             end_time = time.time()

litellm/caching/redis_semantic_cache.py

@@ -35,6 +35,7 @@ class RedisSemanticCache(BaseCache):
     """
 
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
+    CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
 
     def __init__(
         self,
@@ -66,8 +67,8 @@ class RedisSemanticCache(BaseCache):
             Exception: If similarity_threshold is not provided or required Redis
                 connection information is missing
         """
-        from redisvl.extensions.llmcache import SemanticCache
-        from redisvl.utils.vectorize import CustomTextVectorizer
+        from redisvl.extensions.llmcache import SemanticCache  # type: ignore[import-not-found, import-untyped]
+        from redisvl.utils.vectorize import CustomTextVectorizer  # type: ignore[import-not-found, import-untyped]
 
         if index_name is None:
             index_name = self.DEFAULT_REDIS_INDEX_NAME
@@ -109,14 +110,94 @@ class RedisSemanticCache(BaseCache):
         # Initialize the Redis vectorizer and cache
         cache_vectorizer = CustomTextVectorizer(self._get_embedding)
 
-        self.llmcache = SemanticCache(
-            name=index_name,
+        self.llmcache = self._init_semantic_cache(
+            semantic_cache_cls=SemanticCache,
+            index_name=index_name,
             redis_url=redis_url,
-            vectorizer=cache_vectorizer,
-            distance_threshold=self.distance_threshold,
-            overwrite=False,
+            cache_vectorizer=cache_vectorizer,
         )
 
+    @classmethod
+    def _cache_key_filterable_field(cls) -> Dict[str, str]:
+        return {
+            "name": cls.CACHE_KEY_FIELD_NAME,
+            "type": "tag",
+        }
+
+    def _init_semantic_cache(
+        self,
+        semantic_cache_cls: Any,
+        index_name: str,
+        redis_url: str,
+        cache_vectorizer: Any,
+    ) -> Any:
+        def _is_schema_mismatch(exc: ValueError) -> bool:
+            error_message = str(exc).lower()
+            return any(
+                phrase in error_message
+                for phrase in ("schema does not match", "index schema")
+            )
+
+        try:
+            return semantic_cache_cls(
+                name=index_name,
+                redis_url=redis_url,
+                vectorizer=cache_vectorizer,
+                distance_threshold=self.distance_threshold,
+                filterable_fields=[self._cache_key_filterable_field()],
+                overwrite=False,
+            )
+        except ValueError as exc:
+            if not _is_schema_mismatch(exc):
+                raise
+
+            isolated_index_name = f"{index_name}_isolated"
+            print_verbose(
+                "Redis semantic-cache existing index schema is not isolated; "
+                f"using isolated index - {isolated_index_name}"
+            )
+            try:
+                return semantic_cache_cls(
+                    name=isolated_index_name,
+                    redis_url=redis_url,
+                    vectorizer=cache_vectorizer,
+                    distance_threshold=self.distance_threshold,
+                    filterable_fields=[self._cache_key_filterable_field()],
+                    overwrite=False,
+                )
+            except ValueError as isolated_exc:
+                if not _is_schema_mismatch(isolated_exc):
+                    raise
+
+                print_verbose(
+                    "Redis semantic-cache isolated index schema is stale; "
+                    f"recreating isolated index - {isolated_index_name}"
+                )
+                return semantic_cache_cls(
+                    name=isolated_index_name,
+                    redis_url=redis_url,
+                    vectorizer=cache_vectorizer,
+                    distance_threshold=self.distance_threshold,
+                    filterable_fields=[self._cache_key_filterable_field()],
+                    overwrite=True,
+                )
+
+    def _get_cache_filters(self, key: str) -> Dict[str, str]:
+        return {self.CACHE_KEY_FIELD_NAME: str(key)}
+
+    def _get_cache_key_filter_expression(self, key: str) -> Any:
+        from redisvl.query.filter import Tag  # type: ignore[import-not-found, import-untyped]
+
+        return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)
+
+    def _cache_hit_matches_key(self, cache_hit: Dict[str, Any], key: str) -> bool:
+        # Pre-isolation entries with no ``litellm_cache_key`` field cannot be
+        # safely reassigned to a caller's scope and are treated as misses.
+        cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME)
+        if isinstance(cached_key, bytes):
+            cached_key = cached_key.decode("utf-8")
+        return cached_key is not None and str(cached_key) == str(key)
+
     def _get_ttl(self, **kwargs) -> Optional[int]:
         """
         Get the TTL (time-to-live) value for cache entries.
@@ -188,7 +269,7 @@ class RedisSemanticCache(BaseCache):
         Store a value in the semantic cache.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             value: The response value to cache
             **kwargs: Additional arguments including 'messages' for the prompt
                 and optional 'ttl' for time-to-live
@@ -206,12 +287,15 @@ class RedisSemanticCache(BaseCache):
             prompt = get_str_from_messages(messages)
             value_str = str(value)
 
+            store_kwargs: Dict[str, Any] = {
+                "filters": self._get_cache_filters(key),
+            }
+
             # Get TTL and store in Redis semantic cache
             ttl = self._get_ttl(**kwargs)
             if ttl is not None:
-                self.llmcache.store(prompt, value_str, ttl=int(ttl))
-            else:
-                self.llmcache.store(prompt, value_str)
+                store_kwargs["ttl"] = int(ttl)
+            self.llmcache.store(prompt, value_str, **store_kwargs)
         except Exception as e:
             print_verbose(
                 f"Error setting {value_str or value} in the Redis semantic cache: {str(e)}"
@@ -222,7 +306,7 @@ class RedisSemanticCache(BaseCache):
         Retrieve a semantically similar cached response.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             **kwargs: Additional arguments including 'messages' for the prompt
 
         Returns:
@@ -235,18 +319,29 @@ class RedisSemanticCache(BaseCache):
             messages = kwargs.get("messages", [])
             if not messages:
                 print_verbose("No messages provided for semantic cache lookup")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             prompt = get_str_from_messages(messages)
-            # Check the cache for semantically similar prompts
-            results = self.llmcache.check(prompt=prompt)
+            # Check the cache for semantically similar prompts in this exact
+            # LiteLLM cache-key scope.
+            check_kwargs: Dict[str, Any] = {
+                "prompt": prompt,
+                "filter_expression": self._get_cache_key_filter_expression(key),
+            }
+            results = self.llmcache.check(**check_kwargs)
 
             # Return None if no similar prompts found
             if not results:
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             # Process the best matching result
             cache_hit = results[0]
+            if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
+                print_verbose("Redis semantic-cache hit did not match cache key scope")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+                return None
             vector_distance = float(cache_hit["vector_distance"])
 
             # Convert vector distance back to similarity score
@@ -257,6 +352,9 @@ class RedisSemanticCache(BaseCache):
             cached_prompt = cache_hit["prompt"]
             cached_response = cache_hit["response"]
 
+            # update kwargs["metadata"] with similarity, don't rewrite the original metadata
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = similarity
+
             print_verbose(
                 f"Cache hit: similarity threshold: {self.similarity_threshold}, "
                 f"actual similarity: {similarity}, "
@@ -267,6 +365,7 @@ class RedisSemanticCache(BaseCache):
             return self._get_cache_logic(cached_response=cached_response)
         except Exception as e:
             print_verbose(f"Error retrieving from Redis semantic cache: {str(e)}")
+            kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
 
     async def _get_async_embedding(self, prompt: str, **kwargs) -> List[float]:
         """
@@ -321,7 +420,7 @@ class RedisSemanticCache(BaseCache):
         Asynchronously store a value in the semantic cache.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             value: The response value to cache
             **kwargs: Additional arguments including 'messages' for the prompt
                 and optional 'ttl' for time-to-live
@@ -341,21 +440,20 @@ class RedisSemanticCache(BaseCache):
             # Generate embedding for the value (response) to cache
             prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
 
+            store_kwargs: Dict[str, Any] = {
+                "vector": prompt_embedding,
+                "filters": self._get_cache_filters(key),
+            }
+
             # Get TTL and store in Redis semantic cache
             ttl = self._get_ttl(**kwargs)
             if ttl is not None:
-                await self.llmcache.astore(
-                    prompt,
-                    value_str,
-                    vector=prompt_embedding,  # Pass through custom embedding
-                    ttl=ttl,
-                )
-            else:
-                await self.llmcache.astore(
-                    prompt,
-                    value_str,
-                    vector=prompt_embedding,  # Pass through custom embedding
-                )
+                store_kwargs["ttl"] = ttl
+            await self.llmcache.astore(
+                prompt,
+                value_str,
+                **store_kwargs,
+            )
         except Exception as e:
             print_verbose(f"Error in async_set_cache: {str(e)}")
 
@@ -364,7 +462,7 @@ class RedisSemanticCache(BaseCache):
         Asynchronously retrieve a semantically similar cached response.
 
         Args:
-            key: The cache key (not directly used in semantic caching)
+            key: The cache key used to isolate semantic cache entries
             **kwargs: Additional arguments including 'messages' for the prompt
 
         Returns:
@@ -385,17 +483,25 @@ class RedisSemanticCache(BaseCache):
             # Generate embedding for the prompt
             prompt_embedding = await self._get_async_embedding(prompt, **kwargs)
 
-            # Check the cache for semantically similar prompts
-            results = await self.llmcache.acheck(prompt=prompt, vector=prompt_embedding)
+            # Check the cache for semantically similar prompts in this exact
+            # LiteLLM cache-key scope.
+            check_kwargs: Dict[str, Any] = {
+                "prompt": prompt,
+                "vector": prompt_embedding,
+                "filter_expression": self._get_cache_key_filter_expression(key),
+            }
+            results = await self.llmcache.acheck(**check_kwargs)
 
             # handle results / cache hit
             if not results:
-                kwargs.setdefault("metadata", {})[
-                    "semantic-similarity"
-                ] = 0.0  # TODO why here but not above??
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
             cache_hit = results[0]
+            if not self._cache_hit_matches_key(cache_hit=cache_hit, key=key):
+                print_verbose("Redis semantic-cache hit did not match cache key scope")
+                kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
+                return None
             vector_distance = float(cache_hit["vector_distance"])
 
             # Convert vector distance back to similarity

litellm/constants.py

@@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
     | (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
 )
 
+# MCP OAuth2 Token Exchange (OBO) Defaults
+MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
+    os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
+)
+
 LITELLM_UI_ALLOW_HEADERS = [
     "x-litellm-semantic-filter",
     "x-litellm-semantic-filter-tools",
@@ -202,6 +207,12 @@ DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = int(
 DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096)
 )
+DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192)
+)
+DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384)
+)
 MAX_TOKEN_TRIMMING_ATTEMPTS = int(
     os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10)
 )  # Maximum number of attempts to trim the message
@@ -224,6 +235,16 @@ AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
 )
 AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
 AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
+# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs
+# whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT
+# Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing
+# during a long provider call and the NAT reaps the flow before the response
+# arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset
+# the NAT idle timer.
+AIOHTTP_SO_KEEPALIVE = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true"
+AIOHTTP_TCP_KEEPIDLE = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60))
+AIOHTTP_TCP_KEEPINTVL = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30))
+AIOHTTP_TCP_KEEPCNT = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5))
 # enable_cleanup_closed is only needed for Python versions with the SSL leak bug
 # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
 # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
@@ -389,6 +410,8 @@ BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
 BEDROCK_MIN_THINKING_BUDGET_TOKENS = int(
     os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
 )
+# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
+ANTHROPIC_MIN_THINKING_BUDGET_TOKENS = 1024
 REPLICATE_POLLING_DELAY_SECONDS = float(
     os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
 )
@@ -409,9 +432,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0
 AUDIO_SPEECH_CHUNK_SIZE = int(
     os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
 )  # chunk_size for audio speech streaming. Balance between latency and memory usage
-MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
-    os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
-)
 DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
 #### Networking settings ####
 # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
@@ -1383,6 +1403,10 @@ except (ValueError, TypeError):
 LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
 LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
 LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
+# Stable identifier substituted in place of the master key on UserAPIKeyAuth
+# objects so the master key (or its hash) never propagates to spend logs,
+# Prometheus metrics, audit trails, or any other downstream consumer.
+LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
 
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
@@ -1396,12 +1420,22 @@ LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS = int(
     os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600)
 )  # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation
 UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
+LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED = os.getenv(
+    "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED", "false"
+)
+LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS = int(
+    os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400)
+)  # 24 hours default
+LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE = int(
+    os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000)
+)
 LITELLM_PROXY_ADMIN_NAME = "default_user_id"
 
 ########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
 LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
 LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
+CLI_SSO_SESSION_TTL_SECONDS = 600
 CLI_JWT_TOKEN_NAME = "cli-jwt-token"
 # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
 CLI_JWT_EXPIRATION_HOURS = int(
@@ -1425,8 +1459,15 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
 )
 SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
 KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
+EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
+SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
+    os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
+)
+SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
+    os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
+)
 SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(

litellm/cost_calculator.py

@@ -513,7 +513,10 @@ def cost_per_token(  # noqa: PLR0915
         return fireworks_ai_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "azure":
         return azure_openai_cost_per_token(
-            model=model, usage=usage_block, response_time_ms=response_time_ms
+            model=model,
+            usage=usage_block,
+            response_time_ms=response_time_ms,
+            service_tier=service_tier,
         )
     elif custom_llm_provider == "gemini":
         return gemini_cost_per_token(
@@ -539,6 +542,7 @@ def cost_per_token(  # noqa: PLR0915
             usage=usage_block,
             response_time_ms=response_time_ms,
             request_model=request_model,
+            service_tier=service_tier,
         )
     else:
         model_info = _cached_get_model_info_helper(

litellm/experimental_mcp_client/client.py

@@ -366,6 +366,8 @@ class MCPClient:
                     headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.token:
                     headers["Authorization"] = f"token {self._mcp_auth_value}"
+                elif self.auth_type == MCPAuth.oauth2_token_exchange:
+                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

litellm/files/main.py

@@ -10,6 +10,7 @@ import contextvars
 import time
 import uuid as uuid_module
 from functools import partial
+from types import MappingProxyType
 from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 
 import httpx
@@ -85,6 +86,16 @@ bedrock_files_instance = BedrockFilesHandler()
 #################################################
 
 
+def _add_trusted_model_credentials_to_litellm_params(
+    litellm_params_dict: Dict[str, Any], kwargs: Dict[str, Any]
+) -> None:
+    trusted_model_credentials = kwargs.get("_litellm_internal_model_credentials")
+    if isinstance(trusted_model_credentials, type(MappingProxyType({}))):
+        litellm_params_dict["_litellm_internal_model_credentials"] = (
+            trusted_model_credentials
+        )
+
+
 @client
 async def acreate_file(
     file: FileTypes,
@@ -373,6 +384,10 @@ def file_retrieve(
             )
             if provider_config is not None:
                 litellm_params_dict = get_litellm_params(**kwargs)
+                _add_trusted_model_credentials_to_litellm_params(
+                    litellm_params_dict=litellm_params_dict,
+                    kwargs=kwargs,
+                )
                 litellm_params_dict["api_key"] = optional_params.api_key
                 litellm_params_dict["api_base"] = optional_params.api_base
 
@@ -497,6 +512,10 @@ def file_delete(
             pass
         optional_params = GenericLiteLLMParams(**kwargs)
         litellm_params_dict = get_litellm_params(**kwargs)
+        _add_trusted_model_credentials_to_litellm_params(
+            litellm_params_dict=litellm_params_dict,
+            kwargs=kwargs,
+        )
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         # set timeout for 10 minutes by default
@@ -846,6 +865,10 @@ def file_content(
     try:
         optional_params = GenericLiteLLMParams(**kwargs)
         litellm_params_dict = get_litellm_params(**kwargs)
+        _add_trusted_model_credentials_to_litellm_params(
+            litellm_params_dict=litellm_params_dict,
+            kwargs=kwargs,
+        )
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         client = kwargs.get("client")
@@ -993,6 +1016,7 @@ def file_content(
                 vertex_location=vertex_ai_location,
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
+                litellm_params=litellm_params_dict,
             )
         elif custom_llm_provider == "bedrock":
             response = bedrock_files_instance.file_content(

litellm/google_genai/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.success_handler import (
@@ -29,12 +29,14 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
         litellm_logging_obj: LiteLLMLoggingObj,
         request_body: dict,
         model: str,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         self.litellm_logging_obj = litellm_logging_obj
         self.request_body = request_body
         self.start_time = datetime.now()
         self.collected_chunks: List[bytes] = []
         self.model = model
+        self._hidden_params: Dict[str, Any] = hidden_params or {}
 
     async def _handle_async_streaming_logging(
         self,
@@ -76,11 +78,13 @@ class GoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model
@@ -130,11 +134,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model

litellm/integrations/arize/_utils.py

@@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
             safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role)
 
 
+def _safe_get(obj, key, default=None):
+    """Read ``key`` from a dict-like or Pydantic-model-like object.
+
+    The arize/langfuse_otel logger receives ``usage`` objects from many sources:
+    plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI
+    Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and
+    nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT
+    expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` —
+    see https://github.com/BerriAI/litellm/issues/13672.
+    """
+    if obj is None:
+        return default
+    getter = getattr(obj, "get", None)
+    if callable(getter):
+        try:
+            return getter(key, default)
+        except TypeError:
+            # Some objects expose `.get` with a different signature
+            pass
+    return getattr(obj, key, default)
+
+
 def _set_usage_outputs(span: "Span", response_obj, span_attrs):
     usage = response_obj and response_obj.get("usage")
     if not usage:
         return
 
     safe_set_attribute(
-        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")
+        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens")
+    )
+    completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get(
+        usage, "output_tokens"
     )
-    completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens")
     if completion_tokens:
         safe_set_attribute(
             span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens
         )
-    prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens")
+    prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get(
+        usage, "input_tokens"
+    )
     if prompt_tokens:
         safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens)
-    reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens")
+
+    # Reasoning tokens live in `completion_tokens_details` for Chat Completions
+    # API (Usage) and in `output_tokens_details` for Responses API
+    # (ResponseAPIUsage). Both nested objects may be plain Pydantic models
+    # without `.get`.
+    token_details = _safe_get(usage, "completion_tokens_details") or _safe_get(
+        usage, "output_tokens_details"
+    )
+    reasoning_tokens = _safe_get(token_details, "reasoning_tokens")
     if reasoning_tokens:
         safe_set_attribute(
             span,

litellm/integrations/arize/arize_phoenix_client.py

@@ -2,11 +2,23 @@
 Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 """
 
+import urllib.parse
 from typing import Any, Dict, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_id(identifier: str) -> str:
+    """Reject path traversal characters and URL-encode the identifier."""
+    if any(c in identifier for c in ("/", "\\", "#", "?")):
+        raise ValueError(
+            f"Invalid identifier {identifier!r}: contains disallowed characters"
+        )
+    if ".." in identifier:
+        raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
+    return urllib.parse.quote(identifier, safe="")
+
+
 class ArizePhoenixClient:
     """
     Client for interacting with Arize Phoenix API to fetch prompt versions.
@@ -53,7 +65,8 @@ class ArizePhoenixClient:
         Returns:
             Dictionary containing prompt version data, or None if not found
         """
-        url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
+        safe_id = _sanitize_id(prompt_version_id)
+        url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
 
         try:
             # Use the underlying httpx client directly to avoid query param extraction

litellm/integrations/arize/arize_phoenix_prompt_manager.py

@@ -5,7 +5,8 @@ Fetches prompt versions from Arize Phoenix and provides workspace-based access c
 
 from typing import Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import (
@@ -74,7 +75,13 @@ class ArizePhoenixTemplateManager:
             api_key=self.api_key, api_base=self.api_base
         )
 
-        self.jinja_env = Environment(
+        # Templates fetched from Arize Phoenix come from external workspace
+        # users; in a plain `Environment()` a malicious template could reach
+        # `__class__.__init__.__globals__` and execute arbitrary code on the
+        # proxy host. The sandbox blocks that attribute traversal while
+        # leaving normal `{{ var }}` substitution intact. Matches the
+        # dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             # Use Mustache/Handlebars-style delimiters

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
 
 import asyncio
 import os
+import time
 import traceback
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.types.utils import StandardLoggingPayload
+from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 
 class AzureSentinelLogger(CustomBatchLogger):
@@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         tenant_id: Optional[str] = None,
         client_id: Optional[str] = None,
         client_secret: Optional[str] = None,
+        audit_stream_name: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
                 If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
             client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
                 If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
+            audit_stream_name (str, optional): Stream name from DCR for audit logs.
+                If not provided, audit logs use the standard stream name.
         """
         self.async_httpx_client = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.LoggingCallback
         )
 
-        self.dcr_immutable_id = dcr_immutable_id or os.getenv(
+        resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
             "AZURE_SENTINEL_DCR_IMMUTABLE_ID"
         )
-        self.stream_name = stream_name or os.getenv(
-            "AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
+        resolved_stream_name = (
+            stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
         )
-        self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
-        self.tenant_id = (
+        resolved_audit_stream_name = audit_stream_name or resolved_stream_name
+        resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
+        resolved_tenant_id = (
             tenant_id
             or os.getenv("AZURE_SENTINEL_TENANT_ID")
             or os.getenv("AZURE_TENANT_ID")
         )
-        self.client_id = (
+        resolved_client_id = (
             client_id
             or os.getenv("AZURE_SENTINEL_CLIENT_ID")
             or os.getenv("AZURE_CLIENT_ID")
         )
-        self.client_secret = (
+        resolved_client_secret = (
             client_secret
             or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
             or os.getenv("AZURE_CLIENT_SECRET")
         )
 
-        if not self.dcr_immutable_id:
+        if not resolved_dcr_immutable_id:
             raise ValueError(
                 "AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
             )
-        if not self.endpoint:
+        if not resolved_endpoint:
             raise ValueError(
                 "AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
             )
-        if not self.tenant_id:
+        if not resolved_tenant_id:
             raise ValueError(
                 "AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
             )
-        if not self.client_id:
+        if not resolved_client_id:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
             )
-        if not self.client_secret:
+        if not resolved_client_secret:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
             )
 
+        self.dcr_immutable_id = resolved_dcr_immutable_id
+        self.stream_name = resolved_stream_name
+        self.audit_stream_name = resolved_audit_stream_name
+        self.endpoint = resolved_endpoint
+        self.tenant_id = resolved_tenant_id
+        self.client_id = resolved_client_id
+        self.client_secret = resolved_client_secret
+
         # Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
-        self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
+        self.api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_stream_name,
+        )
+        self.audit_api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_audit_stream_name,
+        )
 
         # OAuth2 scope for Azure Monitor
         self.oauth_scope = "https://monitor.azure.com/.default"
@@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
         super().__init__(**kwargs, flush_lock=self.flush_lock)
         asyncio.create_task(self.periodic_flush())
         self.log_queue: List[StandardLoggingPayload] = []
+        self.audit_log_queue: List[StandardAuditLogPayload] = []
+
+    @staticmethod
+    def _build_api_endpoint(
+        endpoint: str, dcr_immutable_id: str, stream_name: str
+    ) -> str:
+        return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
 
     async def _get_oauth_token(self) -> str:
         """
@@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         Returns:
             Bearer token string
         """
-        # Check if we have a valid cached token
-        import time
-
         if (
             self.oauth_token
             and self.oauth_token_expires_at
@@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         if not self.oauth_token:
             raise Exception("OAuth2 token response did not contain access_token")
 
-        # Cache token expiry time
-        import time
-
         self.oauth_token_expires_at = time.time() + expires_in
 
         return self.oauth_token
@@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
             )
             pass
 
+    async def async_log_audit_log_event(
+        self, audit_log: StandardAuditLogPayload
+    ) -> None:
+        """
+        Async log LiteLLM audit log events to Azure Sentinel.
+
+        Audit logs are queued separately from standard LLM logs so mixed callback
+        usage never sends schema-mismatched records in the same ingestion batch.
+        """
+        try:
+            verbose_logger.debug(
+                "Azure Sentinel: Logging audit event id=%s action=%s table=%s",
+                audit_log.get("id"),
+                audit_log.get("action"),
+                audit_log.get("table_name"),
+            )
+
+            self.audit_log_queue.append(audit_log)
+
+            if len(self.audit_log_queue) >= self.batch_size:
+                await self.async_send_audit_batch()
+
+        except Exception as e:
+            verbose_logger.exception(
+                f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
+            )
+            pass
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
+        await self._async_send_batch_to_api(
+            log_queue=self.log_queue,
+            api_endpoint=self.api_endpoint,
+            log_type="logs",
+        )
+
+    async def async_send_audit_batch(self):
+        """
+        Sends the batch of audit logs to Azure Monitor Logs Ingestion API
+        """
+        await self._async_send_batch_to_api(
+            log_queue=self.audit_log_queue,
+            api_endpoint=self.audit_api_endpoint,
+            log_type="audit logs",
+        )
+
+    async def _async_send_batch_to_api(
+        self,
+        log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
+        api_endpoint: str,
+        log_type: str,
+    ) -> None:
         try:
-            if not self.log_queue:
+            if not log_queue:
                 return
 
             verbose_logger.debug(
-                "Azure Sentinel - about to flush %s events", len(self.log_queue)
+                "Azure Sentinel - about to flush %s %s", len(log_queue), log_type
             )
 
-            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-
             # Get OAuth2 token
             bearer_token = await self._get_oauth_token()
 
             # Convert log queue to JSON array format expected by Logs Ingestion API
             # Each log entry should be a JSON object in the array
-            body = safe_dumps(self.log_queue)
+            body = safe_dumps(log_queue)
 
             # Set headers for Logs Ingestion API
             headers = {
@@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             # Send the request
             response = await self.async_httpx_client.post(
-                url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
+                url=api_endpoint, data=body.encode("utf-8"), headers=headers
             )
 
             if response.status_code not in [200, 204]:
@@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
                 f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
             )
         finally:
-            self.log_queue.clear()
+            log_queue.clear()
+
+    async def flush_queue(self):
+        if self.flush_lock is None:
+            return
+
+        async with self.flush_lock:
+            if self.log_queue:
+                await self.async_send_batch()
+            if self.audit_log_queue:
+                await self.async_send_audit_batch()
+            self.last_flush_time = time.time()

litellm/integrations/bitbucket/bitbucket_client.py

@@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
 """
 
 import base64
+import urllib.parse
 from typing import Any, Dict, List, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_file_path(file_path: str) -> str:
+    """Reject path traversal and URL-encode each path segment."""
+    if "#" in file_path or "?" in file_path:
+        raise ValueError(
+            f"Invalid file path {file_path!r}: contains URL special characters"
+        )
+    parts = file_path.split("/")
+    for part in parts:
+        if part == "..":
+            raise ValueError(
+                f"Invalid file path {file_path!r}: path traversal detected"
+            )
+    return "/".join(urllib.parse.quote(part, safe="") for part in parts)
+
+
 class BitBucketClient:
     """
     Client for interacting with BitBucket API to fetch .prompt files.
@@ -72,7 +88,8 @@ class BitBucketClient:
         Returns:
             File content as string, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -119,7 +136,8 @@ class BitBucketClient:
         Returns:
             List of file paths
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
+        safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -211,7 +229,8 @@ class BitBucketClient:
         Returns:
             Dictionary containing file metadata, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             # Use GET with Range header to get just the headers (HEAD equivalent)

litellm/integrations/bitbucket/bitbucket_prompt_manager.py

@@ -5,7 +5,8 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 
@@ -74,7 +75,13 @@ class BitBucketTemplateManager:
         self.prompts: Dict[str, BitBucketPromptTemplate] = {}
         self.bitbucket_client = BitBucketClient(bitbucket_config)
 
-        self.jinja_env = Environment(
+        # Templates fetched from a BitBucket repo are not trustworthy:
+        # anyone with repo write access can ship Jinja syntax that, in a
+        # plain `Environment()`, would reach `__class__.__init__.__globals__`
+        # and pivot into RCE on the proxy host. The sandbox blocks that
+        # attribute traversal while leaving normal `{{ var }}` substitution
+        # intact. Matches the dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             # Use Handlebars-style delimiters to match Dotprompt spec

litellm/integrations/custom_sso_handler.py

@@ -18,6 +18,17 @@ class CustomSSOLoginHandler(CustomLogger):
         self,
         request: Request,
     ) -> OpenID:
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
+        from litellm.proxy.proxy_server import general_settings
+
+        require_trusted_proxy_request(
+            request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
+        )
+
         request_headers_dict = dict(request.headers)
         return OpenID(
             id=request_headers_dict.get("x-litellm-user-id"),

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -6,12 +6,14 @@ import time
 from litellm._uuid import uuid
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
-from urllib.parse import quote
 
 from litellm._logging import verbose_logger
 from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
 from litellm.integrations.additional_logging_utils import AdditionalLoggingUtils
 from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
+from litellm.litellm_core_utils.cloud_storage_security import (
+    sanitize_cloud_object_component,
+)
 from litellm.proxy._types import CommonProxyErrors
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
@@ -335,7 +337,11 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
         _litellm_params = kwargs.get("litellm_params", None) or {}
         _metadata = _litellm_params.get("metadata", None) or {}
         if "gcs_log_id" in _metadata:
-            object_name = _metadata["gcs_log_id"]
+            safe_log_id = sanitize_cloud_object_component(
+                _metadata.get("gcs_log_id"), fallback=""
+            )
+            if safe_log_id:
+                object_name = f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
 
         return object_name
 
@@ -367,8 +373,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
                     request_date_str=date_str,
                     response_id=request_id,
                 )
-                encoded_object_name = quote(object_name, safe="")
-                response = await self.download_gcs_object(encoded_object_name)
+                response = await self.download_gcs_object(object_name)
 
                 if response is not None:
                     loaded_response = json.loads(response)

litellm/integrations/gcs_bucket/gcs_bucket_base.py

@@ -11,6 +11,10 @@ from litellm.integrations.gcs_bucket.gcs_bucket_mock_client import (
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.cloud_storage_security import (
+    encode_gcs_object_name_for_url,
+    split_configured_cloud_bucket_name,
+)
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -133,8 +137,8 @@ class GCSBucketBase(CustomBatchLogger):
             - Returns: bucket_name="my-bucket", object_name="my-folder/dev/my-object"
 
         """
-        if "/" in bucket_name:
-            bucket_name, prefix = bucket_name.split("/", 1)
+        bucket_name, prefix = split_configured_cloud_bucket_name(bucket_name)
+        if prefix:
             object_name = f"{prefix}/{object_name}"
             return bucket_name, object_name
         return bucket_name, object_name
@@ -248,6 +252,7 @@ class GCSBucketBase(CustomBatchLogger):
                 bucket_name=bucket_name,
                 object_name=object_name,
             )
+            object_name = encode_gcs_object_name_for_url(object_name)
 
             url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}?alt=media"
 
@@ -288,6 +293,7 @@ class GCSBucketBase(CustomBatchLogger):
                 bucket_name=bucket_name,
                 object_name=object_name,
             )
+            object_name = encode_gcs_object_name_for_url(object_name)
 
             url = f"https://storage.googleapis.com/storage/v1/b/{bucket_name}/o/{object_name}"
 
@@ -334,10 +340,11 @@ class GCSBucketBase(CustomBatchLogger):
             bucket_name=bucket_name,
             object_name=object_name,
         )
+        encoded_object_name = encode_gcs_object_name_for_url(object_name)
 
         response = await self.async_httpx_client.post(
             headers=headers,
-            url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}",
+            url=f"https://storage.googleapis.com/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={encoded_object_name}",
             data=json_logged_payload,
         )
 

litellm/integrations/generic_api/generic_api_callback.py

@@ -11,8 +11,9 @@ import json
 import os
 import re
 import traceback
-from typing import Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
+import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
@@ -103,6 +104,9 @@ class GenericAPILogger(CustomBatchLogger):
         event_types: Optional[List[API_EVENT_TYPES]] = None,
         callback_name: Optional[str] = None,
         log_format: Optional[LOG_FORMAT_TYPES] = None,
+        max_retries: int = 0,
+        retry_delay: float = 1.0,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
         **kwargs,
     ):
         """
@@ -114,6 +118,9 @@ class GenericAPILogger(CustomBatchLogger):
             event_types: Optional[List[API_EVENT_TYPES]] = None,
             callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
             log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
+            max_retries: Number of retry attempts after the initial request fails. Defaults to 0.
+            retry_delay: Initial retry delay in seconds. Retries use exponential backoff.
+            timeout: Optional timeout to use for Generic API callback requests.
         """
         #########################################################
         # Check if callback_name is provided and load config
@@ -162,6 +169,10 @@ class GenericAPILogger(CustomBatchLogger):
         self.endpoint: str = endpoint
         self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
         self.callback_name: Optional[str] = callback_name
+        self.max_retries = max(0, int(max_retries or 0))
+        retry_delay_value = 0.0 if retry_delay is None else retry_delay
+        self.retry_delay = max(0.0, float(retry_delay_value))
+        self.timeout = timeout
 
         # Validate and store log_format
         if log_format is not None and log_format not in [
@@ -226,6 +237,53 @@ class GenericAPILogger(CustomBatchLogger):
 
         return headers_dict
 
+    def _should_retry_exception(self, exception: Exception) -> bool:
+        if isinstance(exception, (litellm.Timeout, httpx.TransportError)):
+            return True
+
+        if isinstance(exception, httpx.HTTPStatusError):
+            return exception.response.status_code >= 500
+
+        return False
+
+    async def _sleep_before_retry(self, attempt: int) -> None:
+        if self.retry_delay <= 0:
+            return
+
+        delay = self.retry_delay * (2**attempt)
+        await asyncio.sleep(delay)
+
+    async def _post_with_retries(self, data: str) -> httpx.Response:
+        post_kwargs: Dict[str, Any] = {
+            "url": self.endpoint,
+            "headers": self.headers,
+            "data": data,
+        }
+        if self.timeout is not None:
+            post_kwargs["timeout"] = self.timeout
+
+        total_attempts = self.max_retries + 1
+        for attempt in range(total_attempts):
+            try:
+                return await self.async_httpx_client.post(**post_kwargs)
+            except Exception as e:
+                is_last_attempt = attempt == self.max_retries
+                should_retry = self._should_retry_exception(e)
+                if is_last_attempt or not should_retry:
+                    raise
+
+                verbose_logger.warning(
+                    "Generic API Logger - retrying request to %s after error: %s "
+                    "(attempt %s/%s)",
+                    self.endpoint,
+                    str(e),
+                    attempt + 1,
+                    total_attempts,
+                )
+                await self._sleep_before_retry(attempt)
+
+        raise RuntimeError("Generic API Logger retry loop exited unexpectedly")
+
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         """
         Async Log success events to Generic API Endpoint
@@ -325,11 +383,7 @@ class GenericAPILogger(CustomBatchLogger):
                 # Send each log as individual HTTP request in parallel
                 tasks = []
                 for log_entry in self.log_queue:
-                    task = self.async_httpx_client.post(
-                        url=self.endpoint,
-                        headers=self.headers,
-                        data=safe_dumps(log_entry),
-                    )
+                    task = self._post_with_retries(data=safe_dumps(log_entry))
                     tasks.append(task)
 
                 # Execute all requests in parallel
@@ -356,11 +410,7 @@ class GenericAPILogger(CustomBatchLogger):
                     raise ValueError(f"Unknown log_format: {self.log_format}")
 
                 # Make POST request
-                response = await self.async_httpx_client.post(
-                    url=self.endpoint,
-                    headers=self.headers,
-                    data=data,
-                )
+                response = await self._post_with_retries(data=data)
 
                 verbose_logger.debug(
                     f"Generic API Logger - sent batch to {self.endpoint}, "

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -4,7 +4,8 @@ GitLab prompt manager with configurable prompts folder.
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
-from jinja2 import DictLoader, Environment, select_autoescape
+from jinja2 import DictLoader, select_autoescape
+from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 
@@ -90,7 +91,13 @@ class GitLabTemplateManager:
             or ""
         ).strip("/")
 
-        self.jinja_env = Environment(
+        # Templates fetched from a GitLab repo are not trustworthy:
+        # anyone with repo write access can ship Jinja syntax that, in a
+        # plain `Environment()`, would reach `__class__.__init__.__globals__`
+        # and pivot into RCE on the proxy host. The sandbox blocks that
+        # attribute traversal while leaving normal `{{ var }}` substitution
+        # intact. Matches the dotprompt manager's hardening.
+        self.jinja_env = ImmutableSandboxedEnvironment(
             loader=DictLoader({}),
             autoescape=select_autoescape(["html", "xml"]),
             variable_start_string="{{",

litellm/integrations/langfuse/langfuse.py

@@ -90,6 +90,29 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
     return cache_read_input_tokens
 
 
+def resolve_langfuse_credentials(
+    langfuse_public_key=None,
+    langfuse_secret=None,
+    langfuse_secret_key=None,
+    langfuse_host=None,
+    allow_env_credentials: bool = True,
+):
+    if allow_env_credentials is False and langfuse_host is not None:
+        secret_key = langfuse_secret or langfuse_secret_key
+        public_key = langfuse_public_key
+    else:
+        secret_key = (
+            langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
+        )
+        public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
+
+    resolved_host = langfuse_host or os.getenv(
+        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    )
+
+    return public_key, secret_key, resolved_host
+
+
 class LangFuseLogger:
     # Class variables or attributes
     def __init__(
@@ -98,6 +121,7 @@ class LangFuseLogger:
         langfuse_secret=None,
         langfuse_host=None,
         flush_interval=1,
+        allow_env_credentials: bool = True,
     ):
         try:
             import langfuse
@@ -106,11 +130,13 @@ class LangFuseLogger:
             raise Exception(
                 f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n{traceback.format_exc()}\033[0m"
             )
-        # Instance variables
-        self.secret_key = langfuse_secret or os.getenv("LANGFUSE_SECRET_KEY")
-        self.public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-        self.langfuse_host = langfuse_host or os.getenv(
-            "LANGFUSE_HOST", "https://cloud.langfuse.com"
+        self.public_key, self.secret_key, self.langfuse_host = (
+            resolve_langfuse_credentials(
+                langfuse_public_key=langfuse_public_key,
+                langfuse_secret=langfuse_secret,
+                langfuse_host=langfuse_host,
+                allow_env_credentials=allow_env_credentials,
+            )
         )
         if not (
             self.langfuse_host.startswith("http://")
@@ -160,9 +186,10 @@ class LangFuseLogger:
                 project_id = None
 
         if os.getenv("UPSTREAM_LANGFUSE_SECRET_KEY") is not None:
+            upstream_langfuse_debug_env = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
             upstream_langfuse_debug = (
-                str_to_bool(self.upstream_langfuse_debug)
-                if self.upstream_langfuse_debug is not None
+                str_to_bool(upstream_langfuse_debug_env)
+                if upstream_langfuse_debug_env is not None
                 else None
             )
             self.upstream_langfuse_secret_key = os.getenv(
@@ -173,7 +200,7 @@ class LangFuseLogger:
             )
             self.upstream_langfuse_host = os.getenv("UPSTREAM_LANGFUSE_HOST")
             self.upstream_langfuse_release = os.getenv("UPSTREAM_LANGFUSE_RELEASE")
-            self.upstream_langfuse_debug = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
+            self.upstream_langfuse_debug = upstream_langfuse_debug_env
             self.upstream_langfuse = Langfuse(
                 public_key=self.upstream_langfuse_public_key,
                 secret_key=self.upstream_langfuse_secret_key,

litellm/integrations/langfuse/langfuse_handler.py

@@ -115,8 +115,10 @@ class LangFuseHandler:
 
         langfuse_logger = LangFuseLogger(
             langfuse_public_key=credentials.get("langfuse_public_key"),
-            langfuse_secret=credentials.get("langfuse_secret"),
+            langfuse_secret=credentials.get("langfuse_secret")
+            or credentials.get("langfuse_secret_key"),
             langfuse_host=credentials.get("langfuse_host"),
+            allow_env_credentials=credentials.get("langfuse_host") is None,
         )
         in_memory_dynamic_logger_cache.set_cache(
             credentials=credentials,

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -20,7 +20,7 @@ from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import (
     DynamicLoggingCache,
 )
 from ..prompt_management_base import PromptManagementBase
-from .langfuse import LangFuseLogger
+from .langfuse import LangFuseLogger, resolve_langfuse_credentials
 from .langfuse_handler import LangFuseHandler
 
 if TYPE_CHECKING:
@@ -46,6 +46,7 @@ def langfuse_client_init(
     langfuse_secret_key=None,
     langfuse_host=None,
     flush_interval=1,
+    allow_env_credentials: bool = True,
 ) -> LangfuseClass:
     """
     Initialize Langfuse client with caching to prevent multiple initializations.
@@ -70,14 +71,12 @@ def langfuse_client_init(
             f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n\033[0m"
         )
 
-    # Instance variables
-
-    secret_key = (
-        langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
-    )
-    public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-    langfuse_host = langfuse_host or os.getenv(
-        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    public_key, secret_key, langfuse_host = resolve_langfuse_credentials(
+        langfuse_public_key=langfuse_public_key,
+        langfuse_secret=langfuse_secret,
+        langfuse_secret_key=langfuse_secret_key,
+        langfuse_host=langfuse_host,
+        allow_env_credentials=allow_env_credentials,
     )
 
     if not (
@@ -222,6 +221,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,
@@ -246,6 +246,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,

litellm/integrations/langsmith.py

@@ -19,6 +19,7 @@ from litellm.integrations.langsmith_mock_client import (
     create_mock_langsmith_client,
     should_use_langsmith_mock,
 )
+from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
@@ -112,17 +113,28 @@ class LangsmithLogger(CustomBatchLogger):
         langsmith_project: Optional[str] = None,
         langsmith_base_url: Optional[str] = None,
         langsmith_tenant_id: Optional[str] = None,
+        allow_env_credentials: bool = True,
     ) -> LangsmithCredentialsObject:
-        _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
-        _credentials_project = (
-            langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
-        )
-        _credentials_base_url = (
-            langsmith_base_url
-            or os.getenv("LANGSMITH_BASE_URL")
-            or "https://api.smith.langchain.com"
-        )
-        _credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
+        if allow_env_credentials is False and langsmith_base_url is not None:
+            _credentials_api_key = langsmith_api_key
+            _credentials_project = langsmith_project or "litellm-completion"
+            _credentials_base_url = langsmith_base_url
+            _credentials_tenant_id = langsmith_tenant_id
+        else:
+            _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
+            _credentials_project = (
+                langsmith_project
+                or os.getenv("LANGSMITH_PROJECT")
+                or "litellm-completion"
+            )
+            _credentials_base_url = (
+                langsmith_base_url
+                or os.getenv("LANGSMITH_BASE_URL")
+                or "https://api.smith.langchain.com"
+            )
+            _credentials_tenant_id = langsmith_tenant_id or os.getenv(
+                "LANGSMITH_TENANT_ID"
+            )
 
         return LangsmithCredentialsObject(
             LANGSMITH_API_KEY=_credentials_api_key,
@@ -153,6 +165,15 @@ class LangsmithLogger(CustomBatchLogger):
             for key in ("session_id", "thread_id", "conversation_id"):
                 if key in requester_metadata and key not in extra_metadata:
                     extra_metadata[key] = requester_metadata[key]
+
+        # helper is shallow; also scrub nested requester_metadata since
+        # LangSmith forwards the whole dict into `extra`
+        extra_metadata = redact_user_api_key_info(metadata=extra_metadata)
+        nested = extra_metadata.get("requester_metadata")
+        if isinstance(nested, dict):
+            extra_metadata["requester_metadata"] = redact_user_api_key_info(
+                metadata=nested
+            )
         return extra_metadata
 
     def _build_outputs_with_usage(
@@ -540,6 +561,10 @@ class LangsmithLogger(CustomBatchLogger):
                 langsmith_tenant_id=standard_callback_dynamic_params.get(
                     "langsmith_tenant_id", None
                 ),
+                allow_env_credentials=standard_callback_dynamic_params.get(
+                    "langsmith_base_url", None
+                )
+                is None,
             )
         else:
             credentials = self.default_credentials

litellm/integrations/opentelemetry.py

@@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
 
+CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
+CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
+CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
+CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
+_VALID_CAPTURE_MODES = {
+    CAPTURE_MODE_NO_CONTENT,
+    CAPTURE_MODE_SPAN_ONLY,
+    CAPTURE_MODE_EVENT_ONLY,
+    CAPTURE_MODE_SPAN_AND_EVENT,
+}
+
 
 @dataclass
 class OpenTelemetryConfig:
@@ -69,6 +80,11 @@ class OpenTelemetryConfig:
     deployment_environment: Optional[str] = None
     model_id: Optional[str] = None
     ignore_context_propagation: Optional[bool] = None
+    # When True, create a private TracerProvider instead of reusing or setting the global one.
+    skip_set_global: bool = False
+    # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
+    # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
+    capture_message_content: Optional[str] = None
 
     def __post_init__(self) -> None:
         # If endpoint is specified but exporter is still the default "console",
@@ -180,6 +196,9 @@ class OpenTelemetry(CustomLogger):
         super().__init__(**kwargs)
         self._init_metrics(meter_provider)
         self._init_logs(logger_provider)
+        # Sample env-var / config / message_logging at init so subsequent
+        # _capture_in_span / _capture_in_event calls are deterministic.
+        self._capture_mode_cached = self._compute_capture_mode_from_init_state()
         self._init_otel_logger_on_litellm_proxy()
 
     @staticmethod
@@ -259,16 +278,21 @@ class OpenTelemetry(CustomLogger):
         try:
             existing_provider = get_existing_provider_fn()
 
-            # If a real SDK provider exists (set by another SDK like Langfuse), use it
-            # This uses a positive check for SDK providers instead of a negative check for proxy providers
             if isinstance(existing_provider, sdk_provider_class):
-                verbose_logger.debug(
-                    "OpenTelemetry: Using existing %s: %s",
-                    provider_name,
-                    type(existing_provider).__name__,
-                )
-                provider = existing_provider
-                # Don't call set_provider to preserve existing context
+                if skip_set_global:
+                    verbose_logger.debug(
+                        "OpenTelemetry: existing %s found but skip_set_global=True; creating private %s for isolation",
+                        provider_name,
+                        provider_name,
+                    )
+                    provider = create_new_provider_fn()
+                else:
+                    verbose_logger.debug(
+                        "OpenTelemetry: Using existing %s: %s",
+                        provider_name,
+                        type(existing_provider).__name__,
+                    )
+                    provider = existing_provider
             else:
                 # Default proxy provider or unknown type, create our own
                 verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
@@ -293,6 +317,68 @@ class OpenTelemetry(CustomLogger):
 
         return provider
 
+    def _skip_set_global(self) -> bool:
+        # langfuse_otel relies on the Langfuse SDK's providers; don't overwrite them.
+        return self.config.skip_set_global or (
+            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
+        )
+
+    def _compute_capture_mode_from_init_state(self) -> Optional[str]:
+        """Sample explicit settings at init. Returns the resolved mode or
+        None if nothing explicit is set (in which case the legacy
+        ``self.message_logging`` flag is consulted dynamically per request).
+
+        ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
+        ``"false"``/``"0"`` map to ``NO_CONTENT``.
+        Unknown values are ignored.
+        """
+        explicit = self.config.capture_message_content or os.getenv(
+            "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+        )
+        if not explicit:
+            return None
+        normalized = explicit.upper()
+        if normalized in ("TRUE", "1"):
+            return CAPTURE_MODE_EVENT_ONLY
+        if normalized in ("FALSE", "0"):
+            return CAPTURE_MODE_NO_CONTENT
+        if normalized in _VALID_CAPTURE_MODES:
+            return normalized
+        return None
+
+    def _resolve_capture_mode(self) -> str:
+        """Return the active capture mode for this request.
+
+        Precedence:
+          1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
+             (kill-switch checked dynamically).
+          2. Explicit setting sampled at init from
+             ``OpenTelemetryConfig.capture_message_content`` or
+             ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
+          3. Legacy ``self.message_logging`` (checked dynamically).
+        """
+        if litellm.turn_off_message_logging:
+            return CAPTURE_MODE_NO_CONTENT
+        if self._capture_mode_cached is not None:
+            return self._capture_mode_cached
+        return (
+            CAPTURE_MODE_SPAN_AND_EVENT
+            if self.message_logging
+            else CAPTURE_MODE_NO_CONTENT
+        )
+
+    def _capture_in_span(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_SPAN_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
+    def _capture_in_event(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_EVENT_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
     def _init_tracing(self, tracer_provider):
         from opentelemetry import trace
         from opentelemetry.sdk.trace import TracerProvider
@@ -303,11 +389,6 @@ class OpenTelemetry(CustomLogger):
             provider.add_span_processor(self._get_span_processor())
             return provider
 
-        # CRITICAL FIX: For Langfuse OTEL, skip setting global provider to prevent interference
-        skip_global = (
-            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
-        )
-
         tracer_provider = self._get_or_create_provider(
             provider=tracer_provider,
             provider_name="TracerProvider",
@@ -315,16 +396,18 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=TracerProvider,
             create_new_provider_fn=create_tracer_provider,
             set_provider_fn=trace.set_tracer_provider,
-            skip_set_global=skip_global,
+            skip_set_global=self._skip_set_global(),
         )
 
         # Grab our tracer from the TracerProvider (not from global context)
         # This ensures we use the provided TracerProvider (e.g., for testing)
         self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
+        self._tracer_provider = tracer_provider
         self.span_kind = SpanKind
 
     def _init_metrics(self, meter_provider):
         if not self.config.enable_metrics:
+            self._meter_provider = None
             self._operation_duration_histogram = None
             self._token_usage_histogram = None
             self._cost_histogram = None
@@ -350,7 +433,9 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=MeterProvider,
             create_new_provider_fn=create_meter_provider,
             set_provider_fn=metrics.set_meter_provider,
+            skip_set_global=self._skip_set_global(),
         )
+        self._meter_provider = meter_provider
 
         meter = meter_provider.get_meter(__name__)
 
@@ -388,6 +473,7 @@ class OpenTelemetry(CustomLogger):
     def _init_logs(self, logger_provider):
         # nothing to do if events disabled
         if not self.config.enable_events:
+            self._logger_provider = None
             return
 
         from opentelemetry._logs import get_logger_provider, set_logger_provider
@@ -404,13 +490,14 @@ class OpenTelemetry(CustomLogger):
             )
             return provider
 
-        self._get_or_create_provider(
+        self._logger_provider = self._get_or_create_provider(
             provider=logger_provider,
             provider_name="LoggerProvider",
             get_existing_provider_fn=get_logger_provider,
             sdk_provider_class=OTLoggerProvider,
             create_new_provider_fn=create_logger_provider,
             set_provider_fn=set_logger_provider,
+            skip_set_global=self._skip_set_global(),
         )
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
@@ -811,8 +898,7 @@ class OpenTelemetry(CustomLogger):
         from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
-        # only log raw LLM request/response if message_logging is on and not globally turned off
-        if litellm.turn_off_message_logging or not self.message_logging:
+        if not self._capture_in_span():
             return
 
         litellm_params = kwargs.get("litellm_params", {})
@@ -1073,7 +1159,7 @@ class OpenTelemetry(CustomLogger):
         # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
         # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
 
-        from opentelemetry._logs import SeverityNumber, get_logger
+        from opentelemetry._logs import SeverityNumber
 
         try:
             from opentelemetry.sdk._logs import (  # type: ignore[attr-defined]  # OTEL < 1.39.0
@@ -1084,7 +1170,10 @@ class OpenTelemetry(CustomLogger):
                 LogRecord as SdkLogRecord,  # type: ignore[attr-defined]  # OTEL >= 1.39.0
             )
 
-        otel_logger = get_logger(LITELLM_LOGGER_NAME)
+        # Resolve through the handler's own LoggerProvider (which may be a
+        # private one when skip_set_global=True) rather than the module-level
+        # get_logger() which always goes through the global provider.
+        otel_logger = self._logger_provider.get_logger(LITELLM_LOGGER_NAME)
 
         parent_ctx = span.get_span_context()
         provider = (kwargs.get("litellm_params") or {}).get(
@@ -1100,9 +1189,14 @@ class OpenTelemetry(CustomLogger):
             }
             if role == "tool" and msg.get("id"):
                 attrs["id"] = msg["id"]
-            if self.message_logging and msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and msg.get("content"):
                 attrs["gen_ai.prompt"] = msg["content"]
 
+            body = msg.copy()
+            if not capture_event_content:
+                body.pop("content", None)
+
             log_record = SdkLogRecord(
                 timestamp=self._to_ns(datetime.now()),
                 trace_id=parent_ctx.trace_id,
@@ -1110,7 +1204,7 @@ class OpenTelemetry(CustomLogger):
                 trace_flags=parent_ctx.trace_flags,
                 severity_number=SeverityNumber.INFO,
                 severity_text="INFO",
-                body=msg.copy(),
+                body=body,
                 attributes=attrs,
             )
             otel_logger.emit(log_record)
@@ -1124,14 +1218,15 @@ class OpenTelemetry(CustomLogger):
                 "finish_reason": choice.get("finish_reason"),
             }
             body_msg = choice.get("message", {})
-            if self.message_logging and body_msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and body_msg.get("content"):
                 attrs["message.content"] = body_msg["content"]
             body = {
                 "index": idx,
                 "finish_reason": choice.get("finish_reason"),
                 "message": {"role": body_msg.get("role", "assistant")},
             }
-            if self.message_logging and body_msg.get("content"):
+            if capture_event_content and body_msg.get("content"):
                 body["message"]["content"] = body_msg["content"]
 
             log_record = SdkLogRecord(
@@ -1657,9 +1752,7 @@ class OpenTelemetry(CustomLogger):
             ########## LLM Request Medssages / tools / content Attributes ###########
             #########################################################################
 
-            if litellm.turn_off_message_logging is True:
-                return
-            if self.message_logging is not True:
+            if not self._capture_in_span():
                 return
 
             if optional_params.get("tools"):

litellm/integrations/prometheus.py

@@ -25,6 +25,9 @@ from typing import (
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
+    BoundedPrometheusSeriesTracker,
+)
 from litellm.integrations.prometheus_helpers import (
     PrometheusLabelFactoryContext,
     _get_cached_end_user_id_for_cost_tracking,
@@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger):
                 if _custom_buckets is not None
                 else LATENCY_BUCKETS
             )
+            self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
 
             # Create metric factory functions
             self._counter_factory = self._create_metric_factory(Counter)
@@ -265,6 +269,7 @@ class PrometheusLogger(CustomLogger):
             ########################################
             # LiteLLM Virtual API KEY metrics
             ########################################
+
             # Remaining MODEL RPM limit for API Key
             self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
                 "litellm_remaining_api_key_requests_for_model",
@@ -983,6 +988,40 @@ class PrometheusLogger(CustomLogger):
 
         return filtered_labels
 
+    def _track_end_user_metric_series(
+        self,
+        metric: Any,
+        metric_name: DEFINED_PROMETHEUS_METRICS,
+        labels: Dict[str, Optional[str]],
+    ) -> None:
+        """
+        Cap the cardinality of metrics that include the ``end_user`` label.
+
+        Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
+        recorded in prometheus-client's child map before any eviction runs.
+        Series that get evicted before the next scrape lose updates accrued
+        since the last scrape — this is inherent to any cardinality cap.
+        """
+        labelnames = self.get_labels_for_metric(metric_name)
+        if UserAPIKeyLabelNames.END_USER.value not in labelnames:
+            return
+        if labels.get(UserAPIKeyLabelNames.END_USER.value) is None:
+            return
+
+        max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
+        ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
+        if max_series is None and ttl_seconds is None:
+            return
+
+        self._bounded_prometheus_series_tracker.track_series(
+            metric=metric,
+            metric_name=metric_name,
+            label_values=tuple(labels.get(label) for label in labelnames),
+            max_series=max_series,
+            ttl_seconds=ttl_seconds,
+            cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds,
+        )
+
     def _inc_labeled_counter(
         self,
         counter: Any,
@@ -997,6 +1036,7 @@ class PrometheusLogger(CustomLogger):
             label_context=label_context,
         )
         counter.labels(**_labels).inc(amount)
+        self._track_end_user_metric_series(counter, metric_name, _labels)
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         # Define prometheus client
@@ -1046,21 +1086,9 @@ class PrometheusLogger(CustomLogger):
         output_tokens = standard_logging_payload["completion_tokens"]
         tokens_used = standard_logging_payload["total_tokens"]
         response_cost = standard_logging_payload["response_cost"]
-        _requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
-            "requester_metadata"
+        combined_metadata = _get_combined_custom_metadata_from_standard_logging_payload(
+            standard_logging_payload=standard_logging_payload
         )
-        user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[
-            "metadata"
-        ].get("user_api_key_auth_metadata")
-        spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get(
-            "spend_logs_metadata"
-        )
-
-        combined_metadata: Dict[str, Any] = {
-            **(_requester_metadata if _requester_metadata else {}),
-            **(user_api_key_auth_metadata if user_api_key_auth_metadata else {}),
-            **(spend_logs_metadata if spend_logs_metadata else {}),
-        }
         if standard_logging_payload is not None and isinstance(
             standard_logging_payload, dict
         ):
@@ -1415,26 +1443,46 @@ class PrometheusLogger(CustomLogger):
         )
         remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
 
-        remaining_requests = (
-            metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
+        remaining_requests = metadata.get(remaining_requests_variable_name)
+        if remaining_requests is None:
+            remaining_requests = sys.maxsize
+        remaining_tokens = metadata.get(remaining_tokens_variable_name)
+        if remaining_tokens is None:
+            remaining_tokens = sys.maxsize
+
+        enum_values = UserAPIKeyLabelValues(
+            hashed_api_key=user_api_key,
+            api_key_alias=user_api_key_alias,
+            model=model_group,
+            model_id=model_id,
+            custom_metadata_labels=get_custom_labels_from_metadata(
+                metadata=_get_combined_custom_metadata_from_standard_logging_payload(
+                    standard_logging_payload=kwargs.get("standard_logging_object")
+                )
+            ),
         )
-        remaining_tokens = (
-            metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
+        label_context = PrometheusLabelFactoryContext(enum_values)
+        requests_labels = prometheus_label_factory(
+            supported_enum_labels=self.get_labels_for_metric(
+                "litellm_remaining_api_key_requests_for_model"
+            ),
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+        self.litellm_remaining_api_key_requests_for_model.labels(**requests_labels).set(
+            remaining_requests
         )
 
-        self.litellm_remaining_api_key_requests_for_model.labels(
-            _sanitize_prometheus_label_value(user_api_key),
-            _sanitize_prometheus_label_value(user_api_key_alias),
-            _sanitize_prometheus_label_value(model_group),
-            _sanitize_prometheus_label_value(model_id),
-        ).set(remaining_requests)
-
-        self.litellm_remaining_api_key_tokens_for_model.labels(
-            _sanitize_prometheus_label_value(user_api_key),
-            _sanitize_prometheus_label_value(user_api_key_alias),
-            _sanitize_prometheus_label_value(model_group),
-            _sanitize_prometheus_label_value(model_id),
-        ).set(remaining_tokens)
+        tokens_labels = prometheus_label_factory(
+            supported_enum_labels=self.get_labels_for_metric(
+                "litellm_remaining_api_key_tokens_for_model"
+            ),
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+        self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set(
+            remaining_tokens
+        )
 
     def _set_latency_metrics(
         self,
@@ -1470,6 +1518,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_time_to_first_token_metric.labels(
                 **_ttft_labels
             ).observe(time_to_first_token_seconds)
+            self._track_end_user_metric_series(
+                self.litellm_llm_api_time_to_first_token_metric,
+                "litellm_llm_api_time_to_first_token_metric",
+                _ttft_labels,
+            )
         else:
             verbose_logger.debug(
                 "Time to first token metric not emitted, stream option in model_parameters is not True"
@@ -1490,6 +1543,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_latency_metric.labels(**_labels).observe(
                 api_call_total_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_llm_api_latency_metric,
+                "litellm_llm_api_latency_metric",
+                _labels,
+            )
 
         # total request latency
         total_time_seconds = self._safe_duration_seconds(
@@ -1507,6 +1565,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_request_total_latency_metric.labels(**_labels).observe(
                 total_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_request_total_latency_metric,
+                "litellm_request_total_latency_metric",
+                _labels,
+            )
 
         # request queue time (time from arrival to processing start)
         _litellm_params = kwargs.get("litellm_params", {}) or {}
@@ -1524,6 +1587,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_request_queue_time_metric.labels(**_labels).observe(
                 queue_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_request_queue_time_metric,
+                "litellm_request_queue_time_seconds",
+                _labels,
+            )
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         verbose_logger.debug(
@@ -1560,18 +1628,27 @@ class PrometheusLogger(CustomLogger):
         )
 
         try:
-            self.litellm_llm_api_failed_requests_metric.labels(
-                _sanitize_prometheus_label_value(end_user_id),
-                _sanitize_prometheus_label_value(user_api_key),
-                _sanitize_prometheus_label_value(user_api_key_alias),
-                _sanitize_prometheus_label_value(model),
-                _sanitize_prometheus_label_value(user_api_team),
-                _sanitize_prometheus_label_value(user_api_team_alias),
-                _sanitize_prometheus_label_value(user_id),
-                _sanitize_prometheus_label_value(
-                    standard_logging_payload.get("model_id", "")
+            enum_values = UserAPIKeyLabelValues(
+                end_user=end_user_id,
+                hashed_api_key=user_api_key,
+                api_key_alias=user_api_key_alias,
+                model=model,
+                team=user_api_team,
+                team_alias=user_api_team_alias,
+                user=user_id,
+                model_id=standard_logging_payload.get("model_id", ""),
+                custom_metadata_labels=get_custom_labels_from_metadata(
+                    metadata=_get_combined_custom_metadata_from_standard_logging_payload(
+                        standard_logging_payload=standard_logging_payload
+                    )
                 ),
-            ).inc()
+            )
+            PrometheusLogger._inc_labeled_counter(
+                self,
+                self.litellm_llm_api_failed_requests_metric,
+                "litellm_llm_api_failed_requests_metric",
+                enum_values,
+            )
             self.set_llm_deployment_failure_metrics(kwargs)
             await self._set_org_budget_metrics_after_api_request(
                 org_id=user_api_key_org_id,
@@ -1928,7 +2005,7 @@ class PrometheusLogger(CustomLogger):
             or _litellm_params_metadata.get("user_agent"),
         }
 
-    def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
+    def set_llm_deployment_failure_metrics(self, request_kwargs: dict):  # noqa: PLR0915
         """
         Sets Failure metrics when an LLM API call fails
 
@@ -2006,17 +2083,32 @@ class PrometheusLogger(CustomLogger):
                     if code is not None:
                         exception_status = str(code)
 
-            # Create enum_values for the label factory (always create for use in different metrics)
+            # On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
+            # into requested_model and leave deployment-scoped labels empty.
+            deployment_selected = bool(model_id)
+            if deployment_selected:
+                label_litellm_model_name = litellm_model_name
+                label_model_id = model_id
+                label_api_base = api_base
+                label_api_provider = llm_provider
+                label_requested_model = model_group or litellm_model_name
+            else:
+                label_litellm_model_name = ""
+                label_model_id = ""
+                label_api_base = ""
+                label_api_provider = ""
+                label_requested_model = litellm_model_name or model_group or ""
+
             enum_values = UserAPIKeyLabelValues(
-                litellm_model_name=litellm_model_name,
-                model_id=model_id,
-                api_base=api_base,
-                api_provider=llm_provider,
+                litellm_model_name=label_litellm_model_name,
+                model_id=label_model_id,
+                api_base=label_api_base,
+                api_provider=label_api_provider,
                 exception_status=exception_status,
                 exception_class=(
                     self._get_exception_class_name(exception) if exception else None
                 ),
-                requested_model=model_group or litellm_model_name,
+                requested_model=label_requested_model,
                 hashed_api_key=hashed_api_key,
                 api_key_alias=api_key_alias,
                 team=team,
@@ -2030,12 +2122,14 @@ class PrometheusLogger(CustomLogger):
             log these labels
             ["litellm_model_name", "model_id", "api_base", "api_provider"]
             """
-            self.set_deployment_partial_outage(
-                litellm_model_name=litellm_model_name or "",
-                model_id=model_id,
-                api_base=api_base,
-                api_provider=llm_provider or "",
-            )
+            # Only mark a deployment outage when one was actually picked.
+            if deployment_selected:
+                self.set_deployment_partial_outage(
+                    litellm_model_name=litellm_model_name or "",
+                    model_id=model_id,
+                    api_base=api_base,
+                    api_provider=llm_provider or "",
+                )
             _deployment_label_ctx = PrometheusLabelFactoryContext(enum_values)
             if exception is not None:
                 PrometheusLogger._inc_labeled_counter(
@@ -3604,6 +3698,36 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
     return result
 
 
+def _get_combined_custom_metadata_from_standard_logging_payload(
+    standard_logging_payload: Optional[dict],
+) -> Dict[str, Any]:
+    """
+    Combine the metadata sources that can supply custom Prometheus labels.
+    """
+    if not isinstance(standard_logging_payload, dict):
+        return {}
+
+    standard_logging_metadata = standard_logging_payload.get("metadata") or {}
+    if not isinstance(standard_logging_metadata, dict):
+        return {}
+
+    requester_metadata = standard_logging_metadata.get("requester_metadata")
+    user_api_key_auth_metadata = standard_logging_metadata.get(
+        "user_api_key_auth_metadata"
+    )
+    spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata")
+
+    return {
+        **(requester_metadata if isinstance(requester_metadata, dict) else {}),
+        **(
+            user_api_key_auth_metadata
+            if isinstance(user_api_key_auth_metadata, dict)
+            else {}
+        ),
+        **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}),
+    }
+
+
 def _tag_matches_wildcard_configured_pattern(
     tags: Sequence[str], configured_tag: str
 ) -> bool:

litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import time
+from collections import OrderedDict
+from threading import RLock
+from typing import Any, Dict, Optional
+
+
+class BoundedPrometheusSeriesTracker:
+    """
+    Tracks Prometheus child series and removes stale/excess labelsets.
+
+    The tracker is label-agnostic: callers decide which series should be tracked
+    and pass the full label tuple used by the Prometheus metric.
+    """
+
+    def __init__(self) -> None:
+        self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
+        self._last_ttl_cleanup: Dict[str, float] = {}
+        self.lock = RLock()
+
+    def track_series(
+        self,
+        metric: Any,
+        metric_name: str,
+        label_values: tuple[Optional[str], ...],
+        max_series: Optional[int],
+        ttl_seconds: Optional[float],
+        cleanup_interval_seconds: Optional[float],
+    ) -> None:
+        if max_series is None and ttl_seconds is None:
+            return
+
+        now = time.monotonic()
+
+        with self.lock:
+            series = self._series.setdefault(metric_name, OrderedDict())
+            series[label_values] = now
+            series.move_to_end(label_values)
+
+            if ttl_seconds is not None and self._should_run_ttl_cleanup(
+                metric_name=metric_name,
+                now=now,
+                cleanup_interval_seconds=cleanup_interval_seconds,
+            ):
+                expired_label_values = [
+                    tracked_label_values
+                    for tracked_label_values, last_seen in series.items()
+                    if now - last_seen > ttl_seconds
+                ]
+                for tracked_label_values in expired_label_values:
+                    self._remove_metric_series(metric, series, tracked_label_values)
+
+            # max_series <= 0 is treated as "unlimited" so a misconfigured zero
+            # value cannot silently drop every emission for this metric.
+            if max_series is not None and max_series > 0:
+                while len(series) > max_series:
+                    tracked_label_values = next(iter(series))
+                    if not self._remove_metric_child(metric, tracked_label_values):
+                        break
+                    del series[tracked_label_values]
+
+    def _should_run_ttl_cleanup(
+        self,
+        metric_name: str,
+        now: float,
+        cleanup_interval_seconds: Optional[float],
+    ) -> bool:
+        if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0:
+            self._last_ttl_cleanup[metric_name] = now
+            return True
+
+        last_cleanup = self._last_ttl_cleanup.get(metric_name)
+        if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds:
+            self._last_ttl_cleanup[metric_name] = now
+            return True
+        return False
+
+    def _remove_metric_series(
+        self,
+        metric: Any,
+        series: OrderedDict[tuple[Optional[str], ...], float],
+        label_values: tuple[Optional[str], ...],
+    ) -> None:
+        if self._remove_metric_child(metric, label_values):
+            series.pop(label_values, None)
+
+    @staticmethod
+    def _remove_metric_child(
+        metric: Any, label_values: tuple[Optional[str], ...]
+    ) -> bool:
+        """
+        Remove the Prometheus child for ``label_values`` and report whether the
+        tracker should commit the matching state change.
+
+        Returns ``True`` when the child is no longer present in Prometheus
+        (either it was just removed or it was already gone), and ``False`` when
+        ``metric.remove()`` raised an unexpected error and the child likely
+        still exists.
+        """
+        try:
+            metric.remove(*label_values)
+            return True
+        except KeyError:
+            return True
+        except (AttributeError, ValueError):
+            return False

litellm/integrations/prometheus_helpers/prometheus_api.py

@@ -2,6 +2,7 @@
 Helper functions to query prometheus API
 """
 
+import json
 import time
 from datetime import datetime, timedelta
 from typing import Optional
@@ -81,6 +82,24 @@ def is_prometheus_connected() -> bool:
     return False
 
 
+def _quote_promql_string_literal(value: str) -> str:
+    """Render ``value`` as a PromQL double-quoted string literal.
+
+    PromQL string literals follow Go's escape rules
+    (https://prometheus.io/docs/prometheus/latest/querying/basics/): a
+    backslash begins an escape sequence and a bare ``"`` ends the literal.
+    Without escaping, callers that accept arbitrary user-supplied values
+    (like the ``api_key`` filter on ``/global/spend/logs``) can inject extra
+    label matchers or selectors and read cross-tenant metrics.
+
+    JSON's quoting rules are a strict subset of Go's, so ``json.dumps`` of
+    a Python string produces a literal Prometheus accepts: ``\\``, ``\\"``,
+    and the standard ``\\n`` / ``\\t`` / ``\\uNNNN`` control-character
+    escapes. The returned value already includes the surrounding quotes.
+    """
+    return json.dumps(value, ensure_ascii=False)
+
+
 async def get_daily_spend_from_prometheus(api_key: Optional[str]):
     """
     Expected Response Format:
@@ -109,8 +128,11 @@ async def get_daily_spend_from_prometheus(api_key: Optional[str]):
     if api_key is None:
         query = "sum(delta(litellm_spend_metric_total[1d]))"
     else:
+        quoted_api_key = _quote_promql_string_literal(api_key)
         query = (
-            f'sum(delta(litellm_spend_metric_total{{hashed_api_key="{api_key}"}}[1d]))'
+            "sum(delta(litellm_spend_metric_total{"
+            f"hashed_api_key={quoted_api_key}"
+            "}[1d]))"
         )
 
     params = {

litellm/integrations/prompt_management_base.py

@@ -87,9 +87,7 @@ class PromptManagementBase(ABC):
         try:
             messages = compiled_prompt_client["prompt_template"] + client_messages
         except Exception as e:
-            raise ValueError(
-                f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
-            )
+            raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
 
         compiled_prompt_client["completed_messages"] = messages
         return compiled_prompt_client
@@ -116,9 +114,7 @@ class PromptManagementBase(ABC):
         try:
             messages = compiled_prompt_client["prompt_template"] + client_messages
         except Exception as e:
-            raise ValueError(
-                f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
-            )
+            raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
 
         compiled_prompt_client["completed_messages"] = messages
         return compiled_prompt_client

litellm/integrations/s3_v2.py

@@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
 from litellm.integrations.s3 import get_s3_object_key
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
@@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_strip_base64_files: bool = False,
         s3_use_key_prefix: bool = False,
         s3_use_virtual_hosted_style: bool = False,
+        s3_callback_params_override: Optional[dict] = None,
         **kwargs,
     ):
         try:
-            verbose_logger.debug(
-                f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
-            )
+            _masker = SensitiveDataMasker()
+            if s3_callback_params_override is not None:
+                verbose_logger.debug(
+                    f"in init s3 logger (audit override) - "
+                    f"{_masker.mask_dict(dict(s3_callback_params_override))}"
+                )
+            else:
+                verbose_logger.debug(
+                    f"in init s3 logger - s3_callback_params "
+                    f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
+                )
 
             # Initialize S3 params first to get the correct s3_verify value
             self._init_s3_params(
+                params_source=s3_callback_params_override,
                 s3_bucket_name=s3_bucket_name,
                 s3_region_name=s3_region_name,
                 s3_api_version=s3_api_version,
@@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_strip_base64_files: bool = False,
         s3_use_key_prefix: bool = False,
         s3_use_virtual_hosted_style: bool = False,
+        params_source: Optional[dict] = None,
     ):
         """
-        Initialize the s3 params for this logging callback
+        Initialize the s3 params for this logging callback. Reads from
+        `params_source` if given (e.g. `s3_audit_callback_params` for the
+        audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
+        Resolves `os.environ/X` markers into a local dict; never mutates the source.
         """
-        litellm.s3_callback_params = litellm.s3_callback_params or {}
-        # read in .env variables - example os.environ/AWS_BUCKET_NAME
-        for key, value in litellm.s3_callback_params.items():
-            if isinstance(value, str) and value.startswith("os.environ/"):
-                litellm.s3_callback_params[key] = litellm.get_secret(value)
+        if params_source is None:
+            params_source = litellm.s3_callback_params or {}
+        params: dict = {
+            key: (
+                litellm.get_secret(value)
+                if isinstance(value, str) and value.startswith("os.environ/")
+                else value
+            )
+            for key, value in params_source.items()
+        }
 
-        self.s3_bucket_name = (
-            litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
-        )
-        self.s3_region_name = (
-            litellm.s3_callback_params.get("s3_region_name") or s3_region_name
-        )
-        self.s3_api_version = (
-            litellm.s3_callback_params.get("s3_api_version") or s3_api_version
-        )
+        self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
+        self.s3_region_name = params.get("s3_region_name") or s3_region_name
+        self.s3_api_version = params.get("s3_api_version") or s3_api_version
         self.s3_use_ssl = (
-            litellm.s3_callback_params.get("s3_use_ssl", True)
-            if litellm.s3_callback_params.get("s3_use_ssl") is not None
+            params.get("s3_use_ssl", True)
+            if params.get("s3_use_ssl") is not None
             else s3_use_ssl
         )
         self.s3_verify = (
-            litellm.s3_callback_params.get("s3_verify")
-            if litellm.s3_callback_params.get("s3_verify") is not None
+            params.get("s3_verify")
+            if params.get("s3_verify") is not None
             else s3_verify
         )
-        self.s3_endpoint_url = (
-            litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
-        )
+        self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
         self.s3_aws_access_key_id = (
-            litellm.s3_callback_params.get("s3_aws_access_key_id")
-            or s3_aws_access_key_id
+            params.get("s3_aws_access_key_id") or s3_aws_access_key_id
         )
 
         self.s3_aws_secret_access_key = (
-            litellm.s3_callback_params.get("s3_aws_secret_access_key")
-            or s3_aws_secret_access_key
+            params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
         )
 
         self.s3_aws_session_token = (
-            litellm.s3_callback_params.get("s3_aws_session_token")
-            or s3_aws_session_token
+            params.get("s3_aws_session_token") or s3_aws_session_token
         )
 
         self.s3_aws_session_name = (
-            litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
+            params.get("s3_aws_session_name") or s3_aws_session_name
         )
 
         self.s3_aws_profile_name = (
-            litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
+            params.get("s3_aws_profile_name") or s3_aws_profile_name
         )
 
-        self.s3_aws_role_name = (
-            litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
-        )
+        self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
 
         self.s3_aws_web_identity_token = (
-            litellm.s3_callback_params.get("s3_aws_web_identity_token")
-            or s3_aws_web_identity_token
+            params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
         )
 
         self.s3_aws_sts_endpoint = (
-            litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
+            params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
         )
 
-        self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
-        self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
-        # done reading litellm.s3_callback_params
+        self.s3_config = params.get("s3_config") or s3_config
+        self.s3_path = params.get("s3_path") or s3_path
         self.s3_use_team_prefix = (
-            bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
-            or s3_use_team_prefix
+            bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
         )
 
         self.s3_use_key_prefix = (
-            bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
-            or s3_use_key_prefix
+            bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
         )
 
         self.s3_strip_base64_files = (
-            bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
-            or s3_strip_base64_files
+            bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
         )
 
         self.s3_use_virtual_hosted_style = (
-            bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
+            bool(params.get("s3_use_virtual_hosted_style", False))
             or s3_use_virtual_hosted_style
         )
 

litellm/litellm_core_utils/cli_token_utils.py

@@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]:
         return None
 
 
-def get_litellm_gateway_api_key() -> Optional[str]:
+def get_litellm_gateway_api_key(
+    expected_base_url: Optional[str] = None,
+) -> Optional[str]:
     """
     Get the stored CLI API key for use with LiteLLM SDK.
 
     This function reads the token file created by `litellm-proxy login`
     and returns the API key for use in Python scripts.
 
+    Args:
+        expected_base_url: When provided, the key is only returned if it was
+            originally issued for this URL. Pass the target server URL to
+            prevent credential leakage when the client is pointed at a
+            different (possibly malicious) server.
+
     Returns:
-        str: The API key if found, None otherwise
+        str: The API key if found (and origin matches), None otherwise
 
     Example:
         >>> import litellm
@@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]:
         >>>     )
     """
     token_data = load_cli_token()
-    if token_data and "key" in token_data:
-        return token_data["key"]
-    return None
+    if not token_data or "key" not in token_data:
+        return None
+    if expected_base_url is not None:
+        stored_url = token_data.get("base_url")
+        if stored_url != expected_base_url.rstrip("/"):
+            return None
+    return token_data["key"]

litellm/litellm_core_utils/cloud_storage_security.py

@@ -0,0 +1,175 @@
+import posixpath
+import re
+from types import MappingProxyType
+from typing import Any, Mapping, Optional, Sequence, Tuple, cast
+from urllib.parse import quote, unquote
+
+from litellm._uuid import uuid
+
+VERTEX_AI_MANAGED_GCS_PREFIX = "litellm-vertex-files/"
+BEDROCK_MANAGED_S3_BATCH_PREFIX = "litellm-bedrock-files-"
+BEDROCK_MANAGED_S3_UPLOAD_PREFIX = "litellm-bedrock-files/"
+BEDROCK_MANAGED_S3_OUTPUT_PREFIX = "litellm-batch-outputs/"
+BEDROCK_MANAGED_S3_PREFIXES = (
+    BEDROCK_MANAGED_S3_BATCH_PREFIX,
+    BEDROCK_MANAGED_S3_UPLOAD_PREFIX,
+    BEDROCK_MANAGED_S3_OUTPUT_PREFIX,
+)
+_MAPPING_PROXY_TYPE: type = type(MappingProxyType({}))
+
+_SAFE_OBJECT_COMPONENT_PATTERN = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def sanitize_cloud_object_component(
+    value: Optional[str], fallback: str = "file"
+) -> str:
+    if not isinstance(value, str):
+        return fallback
+
+    component = posixpath.basename(value.replace("\\", "/")).strip()
+    if component in {"", ".", ".."}:
+        return fallback
+
+    component = "".join(
+        "_" if ord(char) < 32 or ord(char) == 127 else char for char in component
+    )
+    component = _SAFE_OBJECT_COMPONENT_PATTERN.sub("_", component)
+    component = component.strip("._")
+    if not component:
+        return fallback
+    return component[:255]
+
+
+def sanitize_cloud_object_path(value: Optional[str], fallback: str = "file") -> str:
+    if not isinstance(value, str):
+        return fallback
+
+    segments = []
+    for segment in value.replace("\\", "/").split("/"):
+        sanitized_segment = sanitize_cloud_object_component(segment, fallback="")
+        if sanitized_segment:
+            segments.append(sanitized_segment)
+
+    if not segments:
+        return fallback
+    return "/".join(segments)
+
+
+def build_managed_cloud_object_name(
+    prefix: str, filename: Optional[str], fallback_filename: str = "file"
+) -> str:
+    safe_filename = sanitize_cloud_object_component(
+        filename, fallback=fallback_filename
+    )
+    return f"{prefix}{uuid.uuid4().hex}-{safe_filename}"
+
+
+def _validate_cloud_object_path(object_name: str) -> None:
+    if not object_name:
+        raise ValueError("Cloud storage object name is required")
+    if object_name.startswith("/"):
+        raise ValueError("Cloud storage object name must be relative")
+    if any(ord(char) < 32 or ord(char) == 127 for char in object_name):
+        raise ValueError("Cloud storage object name contains control characters")
+    segments = object_name.split("/")
+    if any(segment in {".", ".."} for segment in segments):
+        raise ValueError("Cloud storage object name contains an invalid path segment")
+    if "" in segments[:-1]:
+        raise ValueError("Cloud storage object name contains an invalid path segment")
+
+
+def split_configured_cloud_bucket_name(bucket_name: str) -> Tuple[str, str]:
+    if not isinstance(bucket_name, str) or not bucket_name.strip():
+        raise ValueError("Cloud storage bucket name is required")
+
+    bucket_name = bucket_name.strip()
+    if "://" in bucket_name or "?" in bucket_name or "#" in bucket_name:
+        raise ValueError(
+            "Cloud storage bucket name must not include a URI scheme or query"
+        )
+    if any(ord(char) < 32 or ord(char) == 127 for char in bucket_name):
+        raise ValueError("Cloud storage bucket name contains control characters")
+
+    bucket, _, prefix = bucket_name.partition("/")
+    if not bucket:
+        raise ValueError("Cloud storage bucket name is required")
+    if "\\" in bucket:
+        raise ValueError("Cloud storage bucket name contains an invalid separator")
+
+    prefix = prefix.strip("/")
+    if prefix:
+        _validate_cloud_object_path(prefix)
+
+    return bucket, prefix
+
+
+def encode_gcs_object_name_for_url(object_name: str) -> str:
+    return quote(unquote(object_name), safe="")
+
+
+def encode_s3_object_key_for_url(object_key: str) -> str:
+    return quote(unquote(object_key), safe="/")
+
+
+def should_allow_legacy_cloud_file_ids(
+    litellm_params: Optional[Mapping[str, Any]] = None,
+) -> bool:
+    value = None
+    if isinstance(litellm_params, Mapping):
+        trusted_model_credentials = litellm_params.get(
+            "_litellm_internal_model_credentials"
+        )
+        if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE):
+            value = cast(Mapping[str, Any], trusted_model_credentials).get(
+                "allow_legacy_cloud_file_ids"
+            )
+
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+    return False
+
+
+def validate_managed_cloud_file_id(
+    file_id: str,
+    scheme: str,
+    configured_bucket_name: str,
+    allowed_object_prefixes: Sequence[str],
+    allow_legacy_cloud_file_ids: bool = False,
+) -> Tuple[str, str]:
+    decoded_file_id = unquote(file_id)
+    if not decoded_file_id.startswith(scheme):
+        raise ValueError(f"file_id must be a {scheme} URI")
+
+    full_path = decoded_file_id[len(scheme) :]
+    if "/" not in full_path:
+        raise ValueError("file_id must include a cloud storage object name")
+
+    bucket_name, object_name = full_path.split("/", 1)
+    configured_bucket, configured_prefix = split_configured_cloud_bucket_name(
+        configured_bucket_name
+    )
+    if bucket_name != configured_bucket:
+        raise ValueError("file_id bucket does not match the configured storage bucket")
+
+    _validate_cloud_object_path(object_name)
+    allowed_prefixes = tuple(allowed_object_prefixes)
+    if configured_prefix:
+        allowed_prefixes = tuple(
+            f"{configured_prefix.rstrip('/')}/{prefix}" for prefix in allowed_prefixes
+        )
+
+    if object_name.startswith(allowed_prefixes):
+        return bucket_name, object_name
+
+    if allow_legacy_cloud_file_ids:
+        if configured_prefix and not object_name.startswith(
+            f"{configured_prefix.rstrip('/')}/"
+        ):
+            raise ValueError(
+                "file_id object does not match the configured storage prefix"
+            )
+        return bucket_name, object_name
+
+    raise ValueError("file_id must reference a LiteLLM-managed storage object")

litellm/litellm_core_utils/custom_logger_registry.py

@@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal
 from litellm.integrations.agentops import AgentOps
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.integrations.argilla import ArgillaLogger
+from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
 from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
 from litellm.integrations.bitbucket import BitBucketPromptManager
 from litellm.integrations.braintrust_logging import BraintrustLogger
@@ -73,6 +74,7 @@ class CustomLoggerRegistry:
         "opik": OpikLogger,
         "argilla": ArgillaLogger,
         "opentelemetry": OpenTelemetry,
+        "azure_sentinel": AzureSentinelLogger,
         "azure_storage": AzureBlobStorageLogger,
         "humanloop": HumanloopLogger,
         # OTEL compatible loggers

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -6,7 +6,8 @@ from typing import Any, Optional
 import httpx
 
 import litellm
-from litellm._logging import _redact_string, verbose_logger
+from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -261,10 +262,18 @@ def exception_type(  # type: ignore  # noqa: PLR0915
         original_exception=original_exception
     )
     try:
-        error_str = str(original_exception)
+        error_str = (
+            redact_string(str(original_exception))
+            if _ENABLE_SECRET_REDACTION
+            else str(original_exception)
+        )
         if model:
             if hasattr(original_exception, "message"):
-                error_str = str(original_exception.message)
+                error_str = (
+                    redact_string(str(original_exception.message))
+                    if _ENABLE_SECRET_REDACTION
+                    else str(original_exception.message)
+                )
             if isinstance(original_exception, BaseException):
                 exception_type = type(original_exception).__name__
             else:
@@ -2431,7 +2440,8 @@ def exception_type(  # type: ignore  # noqa: PLR0915
             else:
                 raise APIConnectionError(
                     message="{}\n{}".format(
-                        str(original_exception), _redact_string(traceback.format_exc())
+                        str(original_exception),
+                        _redact_string(traceback.format_exc()),
                     ),
                     llm_provider=custom_llm_provider,
                     model=model,
@@ -2461,7 +2471,8 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     raise e  # it's already mapped
             raised_exc = APIConnectionError(
                 message="{}\n{}".format(
-                    original_exception, _redact_string(traceback.format_exc())
+                    original_exception,
+                    _redact_string(traceback.format_exc()),
                 ),
                 llm_provider="",
                 model="",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -1,4 +1,5 @@
 from typing import Optional, Tuple
+from urllib.parse import urlparse
 
 import litellm
 from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
@@ -8,6 +9,43 @@ from litellm.secret_managers.main import get_secret, get_secret_str
 from ..types.router import LiteLLM_Params
 
 
+def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool:
+    """
+    Match a registered openai-compatible endpoint against a caller-supplied
+    ``api_base`` using parsed-URL semantics, not unanchored substring search.
+
+    Both inputs may be a bare hostname (``api.perplexity.ai``), host+path
+    (``api.deepinfra.com/v1/openai``), or a full URL
+    (``https://api.cerebras.ai/v1``). Hostnames must match exactly
+    (case-insensitive); if the registered endpoint has a non-trivial path,
+    the api_base path must start with it on a segment boundary.
+
+    The naive ``endpoint in api_base`` shape lets a caller pass
+    ``https://attacker.com/api.groq.com/openai/v1`` to coerce the proxy
+    into reading the server's GROQ_API_KEY from the environment and
+    forwarding it to the attacker's host as a Bearer credential.
+    """
+
+    def _parse(value: str):
+        # Ensure urlparse sees a scheme so it populates hostname / path.
+        normalized = value if "://" in value else f"https://{value}"
+        return urlparse(normalized)
+
+    parsed_endpoint = _parse(endpoint)
+    parsed_url = _parse(api_base)
+
+    endpoint_host = (parsed_endpoint.hostname or "").lower()
+    url_host = (parsed_url.hostname or "").lower()
+    if not endpoint_host or endpoint_host != url_host:
+        return False
+
+    endpoint_path = parsed_endpoint.path.rstrip("/")
+    if not endpoint_path:
+        return True
+    url_path = parsed_url.path.rstrip("/")
+    return url_path == endpoint_path or url_path.startswith(endpoint_path + "/")
+
+
 def _is_non_openai_azure_model(model: str) -> bool:
     try:
         model_name = model.split("/", 1)[1]
@@ -210,7 +248,7 @@ def get_llm_provider(  # noqa: PLR0915
         # check if api base is a known openai compatible endpoint
         if api_base:
             for endpoint in litellm.openai_compatible_endpoints:
-                if endpoint in api_base:
+                if _endpoint_matches_api_base(endpoint, api_base):
                     if endpoint == "api.perplexity.ai":
                         custom_llm_provider = "perplexity"
                         dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY")
@@ -348,6 +386,7 @@ def get_llm_provider(  # noqa: PLR0915
             or "ft:gpt-3.5-turbo" in model
             or "ft:gpt-4" in model  # catches ft:gpt-4-0613, ft:gpt-4o
             or model in litellm.openai_image_generation_models
+            or model.startswith("gpt-image")
             or model in litellm.openai_video_generation_models
         ):
             custom_llm_provider = "openai"
@@ -582,6 +621,18 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
             or "https://integrate.api.nvidia.com/v1"
         )  # type: ignore
         dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
+    elif custom_llm_provider == "nvidia_riva":
+        # NVIDIA Riva is gRPC-based; api_base must be a host:port like
+        # `grpc.nvcf.nvidia.com:443` or `localhost:50051`. There is no
+        # public-default endpoint, so we do not fill one in here.
+        api_base = api_base or get_secret_str("NVIDIA_RIVA_API_BASE")  # type: ignore
+        # Fall back to NVIDIA_NIM_API_KEY because users running both NVCF
+        # services typically reuse the same nvapi-* key.
+        dynamic_api_key = (
+            api_key
+            or get_secret_str("NVIDIA_RIVA_API_KEY")
+            or get_secret_str("NVIDIA_NIM_API_KEY")
+        )
     elif custom_llm_provider == "cerebras":
         api_base = (
             api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None:
     )
 
 
+def validate_no_callback_env_reference(
+    param: str, value: object, *, source: str
+) -> None:
+    if _is_env_reference(value):
+        _raise_env_reference_error(param, source=source)
+
+
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
 _supported_callback_params = [
     "langfuse_public_key",
@@ -30,8 +37,6 @@ _supported_callback_params = [
     "langfuse_secret_key",
     "langfuse_host",
     "langfuse_prompt_version",
-    "gcs_bucket_name",
-    "gcs_path_service_account",
     "langsmith_api_key",
     "langsmith_project",
     "langsmith_base_url",
@@ -50,6 +55,11 @@ _supported_callback_params = [
     "lunary_public_key",
 ]
 
+_request_blocked_callback_params = {
+    "gcs_bucket_name",
+    "gcs_path_service_account",
+}
+
 
 def initialize_standard_callback_dynamic_params(
     kwargs: Optional[Dict] = None,
@@ -57,17 +67,20 @@ def initialize_standard_callback_dynamic_params(
     """
     Initialize the standard callback dynamic params from the kwargs
 
-    checks if langfuse_secret_key, gcs_bucket_name in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
+    checks supported request callback params in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
     """
 
     standard_callback_dynamic_params = StandardCallbackDynamicParams()
     if kwargs:
         # 1. Check top-level kwargs
         for param in _supported_callback_params:
+            if param in _request_blocked_callback_params:
+                continue
             if param in kwargs:
                 _param_value = kwargs.get(param)
-                if _is_env_reference(_param_value):
-                    _raise_env_reference_error(param, source="request body")
+                validate_no_callback_env_reference(
+                    param, _param_value, source="request body"
+                )
                 standard_callback_dynamic_params[param] = _param_value  # type: ignore
 
         # 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
@@ -78,10 +91,13 @@ def initialize_standard_callback_dynamic_params(
 
         if isinstance(metadata, dict):
             for param in _supported_callback_params:
+                if param in _request_blocked_callback_params:
+                    continue
                 if param not in standard_callback_dynamic_params and param in metadata:
                     _param_value = metadata.get(param)
-                    if _is_env_reference(_param_value):
-                        _raise_env_reference_error(param, source="metadata")
+                    validate_no_callback_env_reference(
+                        param, _param_value, source="metadata"
+                    )
                     standard_callback_dynamic_params[param] = _param_value  # type: ignore
 
     return standard_callback_dynamic_params

litellm/litellm_core_utils/litellm_logging.py

@@ -1467,6 +1467,8 @@ class Logging(LiteLLMLoggingBaseClass):
             LiteLLMRealtimeStreamLoggingObject,
             OpenAIModerationResponse,
             "SearchResponse",
+            dict,
+            list,
         ],
         cache_hit: Optional[bool] = None,
         litellm_model_name: Optional[str] = None,
@@ -1725,12 +1727,18 @@ class Logging(LiteLLMLoggingBaseClass):
             return
         if self.model_call_details.get("litellm_params") is None:
             return
-        self.model_call_details["litellm_params"].setdefault("metadata", {})
-        if self.model_call_details["litellm_params"]["metadata"] is None:
-            self.model_call_details["litellm_params"]["metadata"] = {}
-        self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
-            getattr(logging_result, "_hidden_params", {})
-        )
+        metadata_hidden_params = hidden_params.copy()
+        response_cost = self.model_call_details.get("response_cost")
+        if (
+            metadata_hidden_params.get("response_cost") is None
+            and response_cost is not None
+        ):
+            metadata_hidden_params["response_cost"] = response_cost
+
+        litellm_params = self.model_call_details["litellm_params"]
+        metadata = litellm_params.get("metadata") or {}
+        litellm_params["metadata"] = metadata
+        metadata["hidden_params"] = metadata_hidden_params
 
     def _process_hidden_params_and_response_cost(
         self,
@@ -1738,6 +1746,7 @@ class Logging(LiteLLMLoggingBaseClass):
         start_time,
         end_time,
     ):
+        """Resolve hidden params, compute response cost, and emit the standard logging payload."""
         hidden_params = getattr(logging_result, "_hidden_params", {})
         if hidden_params:
             if self.model_call_details.get("litellm_params") is not None:
@@ -1871,24 +1880,12 @@ class Logging(LiteLLMLoggingBaseClass):
             ):
                 if self._is_recognized_call_type_for_logging(
                     logging_result=logging_result
-                ):
+                ) or isinstance(logging_result, (dict, list)):
                     self._process_hidden_params_and_response_cost(
                         logging_result=logging_result,
                         start_time=start_time,
                         end_time=end_time,
                     )
-                elif isinstance(result, dict) or isinstance(result, list):
-                    self.model_call_details["standard_logging_object"] = (
-                        self._build_standard_logging_payload(
-                            result, start_time, end_time
-                        )
-                    )
-                    if (
-                        standard_logging_payload := self.model_call_details.get(
-                            "standard_logging_object"
-                        )
-                    ) is not None:
-                        emit_standard_logging_payload(standard_logging_payload)
             elif standard_logging_object is not None:
                 self.model_call_details["standard_logging_object"] = (
                     standard_logging_object
@@ -3245,10 +3242,15 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                     langfuse_secret=self.standard_callback_dynamic_params.get(
                         "langfuse_secret"
-                    ),
+                    )
+                    or self.standard_callback_dynamic_params.get("langfuse_secret_key"),
                     langfuse_host=self.standard_callback_dynamic_params.get(
                         "langfuse_host"
                     ),
+                    allow_env_credentials=self.standard_callback_dynamic_params.get(
+                        "langfuse_host"
+                    )
+                    is None,
                 )
             return langFuseLogger
 
@@ -4723,7 +4725,7 @@ class StandardLoggingPayloadSetup:
         ):
             for key, value in litellm_params["metadata"].items():
                 # Skip non-serializable objects like UserAPIKeyAuth
-                if key == "user_api_key_auth":
+                if key in {"user_api_key_auth", "user_api_key_budget_reservation"}:
                     continue
                 merged_metadata[key] = value
 
@@ -5438,11 +5440,6 @@ def get_standard_logging_object_payload(
             completion_start_time_float=completion_start_time_float,
             stream=kwargs.get("stream", False),
         )
-        # clean up litellm hidden params
-        clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
-            hidden_params
-        )
-
         # clean up litellm metadata
         clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
             metadata=metadata,
@@ -5476,6 +5473,18 @@ def get_standard_logging_object_payload(
         ## Get model cost information ##
         base_model = _get_base_model_from_metadata(model_call_details=kwargs)
         custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
+        raw_response_cost = kwargs.get("response_cost")
+        response_cost: float = raw_response_cost or 0.0
+
+        # clean up litellm hidden params
+        clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
+            hidden_params
+        )
+        if (
+            clean_hidden_params["response_cost"] is None
+            and raw_response_cost is not None
+        ):
+            clean_hidden_params["response_cost"] = response_cost
 
         model_cost_information = StandardLoggingPayloadSetup.get_model_cost_information(
             base_model=base_model,
@@ -5484,7 +5493,6 @@ def get_standard_logging_object_payload(
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
         )
-        response_cost: float = kwargs.get("response_cost", 0) or 0.0
 
         error_information = StandardLoggingPayloadSetup.get_error_information(
             original_exception=original_exception,

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -982,9 +982,9 @@ class CostCalculatorUtils:
                 image_response=completion_response,
             )
         elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )
@@ -1004,9 +1004,9 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )

litellm/litellm_core_utils/llm_request_utils.py

@@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
     if litellm_params is None:
         return {}
 
-    proxy_request_headers = (
-        litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
-    )
+    proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get(
+        "headers"
+    ) or {}
 
     return proxy_request_headers

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -824,8 +824,6 @@ def convert_to_model_response_object(  # noqa: PLR0915
             stream=stream,
             start_time=start_time,
             end_time=end_time,
-            hidden_params=hidden_params,
-            _response_headers=_response_headers,
             convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
         )
         raise Exception(

litellm/litellm_core_utils/logging_callback_manager.py

@@ -221,6 +221,13 @@ class LoggingCallbackManager:
             headers = callback_config.get("headers")
             event_types = callback_config.get("event_types")
             log_format = callback_config.get("log_format")
+            max_retries = max(0, int(callback_config.get("max_retries", 0) or 0))
+            retry_delay_value = callback_config.get("retry_delay")
+            retry_delay = max(
+                0.0,
+                float(0.0 if retry_delay_value is None else retry_delay_value),
+            )
+            timeout = callback_config.get("timeout")
 
             if endpoint is None or headers is None:
                 verbose_logger.warning(
@@ -236,6 +243,9 @@ class LoggingCallbackManager:
                 and cached_logger.headers == headers
                 and cached_logger.event_types == event_types
                 and cached_logger.log_format == log_format
+                and cached_logger.max_retries == max_retries
+                and cached_logger.retry_delay == retry_delay
+                and cached_logger.timeout == timeout
             ):
                 return cached_logger
 
@@ -244,6 +254,9 @@ class LoggingCallbackManager:
                 headers=headers,
                 event_types=event_types,
                 log_format=log_format,
+                max_retries=max_retries,
+                retry_delay=retry_delay,
+                timeout=timeout,
             )
             _generic_api_logger_cache[callback] = new_logger
             return new_logger

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
     """
     Updates messages with model file ids.
 
+    For managed files (unified file IDs), uses model_file_id_mapping if it
+    resolves the id, otherwise decodes the base64-encoded unified file ID
+    and extracts the llm_output_file_id directly. Mirrors the Responses-API
+    sibling `update_responses_input_with_model_file_ids`.
+
     model_file_id_mapping: Dict[str, Dict[str, str]] = {
         "litellm_proxy/file_id": {
             "model_id": "provider_file_id"
         }
     }
     """
+    from litellm.proxy.openai_files_endpoints.common_utils import (
+        _is_base64_encoded_unified_file_id,
+        convert_b64_uid_to_unified_uid,
+    )
 
     for message in messages:
         if message.get("role") == "user":
@@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
                 if isinstance(content, str):
                     continue
                 for c in content:
-                    if c["type"] == "file":
+                    if not isinstance(c, dict):
+                        # Content list items aren't always dicts. e.g.
+                        # text_completion forwards a token-ids list/list-of-
+                        # lists through this path. Skip non-dict items
+                        # instead of indexing into them.
+                        continue
+                    if c.get("type") == "file":
                         file_object = cast(ChatCompletionFileObject, c)
                         file_object_file_field = file_object.get("file")
                         if not isinstance(file_object_file_field, dict):
@@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
                         if file_id:
                             provider_file_id = (
                                 model_file_id_mapping.get(file_id, {}).get(model_id)
-                                or file_id
+                                if model_file_id_mapping
+                                else None
+                            )
+                            if (
+                                not provider_file_id
+                                and _is_base64_encoded_unified_file_id(file_id)
+                            ):
+                                unified_file_id = convert_b64_uid_to_unified_uid(
+                                    file_id
+                                )
+                                if "llm_output_file_id," in unified_file_id:
+                                    provider_file_id = unified_file_id.split(
+                                        "llm_output_file_id,"
+                                    )[1].split(";")[0]
+                            file_object_file_field["file_id"] = (
+                                provider_file_id or file_id
                             )
-                            file_object_file_field["file_id"] = provider_file_id
                         if format:
                             file_object_file_field["format"] = format
     return messages

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
     return sanitized
 
 
+_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"}
+
+
+def _is_anthropic_document_data_uri(url: str) -> bool:
+    # Anthropic's base64 document source accepts only application/pdf and
+    # text/plain (see select_anthropic_content_block_type_for_file). Routing
+    # other mimes here would produce a document block the API rejects, so we
+    # leave them on the image code path.
+    match = re.match(r"data:([^;,]+)", url)
+    if not match:
+        return False
+    return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
+
+
 def convert_to_anthropic_tool_result(
     message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
     force_base64: bool = False,
@@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result(
     """
     anthropic_content: Union[
         str,
-        List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]],
+        List[
+            Union[
+                AnthropicMessagesToolResultContent,
+                AnthropicMessagesImageParam,
+                AnthropicMessagesDocumentParam,
+            ]
+        ],
     ] = ""
     if isinstance(message["content"], str):
         anthropic_content = message["content"]
     elif isinstance(message["content"], List):
         content_list = message["content"]
         anthropic_content_list: List[
-            Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
+            Union[
+                AnthropicMessagesToolResultContent,
+                AnthropicMessagesImageParam,
+                AnthropicMessagesDocumentParam,
+            ]
         ] = []
         for content in content_list:
             if content["type"] == "text":
@@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result(
                     text_content["cache_control"] = cache_control_value
                 anthropic_content_list.append(text_content)
             elif content["type"] == "image_url":
+                image_url_value = content["image_url"]
                 format = (
-                    content["image_url"].get("format")
-                    if isinstance(content["image_url"], dict)
+                    image_url_value.get("format")
+                    if isinstance(image_url_value, dict)
                     else None
                 )
-                _anthropic_image_param = create_anthropic_image_param(
-                    content["image_url"], format=format, is_bedrock_invoke=force_base64
+                url_str = (
+                    image_url_value.get("url")
+                    if isinstance(image_url_value, dict)
+                    else image_url_value
                 )
-                _anthropic_image_param = add_cache_control_to_content(
-                    anthropic_content_element=_anthropic_image_param,
+                # Data URIs with non-image mime types (e.g. application/pdf) must
+                # translate to Anthropic document blocks, not image blocks —
+                # wrapping a PDF in `type: "image"` is rejected by the API.
+                if isinstance(url_str, str) and _is_anthropic_document_data_uri(
+                    url_str
+                ):
+                    synth_file_message: ChatCompletionFileObject = {
+                        "type": "file",
+                        "file": {"file_data": url_str},
+                    }
+                    _document_block = anthropic_process_openai_file_message(
+                        synth_file_message
+                    )
+                    _document_block = add_cache_control_to_content(
+                        anthropic_content_element=cast(
+                            AnthropicMessagesDocumentParam, _document_block
+                        ),
+                        original_content_element=content,
+                    )
+                    anthropic_content_list.append(
+                        cast(AnthropicMessagesDocumentParam, _document_block)
+                    )
+                else:
+                    _anthropic_image_param = create_anthropic_image_param(
+                        image_url_value,
+                        format=format,
+                        is_bedrock_invoke=force_base64,
+                    )
+                    _anthropic_image_param = add_cache_control_to_content(
+                        anthropic_content_element=_anthropic_image_param,
+                        original_content_element=content,
+                    )
+                    anthropic_content_list.append(
+                        cast(AnthropicMessagesImageParam, _anthropic_image_param)
+                    )
+            elif content["type"] == "file":
+                file_content = cast(ChatCompletionFileObject, content)
+                _file_block = anthropic_process_openai_file_message(file_content)
+                _file_block = add_cache_control_to_content(
+                    anthropic_content_element=cast(
+                        AnthropicMessagesDocumentParam, _file_block
+                    ),
                     original_content_element=content,
                 )
-                anthropic_content_list.append(
-                    cast(AnthropicMessagesImageParam, _anthropic_image_param)
-                )
+                anthropic_content_list.append(_file_block)
 
         anthropic_content = anthropic_content_list
     anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@@ -2059,27 +2124,62 @@ def anthropic_process_openai_file_message(
     )
 
 
+_EMPTY_TEXT_PLACEHOLDER = (
+    "[System: Empty message content sanitised to satisfy protocol]"
+)
+
+
 def _sanitize_empty_text_content(
     message: AllMessageValues,
 ) -> AllMessageValues:
     """
     Case C: Sanitize empty text content
     - Replace empty or whitespace-only text content with a placeholder message.
+    - Handles both string content and list-of-blocks content (rewriting only
+      the empty text blocks in place; non-text blocks like images are left
+      untouched).
 
     Returns:
         The message with sanitized content if needed, otherwise the original message
     """
-    if message.get("role") in ["user", "assistant"]:
-        content = message.get("content")
-        if isinstance(content, str):
-            if not content or not content.strip():
-                message = cast(AllMessageValues, dict(message))  # Make a copy
-                message["content"] = (
-                    "[System: Empty message content sanitised to satisfy protocol]"
-                )
-                verbose_logger.debug(
-                    f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
-                )
+    if message.get("role") not in ["user", "assistant"]:
+        return message
+
+    content = message.get("content")
+
+    if isinstance(content, str):
+        if not content or not content.strip():
+            message = cast(AllMessageValues, dict(message))  # Make a copy
+            message["content"] = _EMPTY_TEXT_PLACEHOLDER
+            verbose_logger.debug(
+                f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message"
+            )
+        return message
+
+    if isinstance(content, list):
+        # Walk the blocks and rewrite any empty text blocks. We rewrite (rather
+        # than drop) so callers don't end up with an entirely empty content
+        # list, which Anthropic also rejects.
+        new_blocks: List[Any] = []
+        rewrote_any = False
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                text = block.get("text")
+                if not isinstance(text, str) or not text or not text.strip():
+                    new_block = dict(block)
+                    new_block["text"] = _EMPTY_TEXT_PLACEHOLDER
+                    new_blocks.append(new_block)
+                    rewrote_any = True
+                    continue
+            new_blocks.append(block)
+
+        if rewrote_any:
+            message = cast(AllMessageValues, dict(message))  # Make a copy
+            message["content"] = new_blocks  # type: ignore
+            verbose_logger.debug(
+                f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message"
+            )
+
     return message
 
 
@@ -2362,6 +2462,18 @@ def anthropic_messages_pt(  # noqa: PLR0915
     # Sanitize messages for tool calling issues when modify_params=True
     messages = sanitize_messages_for_tool_calling(messages)
 
+    # Anthropic rejects empty text content blocks with:
+    #   "messages: text content blocks must be non-empty"
+    # OpenAI/other providers silently tolerate `{"role": "user", "content": ""}`,
+    # so callers (and upstream agent frameworks like pydantic-ai) routinely
+    # send empty user/assistant turns. We always rewrite these to a placeholder
+    # for Anthropic-shaped requests, independent of `litellm.modify_params`,
+    # because there is no way to "pass through" an empty text block — the
+    # request will always 400 otherwise. The richer tool-call sanitization
+    # (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on
+    # `modify_params` because it actually mutates conversation structure.
+    messages = [_sanitize_empty_text_content(m) for m in messages]
+
     # add role=tool support to allow function call result/error submission
     user_message_types = {"user", "tool", "function"}
     # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
@@ -3977,6 +4089,55 @@ def _convert_to_bedrock_tool_call_result(
                     tool_result_content_blocks.append(
                         BedrockToolResultContentBlock(image=_block["image"])
                     )
+                elif "document" in _block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(document=_block["document"])
+                    )
+                else:
+                    verbose_logger.warning(
+                        "Bedrock Converse: unrecognized BedrockContentBlock keys "
+                        "%s for image_url tool-result block %s; dropping.",
+                        list(_block.keys()),
+                        content,
+                    )
+            elif content["type"] == "file":
+                # Match the user-message path (_process_file_message): accept
+                # either file_data (base64 data URI) or file_id (server-side
+                # reference / URL) and hand off to BedrockImageProcessor. Raise
+                # BadRequestError on both-None rather than silently dropping.
+                file_obj = content.get("file") or {}
+                file_data = file_obj.get("file_data")
+                file_id = file_obj.get("file_id")
+                if file_data is None and file_id is None:
+                    raise litellm.BadRequestError(
+                        message="file_data and file_id cannot both be None. Got={}".format(
+                            content
+                        ),
+                        model="",
+                        llm_provider="bedrock",
+                    )
+                file_format = file_obj.get("format")
+                _file_block: BedrockContentBlock = (
+                    BedrockImageProcessor.process_image_sync(
+                        image_url=cast(str, file_id or file_data),
+                        format=file_format,
+                    )
+                )
+                if "document" in _file_block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(document=_file_block["document"])
+                    )
+                elif "image" in _file_block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(image=_file_block["image"])
+                    )
+                else:
+                    verbose_logger.warning(
+                        "Bedrock Converse: unrecognized BedrockContentBlock keys "
+                        "%s for file tool-result block %s; dropping.",
+                        list(_file_block.keys()),
+                        content,
+                    )
 
     message.get("name", "")
     id = str(message.get("tool_call_id", str(uuid.uuid4())))
@@ -4468,6 +4629,11 @@ class BedrockConverseMessagesProcessor:
                                     message=cast(ChatCompletionFileObject, element)
                                 )
                                 _parts.append(_part)
+                            elif element["type"] == "document":
+                                _part = BedrockConverseMessagesProcessor._process_document_message(
+                                    element
+                                )
+                                _parts.append(_part)
                             _cache_point_block = (
                                 litellm.AmazonConverseConfig()._get_cache_point_block(
                                     message_block=cast(
@@ -4750,6 +4916,44 @@ class BedrockConverseMessagesProcessor:
             image_url=cast(str, file_id or file_data), format=format
         )
 
+    @staticmethod
+    def _process_document_message(element: dict) -> BedrockContentBlock:
+        """Convert a document content block to a Bedrock DocumentBlock.
+
+        Handles the Anthropic-style document format:
+        {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "..."}}
+        """
+        source = element["source"]
+        source_type = source.get("type")
+        if source_type != "base64":
+            raise ValueError(
+                f"Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. "
+                "Please convert the document to base64 before sending to Bedrock."
+            )
+        media_type: str = source["media_type"]
+        data: str = source["data"]
+        doc_format = BedrockImageProcessor._validate_format(
+            mime_type=media_type, image_format=media_type.split("/")[1]
+        )
+
+        # Deterministic name using the same hashing pattern as _create_bedrock_block
+        HASH_SAMPLE_BYTES = 64 * 1024
+        normalized = "".join(data.split()).encode("utf-8")
+        sample = normalized[:HASH_SAMPLE_BYTES]
+        hasher = hashlib.sha256()
+        hasher.update(sample)
+        hasher.update(str(len(normalized)).encode("utf-8"))
+        content_hash = hasher.hexdigest()[:16]
+        document_name = f"Document_{content_hash}_{doc_format}"
+
+        return BedrockContentBlock(
+            document=BedrockDocumentBlock(
+                source=BedrockSourceBlock(bytes=data),
+                format=doc_format,
+                name=document_name,
+            )
+        )
+
     @staticmethod
     def add_thinking_blocks_to_assistant_content(
         thinking_blocks: List[BedrockContentBlock],
@@ -4847,6 +5051,11 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                                 )
                             )
                             _parts.append(_part)
+                        elif element["type"] == "document":
+                            _part = BedrockConverseMessagesProcessor._process_document_message(
+                                element
+                            )
+                            _parts.append(_part)
                         _cache_point_block = (
                             litellm.AmazonConverseConfig()._get_cache_point_block(
                                 message_block=cast(

litellm/litellm_core_utils/realtime_streaming.py

@@ -31,6 +31,8 @@ DefaultLoggedRealTimeEventTypes = [
     "session.created",
     "response.create",
     "response.done",
+    "conversation.item.added",  # GA
+    "conversation.item.done",  # GA
 ]
 
 
@@ -44,6 +46,7 @@ class RealTimeStreaming:
         model: str = "",
         user_api_key_dict: Optional[Any] = None,
         request_data: Optional[Dict] = None,
+        backend_uses_beta_protocol: Optional[bool] = None,
     ):
         self.websocket = websocket
         self.backend_ws = backend_ws
@@ -54,6 +57,14 @@ class RealTimeStreaming:
         self.session_tools: List[Dict] = []
         self.tool_calls: List[Dict] = []
 
+        # Detect whether the client is explicitly opting into the beta protocol.
+        self._client_wants_beta = self._detect_beta_header(websocket)
+        self._backend_uses_beta_protocol = (
+            self._client_wants_beta
+            if backend_uses_beta_protocol is None
+            else backend_uses_beta_protocol
+        )
+
         _logged_real_time_event_types = litellm.logged_real_time_event_types
 
         if _logged_real_time_event_types is None:
@@ -76,6 +87,27 @@ class RealTimeStreaming:
         # response.create can be rewritten to include the failure context.
         self._pending_guardrail_message: Optional[str] = None
 
+    _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"])
+    _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = {
+        "pcm16": {"type": "audio/pcm", "rate": 24000},
+        "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
+        "g711_alaw": {"type": "audio/G711-alaw", "rate": 8000},
+    }
+    # GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1)
+    _GA_TO_BETA_EVENT_TYPES: Dict[str, str] = {
+        "conversation.item.added": "conversation.item.created",
+        "response.output_text.delta": "response.text.delta",
+        "response.output_audio.delta": "response.audio.delta",
+        "response.output_audio_transcript.delta": "response.audio_transcript.delta",
+        "response.output_text.done": "response.text.done",
+        "response.output_audio.done": "response.audio.done",
+        "response.output_audio_transcript.done": "response.audio_transcript.done",
+    }
+    _GA_TO_BETA_CONTENT_TYPES: Dict[str, str] = {
+        "output_text": "text",
+        "output_audio": "audio",
+    }
+
     def _should_store_message(
         self,
         message_obj: Union[dict, OpenAIRealtimeEvents],
@@ -92,24 +124,27 @@ class RealTimeStreaming:
         if isinstance(message, bytes):
             message = message.decode("utf-8")
         if isinstance(message, dict):
-            message_obj = message
+            # TypedDict union members do not narrow to plain dict for mypy.
+            message_obj: Dict[str, Any] = cast(Dict[str, Any], message)
         else:
-            message_obj = json.loads(message)
+            message_obj = cast(Dict[str, Any], json.loads(cast(str, message)))
         self._collect_tool_calls_from_response_done(cast(dict, message_obj))
         try:
-            if (
-                not isinstance(message, dict)
-                or message_obj.get("type") == "session.created"
-                or message_obj.get("type") == "session.updated"
-            ):
-                message_obj = OpenAIRealtimeStreamSessionEvents(**message_obj)  # type: ignore
-            elif not isinstance(message, dict):
-                message_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj)  # type: ignore
+            event_type = message_obj.get("type", "")
+            if event_type in self._SESSION_EVENT_TYPES:
+                typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj)  # type: ignore
+            else:
+                # Use the base object as a safe catch-all for all other event types
+                # (both beta and GA), so unknown/new event names never raise here.
+                typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj)  # type: ignore
         except Exception as e:
             verbose_logger.debug(f"Error parsing message for logging: {e}")
-            raise e
-        if self._should_store_message(message_obj):
-            self.messages.append(message_obj)
+            # Don't re-raise — a parse failure must not drop or delay the message
+            if self._should_store_message(message_obj):
+                self.messages.append(message_obj)  # type: ignore[arg-type]
+            return
+        if self._should_store_message(typed_obj):
+            self.messages.append(typed_obj)
 
     def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None:
         """Extract user text content from client WebSocket events for spend logging."""
@@ -147,6 +182,8 @@ class RealTimeStreaming:
                 tools = session.get("tools")
                 if tools and isinstance(tools, list):
                     self.session_tools = tools
+                # GA: session.type is required; log it for traceability but no action needed
+                verbose_logger.debug(f"Realtime session.type: {session.get('type')}")
         except (json.JSONDecodeError, AttributeError, TypeError):
             pass
 
@@ -228,6 +265,23 @@ class RealTimeStreaming:
         else:
             await self.backend_ws.send(message)  # type: ignore[union-attr, attr-defined]
 
+    def _make_disable_auto_response_message(self) -> str:
+        """Return a session.update that disables VAD auto-response."""
+        if self._backend_uses_beta_protocol:
+            session: Dict[str, Any] = {
+                "turn_detection": {"create_response": False},
+            }
+        else:
+            session = {
+                "type": "realtime",
+                "audio": {
+                    "input": {
+                        "turn_detection": {"create_response": False},
+                    }
+                },
+            }
+        return json.dumps({"type": "session.update", "session": session})
+
     def _has_realtime_guardrails(self) -> bool:
         """Return True if any callback is registered for realtime guardrail event types."""
         from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -435,14 +489,7 @@ class RealTimeStreaming:
             ):
                 self.store_message(event_str)
                 await self.websocket.send_text(event_str)
-                await self._send_to_backend(
-                    json.dumps(
-                        {
-                            "type": "session.update",
-                            "session": {"turn_detection": {"create_response": False}},
-                        }
-                    )
-                )
+                await self._send_to_backend(self._make_disable_auto_response_message())
                 continue
             ## GUARDRAIL: run on transcription events in provider_config path too
             if (
@@ -484,14 +531,7 @@ class RealTimeStreaming:
             ):
                 self.store_message(raw_response)
                 await self.websocket.send_text(raw_response)
-                await self._send_to_backend(
-                    json.dumps(
-                        {
-                            "type": "session.update",
-                            "session": {"turn_detection": {"create_response": False}},
-                        }
-                    )
-                )
+                await self._send_to_backend(self._make_disable_auto_response_message())
                 return True
 
             if (
@@ -542,7 +582,20 @@ class RealTimeStreaming:
                         continue
                     ## LOGGING
                     self.store_message(raw_response)
-                    await self.websocket.send_text(raw_response)
+
+                    # If the client opted into beta protocol, translate GA event
+                    # names/shapes back to the beta equivalents before forwarding.
+                    if self._client_wants_beta:
+                        try:
+                            event_dict = json.loads(raw_response)
+                            translated = self._translate_event_to_beta(event_dict)
+                            if translated is None:
+                                continue  # drop GA-only events (e.g. conversation.item.done)
+                            await self.websocket.send_text(json.dumps(translated))
+                        except Exception:
+                            await self.websocket.send_text(raw_response)
+                    else:
+                        await self.websocket.send_text(raw_response)
 
         except websockets.exceptions.ConnectionClosed as e:  # type: ignore
             verbose_logger.exception(
@@ -553,6 +606,183 @@ class RealTimeStreaming:
         finally:
             await self.log_messages()
 
+    @staticmethod
+    def _detect_beta_header(websocket: Any) -> bool:
+        """Return True if the client sent 'OpenAI-Beta: realtime=v1'.
+
+        Checks the raw ASGI scope headers so it works for both FastAPI WebSocket
+        objects and any test doubles that expose a .scope dict.
+        """
+        try:
+            headers = websocket.scope.get("headers", [])
+            for name, value in headers:
+                if isinstance(name, bytes):
+                    name = name.decode("latin-1")
+                if isinstance(value, bytes):
+                    value = value.decode("latin-1")
+                if name.lower() == "openai-beta" and "realtime=v1" in value.lower():
+                    return True
+        except Exception:
+            pass
+        return False
+
+    @staticmethod
+    def _remap_beta_session_to_ga(session: dict) -> dict:
+        """
+        Convert a beta-style session.update payload to the GA nested schema.
+
+        Beta → GA field mappings
+        ─────────────────────────────────────────────────────────────────────
+        session.type                    (inject "realtime" if absent)
+        session.modalities              → session.output_modalities
+        session.voice                   → session.audio.output.voice
+        session.input_audio_format      → session.audio.input.format  (with type/rate)
+        session.output_audio_format     → session.audio.output.format (with type/rate)
+        session.turn_detection          → session.audio.input.turn_detection
+        session.input_audio_transcription → session.audio.input.transcription
+        ─────────────────────────────────────────────────────────────────────
+        Fields not in the mapping (instructions, tools, etc.) are passed through.
+        GA clients that already use the nested shape are unaffected.
+        """
+        # Work on a shallow copy so we don't mutate the caller's dict
+        session = dict(session)
+
+        # 1. Ensure session.type is present
+        if "type" not in session:
+            session["type"] = "realtime"
+
+        # 2. Rename modalities → output_modalities and normalise combinations.
+        # Beta allowed ["audio", "text"] together; GA only supports ["audio"] or
+        # ["text"] as single-element lists. When both are present we prefer
+        # ["audio"] because audio mode already delivers transcripts via events.
+        if "modalities" in session:
+            mods = session.pop("modalities")
+            if "output_modalities" not in session:
+                mods_set = {m.lower() for m in (mods or [])}
+                if "audio" in mods_set:
+                    session["output_modalities"] = ["audio"]
+                elif "text" in mods_set:
+                    session["output_modalities"] = ["text"]
+
+        # 3-7. Lift flat audio fields into the nested audio object
+        audio: Dict[str, Any] = {}
+        inp: Dict[str, Any] = {}
+        out: Dict[str, Any] = {}
+
+        # voice → audio.output.voice
+        if "voice" in session:
+            out["voice"] = session.pop("voice")
+
+        # input_audio_format → audio.input.format
+        if "input_audio_format" in session:
+            raw = session.pop("input_audio_format")
+            inp["format"] = (
+                RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
+                if isinstance(raw, str)
+                else raw
+            )
+
+        # output_audio_format → audio.output.format
+        if "output_audio_format" in session:
+            raw = session.pop("output_audio_format")
+            out["format"] = (
+                RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw)
+                if isinstance(raw, str)
+                else raw
+            )
+
+        # turn_detection → audio.input.turn_detection
+        if "turn_detection" in session:
+            inp["turn_detection"] = session.pop("turn_detection")
+
+        # input_audio_transcription → audio.input.transcription
+        if "input_audio_transcription" in session:
+            inp["transcription"] = session.pop("input_audio_transcription")
+
+        if inp:
+            audio["input"] = inp
+        if out:
+            audio["output"] = out
+
+        if audio:
+            # Merge with any existing GA-style `audio` block the client already set,
+            # letting the remapped values take precedence within each sub-key.
+            existing = session.get("audio") or {}
+            for sub_key, sub_val in audio.items():
+                if (
+                    sub_key in existing
+                    and isinstance(existing[sub_key], dict)
+                    and isinstance(sub_val, dict)
+                ):
+                    existing[sub_key] = {**existing[sub_key], **sub_val}
+                else:
+                    existing[sub_key] = sub_val
+            session["audio"] = existing
+
+        return session
+
+    @staticmethod
+    def _translate_event_to_beta(event: dict) -> Optional[dict]:
+        """Translate a single GA event dict to its beta equivalent.
+
+        Returns None if the event should be dropped entirely (e.g. the GA-only
+        conversation.item.done has no beta counterpart).
+        Returns the (possibly mutated copy of the) event otherwise.
+        """
+        event_type = event.get("type", "")
+
+        # conversation.item.done has no beta equivalent — the client already
+        # received conversation.item.created (translated from .added).
+        if event_type == "conversation.item.done":
+            return None
+
+        # Shallow-copy so we don't mutate the stored message
+        translated = dict(event)
+
+        # Rename the type field
+        if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES:
+            translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type]
+
+        # Fix content block types inside items (response.done output list,
+        # conversation.item.created item content, etc.)
+        if "item" in translated and isinstance(translated["item"], dict):
+            translated["item"] = RealTimeStreaming._translate_item_content_types(
+                dict(translated["item"])
+            )
+        if "response" in translated and isinstance(translated["response"], dict):
+            resp = dict(translated["response"])
+            if "output" in resp and isinstance(resp["output"], list):
+                resp["output"] = [
+                    (
+                        RealTimeStreaming._translate_item_content_types(dict(o))
+                        if isinstance(o, dict)
+                        else o
+                    )
+                    for o in resp["output"]
+                ]
+            translated["response"] = resp
+
+        return translated
+
+    @staticmethod
+    def _translate_item_content_types(item: dict) -> dict:
+        """Replace GA content type names with beta names inside a single item."""
+        if "content" not in item or not isinstance(item["content"], list):
+            return item
+        new_content = []
+        for block in item["content"]:
+            if (
+                isinstance(block, dict)
+                and block.get("type") in RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES
+            ):
+                block = dict(block)
+                block["type"] = RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES[
+                    block["type"]
+                ]
+            new_content.append(block)
+        item["content"] = new_content
+        return item
+
     async def client_ack_messages(self):
         try:
             while True:
@@ -594,6 +824,19 @@ class RealTimeStreaming:
                         self._pending_guardrail_message = None
                         continue
 
+                    # GA compatibility: remap beta-style session fields only when
+                    # the upstream is in GA mode. Beta upstreams expect the flat
+                    # session shape unchanged.
+                    if (
+                        msg_type == "session.update"
+                        and not self._backend_uses_beta_protocol
+                    ):
+                        session = msg_obj.get("session", {})
+                        if isinstance(session, dict):
+                            session = self._remap_beta_session_to_ga(session)
+                            msg_obj["session"] = session
+                            message = json.dumps(msg_obj)
+
                 except (json.JSONDecodeError, AttributeError):
                     pass
 
@@ -627,3 +870,8 @@ class RealTimeStreaming:
                     await forward_task
                 except asyncio.CancelledError:
                     pass
+
+
+def client_sent_openai_beta_realtime_header(websocket: Any) -> bool:
+    """True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``."""
+    return RealTimeStreaming._detect_beta_header(websocket)

litellm/litellm_core_utils/redact_messages.py

@@ -60,6 +60,9 @@ def _redact_choice_content(choice):
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
+        if hasattr(output_item, "text"):
+            output_item.text = "redacted-by-litellm"
+
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
                 if hasattr(content_part, "text"):
@@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items):
                         summary_item.text = "redacted-by-litellm"
 
 
+def _redact_responses_api_output_dict(output_items, redacted_str: str):
+    """Helper to redact ResponsesAPIResponse output items in dict form."""
+    for output_item in output_items:
+        if not isinstance(output_item, dict):
+            continue
+
+        if "text" in output_item:
+            output_item["text"] = redacted_str
+
+        if isinstance(output_item.get("content"), list):
+            for content_item in output_item["content"]:
+                if isinstance(content_item, dict) and "text" in content_item:
+                    content_item["text"] = redacted_str
+
+        if output_item.get("type") == "reasoning" and isinstance(
+            output_item.get("summary"), list
+        ):
+            for summary_item in output_item["summary"]:
+                if isinstance(summary_item, dict) and "text" in summary_item:
+                    summary_item["text"] = redacted_str
+
+
 def _redact_standard_logging_object(model_call_details: dict):
     """Redact messages and response inside standard_logging_object if present."""
     standard_logging_object = model_call_details.get("standard_logging_object")
@@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict):
         if isinstance(response, dict) and "output" in response:
             # ResponsesAPIResponse format - redact content in output items
             if isinstance(response.get("output"), list):
-                for output_item in response["output"]:
-                    if isinstance(output_item, dict) and "content" in output_item:
-                        if isinstance(output_item["content"], list):
-                            for content_item in output_item["content"]:
-                                if (
-                                    isinstance(content_item, dict)
-                                    and "text" in content_item
-                                ):
-                                    content_item["text"] = redacted_str
+                _redact_responses_api_output_dict(response["output"], redacted_str)
         elif isinstance(response, dict) and "choices" in response:
             # ModelResponse dict format - redact content in choices
             if isinstance(response.get("choices"), list):
-                for choice in response["choices"]:
-                    if isinstance(choice, dict):
-                        if "message" in choice and isinstance(choice["message"], dict):
-                            choice["message"]["content"] = redacted_str
-                            if "audio" in choice["message"]:
-                                choice["message"]["audio"] = None
-                        elif "delta" in choice and isinstance(choice["delta"], dict):
-                            choice["delta"]["content"] = redacted_str
-                            if "audio" in choice["delta"]:
-                                choice["delta"]["audio"] = None
+                _redact_model_response_dict_choices(response["choices"], redacted_str)
         elif isinstance(response, str):
             standard_logging_object["response"] = redacted_str
         else:
@@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
+def _redact_model_response_dict_choices(choices, redacted_str: str):
+    for choice in choices:
+        if isinstance(choice, dict):
+            if "message" in choice and isinstance(choice["message"], dict):
+                choice["message"]["content"] = redacted_str
+                if "reasoning_content" in choice["message"]:
+                    choice["message"]["reasoning_content"] = redacted_str
+                if "thinking_blocks" in choice["message"]:
+                    choice["message"]["thinking_blocks"] = None
+                if "audio" in choice["message"]:
+                    choice["message"]["audio"] = None
+            elif "delta" in choice and isinstance(choice["delta"], dict):
+                choice["delta"]["content"] = redacted_str
+                if "reasoning_content" in choice["delta"]:
+                    choice["delta"]["reasoning_content"] = redacted_str
+                if "thinking_blocks" in choice["delta"]:
+                    choice["delta"]["thinking_blocks"] = None
+                if "audio" in choice["delta"]:
+                    choice["delta"]["audio"] = None
+        else:
+            _redact_choice_content(choice)
+
+
 def perform_redaction(model_call_details: dict, result):
     """
     Performs the actual redaction on the logging object and result.
@@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result):
     ]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""
+    _redact_standard_logging_object(model_call_details)
 
     # Redact streaming response
     if (
@@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result):
         elif isinstance(_result, dict) and "choices" in _result:
             # Handle dict representation of ModelResponse (e.g., from model_dump())
             if _result.get("choices") is not None:
-                for choice in _result["choices"]:
-                    if isinstance(choice, dict):
-                        if "message" in choice and isinstance(choice["message"], dict):
-                            choice["message"]["content"] = "redacted-by-litellm"
-                            if "reasoning_content" in choice["message"]:
-                                choice["message"][
-                                    "reasoning_content"
-                                ] = "redacted-by-litellm"
-                            if "thinking_blocks" in choice["message"]:
-                                choice["message"]["thinking_blocks"] = None
-                            if "audio" in choice["message"]:
-                                choice["message"]["audio"] = None
-                        elif "delta" in choice and isinstance(choice["delta"], dict):
-                            choice["delta"]["content"] = "redacted-by-litellm"
-                            if "reasoning_content" in choice["delta"]:
-                                choice["delta"][
-                                    "reasoning_content"
-                                ] = "redacted-by-litellm"
-                            if "thinking_blocks" in choice["delta"]:
-                                choice["delta"]["thinking_blocks"] = None
-                            if "audio" in choice["delta"]:
-                                choice["delta"]["audio"] = None
-                    else:
-                        _redact_choice_content(choice)
+                _redact_model_response_dict_choices(
+                    _result["choices"], "redacted-by-litellm"
+                )
+        elif isinstance(_result, dict) and "output" in _result:
+            if isinstance(_result.get("output"), list):
+                _redact_responses_api_output_dict(
+                    _result["output"], "redacted-by-litellm"
+                )
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)

litellm/litellm_core_utils/secret_redaction.py

@@ -0,0 +1,81 @@
+"""
+Credential/secret redaction utilities.
+
+This module owns the compiled regex and the public `redact_string` helper so
+that any part of the codebase (logging, exception mapping, etc.) can scrub
+secrets from strings without depending on the logging-configuration module.
+"""
+
+import re
+from typing import List
+
+_REDACTED = "REDACTED"
+
+
+def _build_secret_patterns() -> "re.Pattern[str]":
+    patterns: List[str] = [
+        # PEM private key / certificate blocks
+        r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
+        # GCP OAuth2 access tokens (ya29.*)
+        r"\bya29\.[A-Za-z0-9_.~+/-]+",
+        # Credential %s formatting (space separator, no key= prefix)
+        r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
+        # AWS access key IDs
+        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
+        # AWS secrets / session tokens / access key IDs (key=value)
+        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
+        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
+        # Bearer tokens (OAuth, JWT, etc.)
+        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
+        # Basic auth headers
+        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
+        # OpenAI / Anthropic sk- prefixed keys
+        r"sk-[A-Za-z0-9\-_]{20,}",
+        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
+        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
+        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
+        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Anthropic internal header keys
+        r"x-ak-[A-Za-z0-9\-_]{20,}",
+        # Google API keys (bare key value)
+        r"AIza[0-9A-Za-z\-_]{35}",
+        # URL query-param key=VALUE (e.g. ?key=AIza... or &key=...) — catches the
+        # full "key=<secret>" fragment so the value is redacted regardless of format.
+        r"(?<=[?&])key=[^\s&'\"]{8,}",
+        # Password / secret params (handles key=value and 'key': 'value')
+        # Word boundary prevents O(n^2) backtracking on long word-char runs.
+        r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
+        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Database connection string credentials (scheme://user:pass@host)
+        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
+        # Databricks personal access tokens
+        r"dapi[0-9a-f]{32}",
+        # ── Key-name-based redaction ──
+        # Catches secrets inside dicts/config dumps by matching on the KEY name
+        # regardless of what the value looks like.
+        # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
+        # private_key with PEM-aware value capture
+        r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
+        r"(?:master_key|database_url|db_url|connection_string|"
+        r"signing_key|encryption_key|"
+        r"auth_token|access_token|refresh_token|"
+        r"slack_webhook_url|webhook_url|"
+        r"database_connection_string|"
+        r"huggingface_token|jwt_secret)"
+        r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
+        # Raw JWTs (without Bearer prefix)
+        r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
+        # Azure SAS tokens in URLs
+        r"[?&]sig=[A-Za-z0-9%+/=]+",
+        # Full JSON service-account blobs (single-line and multi-line)
+        r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
+    ]
+    return re.compile("|".join(patterns), re.IGNORECASE)
+
+
+_SECRET_RE = _build_secret_patterns()
+
+
+def redact_string(value: str) -> str:
+    """Scrub known secret/credential patterns from *value* and return the result."""
+    return _SECRET_RE.sub(_REDACTED, value)

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -21,6 +21,8 @@ class SensitiveDataMasker:
             "auth",
             "authorization",
             "credential",
+            # Plural form: Vertex uses ``vertex_credentials``; segment-exact
+            # matching otherwise misses it because "credential" != "credentials".
             "credentials",
             "access",
             "private",

litellm/litellm_core_utils/streaming_handler.py

@@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
                 asyncio.create_task(
                     self.logging_obj.async_failure_handler(e, traceback_exception)
                 )
-            raise e
+            self._handle_stream_fallback_error(e)
         except Exception as e:
             traceback_exception = traceback.format_exc()
             if self.logging_obj is not None:

litellm/litellm_core_utils/url_utils.py

@@ -21,8 +21,8 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
 
 import socket
 from ipaddress import ip_address, ip_network
-from typing import Any, List, Set, Tuple
-from urllib.parse import urlparse, urlunparse
+from typing import Any, List, Optional, Set, Tuple
+from urllib.parse import quote, urlparse, urlunparse
 
 import httpx
 
@@ -46,6 +46,46 @@ class SSRFError(ValueError):
     pass
 
 
+def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str:
+    """Percent-encode one user-controlled URL path segment.
+
+    ``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986
+    unreserved characters such as ``.`` unescaped, so reject standalone dot
+    segments before they can be appended to an upstream URL and normalized by
+    the HTTP client.
+    """
+    if value is None:
+        raise ValueError(f"{field_name} is required")
+
+    value_str = str(value)
+    if value_str == "":
+        raise ValueError(f"{field_name} is required")
+    if value_str in {".", ".."}:
+        raise ValueError(f"{field_name} cannot be a dot path segment")
+
+    return quote(value_str, safe="")
+
+
+def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str:
+    """Percent-encode a user-controlled URL path made of multiple segments.
+
+    Empty segments are rejected, so leading, trailing, or consecutive slashes
+    fail closed instead of being normalized by the HTTP client.
+    """
+    if value is None:
+        raise ValueError(f"{field_name} is required")
+
+    value_str = str(value)
+    if value_str == "":
+        raise ValueError(f"{field_name} is required")
+
+    encoded_segments = []
+    for segment in value_str.split("/"):
+        encoded_segments.append(encode_url_path_segment(segment, field_name=field_name))
+
+    return "/".join(encoded_segments)
+
+
 def _is_blocked_ip(addr: str) -> bool:
     """Return True for any IP not safe to reach from a user-supplied URL.
 
@@ -70,6 +110,85 @@ def _normalize_host(host: str) -> str:
     return host.lower().rstrip(".")
 
 
+def _default_port_for_scheme(scheme: str) -> int:
+    return 443 if scheme == "https" else 80
+
+
+def _parse_url_destination_allowlist_entry(
+    entry: str,
+) -> Optional[Tuple[str, Optional[str], Optional[int]]]:
+    """Parse an admin allowlist entry into host, optional scheme, optional port.
+
+    Entries may be bare hosts (``api.example.com``), host+port
+    (``api.example.com:8443``), or origins (``https://api.example.com``).
+    URL paths are intentionally ignored so admins can paste an api_base value.
+    """
+    entry = entry.strip()
+    if not entry:
+        return None
+
+    has_scheme = "://" in entry
+    parsed = urlparse(entry if has_scheme else f"//{entry}")
+    if has_scheme and parsed.scheme not in _ALLOWED_SCHEMES:
+        return None
+    if parsed.username is not None or parsed.password is not None:
+        return None
+    if not parsed.hostname:
+        return None
+
+    try:
+        port = parsed.port
+    except ValueError:
+        return None
+
+    scheme: Optional[str] = parsed.scheme if has_scheme else None
+    if scheme is not None and port is None:
+        port = _default_port_for_scheme(scheme)
+
+    return _normalize_host(parsed.hostname), scheme, port
+
+
+def is_url_destination_allowed_by_host(url: str, allowed_hosts: List[str]) -> bool:
+    """Return True when a credential-bearing provider URL is admin-allowlisted.
+
+    This does not fetch, resolve, or rewrite URLs. It only answers whether the
+    destination origin is explicitly trusted by configuration. Use ``safe_get``
+    for user-controlled content fetches that require SSRF protection.
+    """
+    parsed = urlparse(url)
+    if parsed.scheme not in _ALLOWED_SCHEMES:
+        return False
+    if parsed.username is not None or parsed.password is not None:
+        return False
+    if not parsed.hostname:
+        return False
+
+    try:
+        effective_port = parsed.port or _default_port_for_scheme(parsed.scheme)
+    except ValueError:
+        return False
+
+    normalized_host = _normalize_host(parsed.hostname)
+    configured_entries = (
+        [allowed_hosts] if isinstance(allowed_hosts, str) else allowed_hosts
+    )
+    for entry in configured_entries or []:
+        if not isinstance(entry, str):
+            continue
+        parsed_entry = _parse_url_destination_allowlist_entry(entry)
+        if parsed_entry is None:
+            continue
+        allowed_host, allowed_scheme, allowed_port = parsed_entry
+        if allowed_host != normalized_host:
+            continue
+        if allowed_scheme is not None and allowed_scheme != parsed.scheme:
+            continue
+        if allowed_port is not None and allowed_port != effective_port:
+            continue
+        return True
+    return False
+
+
 def _format_host_header(hostname: str, port: int, default_port: int) -> str:
     """Build an RFC 7230 Host header value, bracketing IPv6 literals."""
     bracketed = f"[{hostname}]" if ":" in hostname else hostname
@@ -145,7 +264,7 @@ def validate_url(url: str) -> Tuple[str, str]:
         raise SSRFError("URL has no hostname")
 
     port = parsed.port
-    default_port = 443 if parsed.scheme == "https" else 80
+    default_port = _default_port_for_scheme(parsed.scheme)
     effective_port = port if port is not None else default_port
     host_header = _format_host_header(hostname, effective_port, default_port)
 
@@ -199,13 +318,54 @@ def validate_url(url: str) -> Tuple[str, str]:
     return rewritten, host_header
 
 
+def assert_same_origin(candidate_url: str, expected_url: str) -> None:
+    """Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``.
+
+    Use when an upstream API returns a URL meant for follow-up requests
+    (e.g. an async-job polling URL that will be hit with the operator's
+    API key in the headers). The upstream is trusted because the operator
+    configured ``api_base``, but the URL it hands back must actually point
+    back at the same origin or we'd be blindly forwarding credentials
+    wherever the upstream told us to.
+
+    Hostnames are compared case-insensitively. Default ports are made
+    explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...``
+    and ``https://api.example.com/...`` are treated as the same origin.
+
+    Error messages identify *which* component mismatched but never echo
+    the operator's ``expected`` host or the candidate's hostname back to
+    the caller — in the SSRF threat model the caller is the attacker,
+    and reflecting host info would be a secondary leak of operator
+    infrastructure details.
+    """
+    candidate = urlparse(candidate_url)
+    expected = urlparse(expected_url)
+
+    if candidate.scheme not in _ALLOWED_SCHEMES:
+        raise SSRFError("URL scheme is not allowed")
+
+    if candidate.scheme != expected.scheme:
+        raise SSRFError("Origin mismatch on scheme")
+
+    candidate_host = _normalize_host(candidate.hostname or "")
+    expected_host = _normalize_host(expected.hostname or "")
+    if not candidate_host or candidate_host != expected_host:
+        raise SSRFError("Origin mismatch on host")
+
+    default_port = 443 if candidate.scheme == "https" else 80
+    candidate_port = candidate.port if candidate.port is not None else default_port
+    expected_port = expected.port if expected.port is not None else default_port
+    if candidate_port != expected_port:
+        raise SSRFError("Origin mismatch on port")
+
+
 _MAX_REDIRECTS = 10
 
 
 def _extract_redirect_url(response: Any, request_url: str) -> str:
     """Extract and resolve the redirect target from a response's Location header."""
     location = response.headers.get("location")
-    if not location:
+    if not isinstance(location, str) or not location:
         raise SSRFError("Redirect response has no Location header")
     # Resolve relative URLs against the request URL
     return str(httpx.URL(request_url).join(location))