diff --git a/.circleci/config.yml b/.circleci/config.yml index 3019fabd6ff..0966da461ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -350,7 +350,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "langfuse" + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -k \"langfuse\"" no_output_timeout: 15m # Store test results - store_test_results: @@ -395,7 +403,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/proxy_admin_ui_tests -x --junitxml=test-results/junit.xml --durations=5 -n 2 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" no_output_timeout: 15m # Store test results @@ -471,7 +487,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/router_unit_tests -x --junitxml=test-results/junit.xml --durations=5 -n 4 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" no_output_timeout: 15m # Store test results - store_test_results: @@ -495,7 +519,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest tests/local_testing/ -v -k "assistants" -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -k \"assistants\"" no_output_timeout: 15m # Store test results - store_test_results: @@ -528,14 +560,19 @@ jobs: # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging # Subdirectories with dedicated jobs (maintain this list as new jobs are added) - IGNORE_DIRS=( - "tests/llm_translation/realtime" - ) - IGNORE_ARGS="" - for dir in "${IGNORE_DIRS[@]}"; do - IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir" - done - uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5 + mkdir -p test-results + # Glob excludes the realtime/ subdirectory since it has its own job + TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=120 --timeout_method=thread \ + --retries 2 --retry-delay 5 \ + --max-worker-restart=5" no_output_timeout: 15m # Store test results @@ -560,7 +597,17 @@ jobs: command: | # Add --timeout to kill hanging tests after 120s (2 min) # Add --durations=20 to show 20 slowest tests for debugging - uv run --no-sync python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=20 \ + -n 4 \ + --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files @@ -593,7 +640,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/agent_tests --ignore=tests/agent_tests/local_only_agent_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files @@ -626,7 +681,18 @@ jobs: - run: name: Run tests command: | - LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/guardrails_tests -vv --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -n 2 --timeout=120 --timeout_method=thread + mkdir -p test-results + export LITELLM_LOG=WARNING + TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2 \ + --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files @@ -660,7 +726,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/unified_google_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 --retries 3 --retry-delay 5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --retries 3 --retry-delay 5" no_output_timeout: 15m - run: name: Rename the coverage files @@ -702,7 +777,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/llm_responses_api_testing -x --junitxml=test-results/junit.xml --durations=5 -n 8 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 8" no_output_timeout: 15m # Store test results @@ -725,7 +808,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" no_output_timeout: 15m - run: name: Rename the coverage files @@ -758,7 +850,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" no_output_timeout: 15m - run: name: Rename the coverage files @@ -793,7 +894,15 @@ jobs: name: Run enterprise tests command: | uv run --no-sync python -m prisma generate - uv run --no-sync python -m pytest -v tests/enterprise -x --junitxml=test-results/junit-enterprise.xml --durations=10 -n 4 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-enterprise.xml \ + --durations=10 \ + -n 4" no_output_timeout: 15m # Store test results - store_test_results: @@ -815,7 +924,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/batches_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" no_output_timeout: 15m - run: name: Rename the coverage files @@ -848,7 +966,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/litellm_utils_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 -n 2 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 2" no_output_timeout: 15m - run: name: Rename the coverage files @@ -882,7 +1009,16 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" no_output_timeout: 15m - run: name: Rename the coverage files @@ -916,7 +1052,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/image_gen_tests -n 4 -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + -n 4" no_output_timeout: 15m # Store test results - store_test_results: @@ -939,7 +1083,18 @@ jobs: - run: name: Run tests command: | - LITELLM_LOG=WARNING uv run --no-sync python -m pytest tests/logging_callback_tests -vv --cov=litellm --cov-report=xml -n 4 --junitxml=test-results/junit.xml --durations=5 --timeout=120 --timeout_method=thread + mkdir -p test-results + export LITELLM_LOG=WARNING + TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv \ + --cov=litellm --cov-report=xml \ + -n 4 \ + --junitxml=test-results/junit.xml \ + --durations=5 \ + --timeout=120 --timeout_method=thread" no_output_timeout: 15m - run: name: Rename the coverage files @@ -972,7 +1127,15 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m - run: name: Rename the coverage files @@ -1012,14 +1175,19 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv \ + mkdir -p test-results + TEST_FILES=$(printf "%s\n" \ tests/local_testing/test_dual_cache.py \ tests/local_testing/test_redis_batch_optimizations.py \ - tests/local_testing/test_router_utils.py \ - --cov=litellm --cov-report=xml \ - -x -s -v --junitxml=test-results/junit.xml \ - --durations=5 -n 2 \ - --reruns 2 --reruns-delay 1 + tests/local_testing/test_router_utils.py) + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --cov=litellm --cov-report=xml \ + --junitxml=test-results/junit.xml \ + --durations=5 -n 2 \ + --reruns 2 --reruns-delay 1" no_output_timeout: 20m - run: name: Rename the coverage files @@ -1260,8 +1428,17 @@ jobs: - run: name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) command: | - uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" no_output_timeout: 15m + - store_test_results: + path: test-results build_and_test: machine: @@ -1331,7 +1508,18 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -s -v tests/*.py -x --junitxml=test-results/junit.xml -n 4 --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/guardrails_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + mkdir -p test-results + # Original used `tests/*.py` (top-level only); the `--ignore=...` + # flags were vestigial since shell globbing did not descend into + # subdirectories. Replicate by globbing only top-level test files. + TEST_FILES=$(circleci tests glob "tests/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -s -v -x \ + --junitxml=test-results/junit.xml \ + -n 4 \ + --durations=5" no_output_timeout: 15m # Store test results @@ -1406,7 +1594,14 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -s -vv \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m # Store test results @@ -1475,7 +1670,14 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/otel_tests --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m # Clean up first container - run: @@ -1518,7 +1720,14 @@ jobs: - run: name: Run second round of tests command: | - uv run --no-sync python -m pytest -v tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" no_output_timeout: 15m # Store test results @@ -1587,8 +1796,17 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m + - store_test_results: + path: test-results - run: name: Stop and remove first container when: always @@ -1676,7 +1894,14 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/multi_instance_e2e_tests -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m # Clean up first container # Store test results @@ -1732,7 +1957,14 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m - run: name: Stop and remove containers @@ -1805,9 +2037,18 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x \ + --junitxml=test-results/junit-2.xml \ + --durations=5" no_output_timeout: 15m # Clean up first container + - store_test_results: + path: test-results - run: name: Stop and remove first container command: | @@ -1940,7 +2181,14 @@ jobs: - run: name: Run tests command: | - uv run --no-sync python -m pytest -v tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5 + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -v -x \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m # Store test results @@ -1997,9 +2245,16 @@ jobs: - run: name: Run Claude Agent SDK E2E Tests command: | + mkdir -p test-results export LITELLM_PROXY_URL="http://localhost:4000" export LITELLM_API_KEY="sk-1234" - uv run --no-sync python -m pytest -vv tests/proxy_e2e_anthropic_messages_tests/ -x -s --junitxml=test-results/junit.xml --durations=5 + TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") + echo "$TEST_FILES" | circleci tests run \ + --verbose \ + --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + -vv -x -s \ + --junitxml=test-results/junit.xml \ + --durations=5" no_output_timeout: 15m # Store test results diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index d5781f767f6..5a9688db9c4 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -141,6 +141,7 @@ jobs: tests/proxy_unit_tests/test_server_root_path.py tests/proxy_unit_tests/test_proxy_pass_user_config.py tests/proxy_unit_tests/test_proxy_token_counter.py + tests/proxy_unit_tests/test_request_size_limit_middleware.py workers: 4 dist: loadscope timeout: 15 diff --git a/.gitignore b/.gitignore index 59812ed6ed4..20355a8e4ef 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,5 @@ STABILIZATION_TODO.md **/playwright-report **/*.storageState.json **/coverage -test-config \ No newline at end of file +test-config +.vscode \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 915daff999c..9ad9ab31b65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \ - npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \ +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \ + npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \ GLOBAL="$(npm root -g)" && \ for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \ name="${pkg##*/}"; \ @@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}" COPY --from=builder /app /app # Prisma binaries live in $HOME/.cache (default prisma-python location), -# which is /root/.cache here. Copy them from the builder so they survive -# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem -# + emptyDir) — otherwise the mount would shadow the baked-in query engine. -COPY --from=builder /root/.cache /root/.cache +# which is /root/.cache here. Copy only the Prisma subdirs — copying the +# whole /root/.cache drags in the uv build cache (~660 MB, includes a +# setuptools wheel that surfaces as a CVE finding even though it's not +# on the runtime sys.path). +COPY --from=builder /root/.cache/prisma /root/.cache/prisma +COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ find /app/.venv -type d -path "*/tornado/test" -delete EXPOSE 4000/tcp -COPY docker/supervisord.conf /etc/supervisord.conf - ENTRYPOINT ["docker/prod_entrypoint.sh"] CMD ["--port", "4000"] diff --git a/codecov.yaml b/codecov.yaml index 09fccc6b995..8609d3143d6 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -1,3 +1,8 @@ +codecov: + require_ci_to_pass: false # post coverage status even if CI has unrelated failures + notify: + wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI + component_management: individual_components: - component_id: "Router" @@ -28,7 +33,7 @@ coverage: project: default: target: auto - threshold: 1% # at maximum allow project coverage to drop by 1% + threshold: 0% # do not allow project coverage to drop patch: default: target: auto diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 97123e5df69..6aa1771b7bb 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -116,21 +116,32 @@ spec: name: {{ include "redis.secretName" .Subcharts.redis }} key: {{include "redis.secretPasswordKey" .Subcharts.redis }} {{- end }} + {{- /* + Inject LITELLM_LOG only when envVars does not already define it. + */}} + {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} + - name: LITELLM_LOG + value: {{ .Values.logLevel | quote }} + {{- end }} {{- if .Values.envVars }} {{- range $key, $val := .Values.envVars }} - name: {{ $key }} value: {{ $val | quote }} {{- end }} {{- end }} - {{- if .Values.separateHealthApp }} - - name: SEPARATE_HEALTH_APP - value: "1" - - name: SEPARATE_HEALTH_PORT - value: {{ .Values.separateHealthPort | default "8081" | quote }} - {{- end }} {{- with .Values.extraEnvVars }} {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.migrationJob.enabled }} + # Schema updates are owned by the dedicated migrations Job; skip + # the proxy's startup `prisma db push` so N replicas don't race + # one DB on every rollout. Placed last (after envVars and + # extraEnvVars) so this override can't be silently shadowed by a + # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env + # semantics — same pattern the migrations Job uses. + - name: DISABLE_SCHEMA_UPDATE + value: "true" + {{- end }} envFrom: {{- range .Values.environmentSecrets }} - secretRef: @@ -158,15 +169,10 @@ spec: - name: http containerPort: {{ .Values.service.port }} protocol: TCP - {{- if .Values.separateHealthApp }} - - name: health - containerPort: {{ .Values.separateHealthPort | default 8081 }} - protocol: TCP - {{- end }} livenessProbe: httpGet: path: {{ .Values.livenessProbe.path | quote }} - port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} + port: "http" initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} @@ -175,7 +181,7 @@ spec: readinessProbe: httpGet: path: {{ .Values.readinessProbe.path | quote }} - port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} + port: "http" initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} @@ -184,7 +190,7 @@ spec: startupProbe: httpGet: path: {{ .Values.startupProbe.path | quote }} - port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }} + port: "http" initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} periodSeconds: {{ .Values.startupProbe.periodSeconds }} timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index b1cbafaf408..df6d1345644 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -257,16 +257,16 @@ tests: value: 0 - equal: path: spec.template.spec.containers[0].livenessProbe.periodSeconds - value: 10 + value: 15 - equal: path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds - value: 1 + value: 5 - equal: path: spec.template.spec.containers[0].livenessProbe.successThreshold value: 1 - equal: path: spec.template.spec.containers[0].livenessProbe.failureThreshold - value: 3 + value: 5 - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path value: /health/readiness @@ -278,7 +278,7 @@ tests: value: 10 - equal: path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds - value: 1 + value: 5 - equal: path: spec.template.spec.containers[0].readinessProbe.successThreshold value: 1 @@ -296,7 +296,7 @@ tests: value: 10 - equal: path: spec.template.spec.containers[0].startupProbe.timeoutSeconds - value: 1 + value: 5 - equal: path: spec.template.spec.containers[0].startupProbe.successThreshold value: 1 diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index 690ca69e730..ba4059e0840 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -88,26 +88,20 @@ service: # optionally specify loadBalancerClass # loadBalancerClass: tailscale -# Separate health app configuration -# When enabled, health checks will use a separate port and the application -# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables -separateHealthApp: false -separateHealthPort: 8081 - -# Probe tuning for proxy container +# Probes for LiteLLM gateway container livenessProbe: path: /health/liveliness initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 1 + periodSeconds: 15 + timeoutSeconds: 5 successThreshold: 1 - failureThreshold: 3 + failureThreshold: 5 readinessProbe: path: /health/readiness initialDelaySeconds: 0 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 @@ -115,7 +109,7 @@ startupProbe: path: /health/readiness initialDelaySeconds: 0 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 5 successThreshold: 1 failureThreshold: 30 @@ -329,6 +323,17 @@ migrationJob: helm: enabled: false +# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env). +# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over +# any `envFrom:` source. If you currently source LITELLM_LOG from an +# environmentSecret or environmentConfigMap, set `logLevel: ""` here to +# disable injection — otherwise this value silently overrides your secret / +# configmap entry. +# +# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips +# this injection entirely when envVars already defines LITELLM_LOG. +logLevel: INFO + # Additional environment variables to be added to the deployment as a map of key-value pairs envVars: {} diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 671f374ca27..c84003a065f 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root -RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \ +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \ npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \ GLOBAL="$(npm root -g)" && \ find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \ @@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ EXPOSE 4000/tcp -COPY docker/supervisord.conf /etc/supervisord.conf - ENTRYPOINT ["docker/prod_entrypoint.sh"] CMD ["--port", "4000"] diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 3fa73f42437..4de4a55981d 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -103,13 +103,12 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \ + apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \ done COPY --from=builder /app /app COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets -COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf ENV PATH="/app/.venv/bin:${PATH}" \ PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh index 28d1bdcc294..bd78bf6687b 100644 --- a/docker/prod_entrypoint.sh +++ b/docker/prod_entrypoint.sh @@ -1,14 +1,8 @@ #!/bin/sh -if [ "$SEPARATE_HEALTH_APP" = "1" ]; then - export LITELLM_ARGS="$@" - export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}" - exec supervisord -c /etc/supervisord.conf -fi - if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False" exec ddtrace-run litellm "$@" else exec litellm "$@" -fi \ No newline at end of file +fi diff --git a/docker/supervisord.conf b/docker/supervisord.conf deleted file mode 100644 index ba9d99d18a5..00000000000 --- a/docker/supervisord.conf +++ /dev/null @@ -1,46 +0,0 @@ -[supervisord] -nodaemon=true -loglevel=info -logfile=/tmp/supervisord.log -pidfile=/tmp/supervisord.pid - -[group:litellm] -programs=main,health - -[program:main] -command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi' -autostart=true -autorestart=true -startretries=3 -priority=1 -exitcodes=0 -stopasgroup=true -killasgroup=true -stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s -stdout_logfile=/dev/stdout -stderr_logfile=/dev/stderr -stdout_logfile_maxbytes = 0 -stderr_logfile_maxbytes = 0 -environment=PYTHONUNBUFFERED=true - -[program:health] -command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0' -autostart=true -autorestart=true -startretries=3 -priority=2 -exitcodes=0 -stopasgroup=true -killasgroup=true -stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s -stdout_logfile=/dev/stdout -stderr_logfile=/dev/stderr -stdout_logfile_maxbytes = 0 -stderr_logfile_maxbytes = 0 -environment=PYTHONUNBUFFERED=true - -[eventlistener:process_monitor] -command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]" -events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL -autostart=true -autorestart=true \ No newline at end of file diff --git a/docs/images/local-testing/hosted-vllm-custom-tool-local-test.png b/docs/images/local-testing/hosted-vllm-custom-tool-local-test.png new file mode 100644 index 00000000000..9fb6665d373 Binary files /dev/null and b/docs/images/local-testing/hosted-vllm-custom-tool-local-test.png differ diff --git a/litellm-js/spend-logs/package-lock.json b/litellm-js/spend-logs/package-lock.json index e9fc4b00f6a..e33079766c9 100644 --- a/litellm-js/spend-logs/package-lock.json +++ b/litellm-js/spend-logs/package-lock.json @@ -535,9 +535,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.13.0", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", - "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==", + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/litellm/__init__.py b/litellm/__init__.py index 5305edc9be6..cf05fc4c980 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -388,6 +388,7 @@ anthropic_beta_headers_url: str = os.getenv( suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None +s3_audit_callback_params: Optional[Dict] = None datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None datadog_params: Optional[Union[DatadogInitParams, Dict]] = None aws_sqs_callback_params: Optional[Dict] = None @@ -414,6 +415,9 @@ custom_prometheus_metadata_labels: List[str] = [] custom_prometheus_tags: List[str] = [] prometheus_metrics_config: Optional[List] = None prometheus_emit_stream_label: bool = False +prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000 +prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0 +prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0 disable_add_prefix_to_prompt: bool = ( False # used by anthropic, to disable adding prefix to prompt ) @@ -586,6 +590,7 @@ anyscale_models: Set = set() cerebras_models: Set = set() galadriel_models: Set = set() nvidia_nim_models: Set = set() +nvidia_riva_models: Set = set() sambanova_models: Set = set() sambanova_embedding_models: Set = set() novita_models: Set = set() @@ -812,6 +817,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): galadriel_models.add(key) elif value.get("litellm_provider") == "nvidia_nim": nvidia_nim_models.add(key) + elif value.get("litellm_provider") == "nvidia_riva": + nvidia_riva_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": @@ -971,6 +978,7 @@ model_list = list( | cerebras_models | galadriel_models | nvidia_nim_models + | nvidia_riva_models | sambanova_models | azure_text_models | novita_models @@ -1067,6 +1075,7 @@ models_by_provider: dict = { "cerebras": cerebras_models, "galadriel": galadriel_models, "nvidia_nim": nvidia_nim_models, + "nvidia_riva": nvidia_riva_models, "sambanova": sambanova_models | sambanova_embedding_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, @@ -1618,6 +1627,9 @@ if TYPE_CHECKING: from .llms.deepgram.audio_transcription.transformation import ( DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig, ) + from .llms.nvidia_riva.audio_transcription.transformation import ( + NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig, + ) from .llms.topaz.image_variations.transformation import ( TopazImageVariationConfig as TopazImageVariationConfig, ) diff --git a/litellm/constants.py b/litellm/constants.py index 6918e40cad1..072c2c358f7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset( | (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""}) ) +# MCP OAuth2 Token Exchange (OBO) Defaults +MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int( + os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500") +) + LITELLM_UI_ALLOW_HEADERS = [ "x-litellm-semantic-filter", "x-litellm-semantic-filter-tools", @@ -1457,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job" EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job" SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) +SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int( + os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) +) +SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float( + os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5) +) SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100)) SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0)) SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int( diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index e703a3956b9..0dc56b6a3bc 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -366,6 +366,8 @@ class MCPClient: headers["Authorization"] = f"Bearer {self._mcp_auth_value}" elif self.auth_type == MCPAuth.token: headers["Authorization"] = f"token {self._mcp_auth_value}" + elif self.auth_type == MCPAuth.oauth2_token_exchange: + headers["Authorization"] = f"Bearer {self._mcp_auth_value}" elif isinstance(self._mcp_auth_value, dict): headers.update(self._mcp_auth_value) # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request diff --git a/litellm/integrations/azure_sentinel/azure_sentinel.py b/litellm/integrations/azure_sentinel/azure_sentinel.py index dd508e6c6c2..0cfd49cda37 100644 --- a/litellm/integrations/azure_sentinel/azure_sentinel.py +++ b/litellm/integrations/azure_sentinel/azure_sentinel.py @@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class import asyncio import os +import time import traceback -from typing import List, Optional +from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload class AzureSentinelLogger(CustomBatchLogger): @@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger): tenant_id: Optional[str] = None, client_id: Optional[str] = None, client_secret: Optional[str] = None, + audit_stream_name: Optional[str] = None, **kwargs, ): """ @@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger): If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var. client_secret (str, optional): Azure Client Secret for OAuth2 authentication. If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var. + audit_stream_name (str, optional): Stream name from DCR for audit logs. + If not provided, audit logs use the standard stream name. """ self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) - self.dcr_immutable_id = dcr_immutable_id or os.getenv( + resolved_dcr_immutable_id = dcr_immutable_id or os.getenv( "AZURE_SENTINEL_DCR_IMMUTABLE_ID" ) - self.stream_name = stream_name or os.getenv( - "AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM" + resolved_stream_name = ( + stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM" ) - self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT") - self.tenant_id = ( + resolved_audit_stream_name = audit_stream_name or resolved_stream_name + resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT") + resolved_tenant_id = ( tenant_id or os.getenv("AZURE_SENTINEL_TENANT_ID") or os.getenv("AZURE_TENANT_ID") ) - self.client_id = ( + resolved_client_id = ( client_id or os.getenv("AZURE_SENTINEL_CLIENT_ID") or os.getenv("AZURE_CLIENT_ID") ) - self.client_secret = ( + resolved_client_secret = ( client_secret or os.getenv("AZURE_SENTINEL_CLIENT_SECRET") or os.getenv("AZURE_CLIENT_SECRET") ) - if not self.dcr_immutable_id: + if not resolved_dcr_immutable_id: raise ValueError( "AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter." ) - if not self.endpoint: + if not resolved_endpoint: raise ValueError( "AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter." ) - if not self.tenant_id: + if not resolved_tenant_id: raise ValueError( "AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter." ) - if not self.client_id: + if not resolved_client_id: raise ValueError( "AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter." ) - if not self.client_secret: + if not resolved_client_secret: raise ValueError( "AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter." ) + self.dcr_immutable_id = resolved_dcr_immutable_id + self.stream_name = resolved_stream_name + self.audit_stream_name = resolved_audit_stream_name + self.endpoint = resolved_endpoint + self.tenant_id = resolved_tenant_id + self.client_id = resolved_client_id + self.client_secret = resolved_client_secret + # Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01 - self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01" + self.api_endpoint = self._build_api_endpoint( + endpoint=resolved_endpoint, + dcr_immutable_id=resolved_dcr_immutable_id, + stream_name=resolved_stream_name, + ) + self.audit_api_endpoint = self._build_api_endpoint( + endpoint=resolved_endpoint, + dcr_immutable_id=resolved_dcr_immutable_id, + stream_name=resolved_audit_stream_name, + ) # OAuth2 scope for Azure Monitor self.oauth_scope = "https://monitor.azure.com/.default" @@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger): super().__init__(**kwargs, flush_lock=self.flush_lock) asyncio.create_task(self.periodic_flush()) self.log_queue: List[StandardLoggingPayload] = [] + self.audit_log_queue: List[StandardAuditLogPayload] = [] + + @staticmethod + def _build_api_endpoint( + endpoint: str, dcr_immutable_id: str, stream_name: str + ) -> str: + return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01" async def _get_oauth_token(self) -> str: """ @@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger): Returns: Bearer token string """ - # Check if we have a valid cached token - import time - if ( self.oauth_token and self.oauth_token_expires_at @@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger): if not self.oauth_token: raise Exception("OAuth2 token response did not contain access_token") - # Cache token expiry time - import time - self.oauth_token_expires_at = time.time() + expires_in return self.oauth_token @@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger): ) pass + async def async_log_audit_log_event( + self, audit_log: StandardAuditLogPayload + ) -> None: + """ + Async log LiteLLM audit log events to Azure Sentinel. + + Audit logs are queued separately from standard LLM logs so mixed callback + usage never sends schema-mismatched records in the same ingestion batch. + """ + try: + verbose_logger.debug( + "Azure Sentinel: Logging audit event id=%s action=%s table=%s", + audit_log.get("id"), + audit_log.get("action"), + audit_log.get("table_name"), + ) + + self.audit_log_queue.append(audit_log) + + if len(self.audit_log_queue) >= self.batch_size: + await self.async_send_audit_batch() + + except Exception as e: + verbose_logger.exception( + f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}" + ) + pass + async def async_send_batch(self): """ Sends the batch of logs to Azure Monitor Logs Ingestion API @@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger): Raises: Raises a NON Blocking verbose_logger.exception if an error occurs """ + await self._async_send_batch_to_api( + log_queue=self.log_queue, + api_endpoint=self.api_endpoint, + log_type="logs", + ) + + async def async_send_audit_batch(self): + """ + Sends the batch of audit logs to Azure Monitor Logs Ingestion API + """ + await self._async_send_batch_to_api( + log_queue=self.audit_log_queue, + api_endpoint=self.audit_api_endpoint, + log_type="audit logs", + ) + + async def _async_send_batch_to_api( + self, + log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]], + api_endpoint: str, + log_type: str, + ) -> None: try: - if not self.log_queue: + if not log_queue: return verbose_logger.debug( - "Azure Sentinel - about to flush %s events", len(self.log_queue) + "Azure Sentinel - about to flush %s %s", len(log_queue), log_type ) - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - # Get OAuth2 token bearer_token = await self._get_oauth_token() # Convert log queue to JSON array format expected by Logs Ingestion API # Each log entry should be a JSON object in the array - body = safe_dumps(self.log_queue) + body = safe_dumps(log_queue) # Set headers for Logs Ingestion API headers = { @@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger): # Send the request response = await self.async_httpx_client.post( - url=self.api_endpoint, data=body.encode("utf-8"), headers=headers + url=api_endpoint, data=body.encode("utf-8"), headers=headers ) if response.status_code not in [200, 204]: @@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger): f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}" ) finally: - self.log_queue.clear() + log_queue.clear() + + async def flush_queue(self): + if self.flush_lock is None: + return + + async with self.flush_lock: + if self.log_queue: + await self.async_send_batch() + if self.audit_log_queue: + await self.async_send_audit_batch() + self.last_flush_time = time.time() diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 7d3856ea63a..f9b1c666439 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -25,6 +25,9 @@ from typing import ( import litellm from litellm._logging import print_verbose, verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import ( + BoundedPrometheusSeriesTracker, +) from litellm.integrations.prometheus_helpers import ( PrometheusLabelFactoryContext, _get_cached_end_user_id_for_cost_tracking, @@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger): if _custom_buckets is not None else LATENCY_BUCKETS ) + self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker() # Create metric factory functions self._counter_factory = self._create_metric_factory(Counter) @@ -984,6 +988,40 @@ class PrometheusLogger(CustomLogger): return filtered_labels + def _track_end_user_metric_series( + self, + metric: Any, + metric_name: DEFINED_PROMETHEUS_METRICS, + labels: Dict[str, Optional[str]], + ) -> None: + """ + Cap the cardinality of metrics that include the ``end_user`` label. + + Called *after* ``metric.labels(...).inc()/observe()`` so the emission is + recorded in prometheus-client's child map before any eviction runs. + Series that get evicted before the next scrape lose updates accrued + since the last scrape — this is inherent to any cardinality cap. + """ + labelnames = self.get_labels_for_metric(metric_name) + if UserAPIKeyLabelNames.END_USER.value not in labelnames: + return + if labels.get(UserAPIKeyLabelNames.END_USER.value) is None: + return + + max_series = litellm.prometheus_end_user_metrics_max_series_per_metric + ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds + if max_series is None and ttl_seconds is None: + return + + self._bounded_prometheus_series_tracker.track_series( + metric=metric, + metric_name=metric_name, + label_values=tuple(labels.get(label) for label in labelnames), + max_series=max_series, + ttl_seconds=ttl_seconds, + cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds, + ) + def _inc_labeled_counter( self, counter: Any, @@ -998,6 +1036,7 @@ class PrometheusLogger(CustomLogger): label_context=label_context, ) counter.labels(**_labels).inc(amount) + self._track_end_user_metric_series(counter, metric_name, _labels) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): # Define prometheus client @@ -1047,21 +1086,9 @@ class PrometheusLogger(CustomLogger): output_tokens = standard_logging_payload["completion_tokens"] tokens_used = standard_logging_payload["total_tokens"] response_cost = standard_logging_payload["response_cost"] - _requester_metadata: Optional[dict] = standard_logging_payload["metadata"].get( - "requester_metadata" + combined_metadata = _get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload=standard_logging_payload ) - user_api_key_auth_metadata: Optional[dict] = standard_logging_payload[ - "metadata" - ].get("user_api_key_auth_metadata") - spend_logs_metadata: Optional[dict] = standard_logging_payload["metadata"].get( - "spend_logs_metadata" - ) - - combined_metadata: Dict[str, Any] = { - **(_requester_metadata if _requester_metadata else {}), - **(user_api_key_auth_metadata if user_api_key_auth_metadata else {}), - **(spend_logs_metadata if spend_logs_metadata else {}), - } if standard_logging_payload is not None and isinstance( standard_logging_payload, dict ): @@ -1416,26 +1443,46 @@ class PrometheusLogger(CustomLogger): ) remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}" - remaining_requests = ( - metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize + remaining_requests = metadata.get(remaining_requests_variable_name) + if remaining_requests is None: + remaining_requests = sys.maxsize + remaining_tokens = metadata.get(remaining_tokens_variable_name) + if remaining_tokens is None: + remaining_tokens = sys.maxsize + + enum_values = UserAPIKeyLabelValues( + hashed_api_key=user_api_key, + api_key_alias=user_api_key_alias, + model=model_group, + model_id=model_id, + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload=kwargs.get("standard_logging_object") + ) + ), ) - remaining_tokens = ( - metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize + label_context = PrometheusLabelFactoryContext(enum_values) + requests_labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + "litellm_remaining_api_key_requests_for_model" + ), + enum_values=enum_values, + label_context=label_context, + ) + self.litellm_remaining_api_key_requests_for_model.labels(**requests_labels).set( + remaining_requests ) - self.litellm_remaining_api_key_requests_for_model.labels( - _sanitize_prometheus_label_value(user_api_key), - _sanitize_prometheus_label_value(user_api_key_alias), - _sanitize_prometheus_label_value(model_group), - _sanitize_prometheus_label_value(model_id), - ).set(remaining_requests) - - self.litellm_remaining_api_key_tokens_for_model.labels( - _sanitize_prometheus_label_value(user_api_key), - _sanitize_prometheus_label_value(user_api_key_alias), - _sanitize_prometheus_label_value(model_group), - _sanitize_prometheus_label_value(model_id), - ).set(remaining_tokens) + tokens_labels = prometheus_label_factory( + supported_enum_labels=self.get_labels_for_metric( + "litellm_remaining_api_key_tokens_for_model" + ), + enum_values=enum_values, + label_context=label_context, + ) + self.litellm_remaining_api_key_tokens_for_model.labels(**tokens_labels).set( + remaining_tokens + ) def _set_latency_metrics( self, @@ -1471,6 +1518,11 @@ class PrometheusLogger(CustomLogger): self.litellm_llm_api_time_to_first_token_metric.labels( **_ttft_labels ).observe(time_to_first_token_seconds) + self._track_end_user_metric_series( + self.litellm_llm_api_time_to_first_token_metric, + "litellm_llm_api_time_to_first_token_metric", + _ttft_labels, + ) else: verbose_logger.debug( "Time to first token metric not emitted, stream option in model_parameters is not True" @@ -1491,6 +1543,11 @@ class PrometheusLogger(CustomLogger): self.litellm_llm_api_latency_metric.labels(**_labels).observe( api_call_total_time_seconds ) + self._track_end_user_metric_series( + self.litellm_llm_api_latency_metric, + "litellm_llm_api_latency_metric", + _labels, + ) # total request latency total_time_seconds = self._safe_duration_seconds( @@ -1508,6 +1565,11 @@ class PrometheusLogger(CustomLogger): self.litellm_request_total_latency_metric.labels(**_labels).observe( total_time_seconds ) + self._track_end_user_metric_series( + self.litellm_request_total_latency_metric, + "litellm_request_total_latency_metric", + _labels, + ) # request queue time (time from arrival to processing start) _litellm_params = kwargs.get("litellm_params", {}) or {} @@ -1525,6 +1587,11 @@ class PrometheusLogger(CustomLogger): self.litellm_request_queue_time_metric.labels(**_labels).observe( queue_time_seconds ) + self._track_end_user_metric_series( + self.litellm_request_queue_time_metric, + "litellm_request_queue_time_seconds", + _labels, + ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): verbose_logger.debug( @@ -1561,18 +1628,27 @@ class PrometheusLogger(CustomLogger): ) try: - self.litellm_llm_api_failed_requests_metric.labels( - _sanitize_prometheus_label_value(end_user_id), - _sanitize_prometheus_label_value(user_api_key), - _sanitize_prometheus_label_value(user_api_key_alias), - _sanitize_prometheus_label_value(model), - _sanitize_prometheus_label_value(user_api_team), - _sanitize_prometheus_label_value(user_api_team_alias), - _sanitize_prometheus_label_value(user_id), - _sanitize_prometheus_label_value( - standard_logging_payload.get("model_id", "") + enum_values = UserAPIKeyLabelValues( + end_user=end_user_id, + hashed_api_key=user_api_key, + api_key_alias=user_api_key_alias, + model=model, + team=user_api_team, + team_alias=user_api_team_alias, + user=user_id, + model_id=standard_logging_payload.get("model_id", ""), + custom_metadata_labels=get_custom_labels_from_metadata( + metadata=_get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload=standard_logging_payload + ) ), - ).inc() + ) + PrometheusLogger._inc_labeled_counter( + self, + self.litellm_llm_api_failed_requests_metric, + "litellm_llm_api_failed_requests_metric", + enum_values, + ) self.set_llm_deployment_failure_metrics(kwargs) await self._set_org_budget_metrics_after_api_request( org_id=user_api_key_org_id, @@ -3622,6 +3698,36 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]: return result +def _get_combined_custom_metadata_from_standard_logging_payload( + standard_logging_payload: Optional[dict], +) -> Dict[str, Any]: + """ + Combine the metadata sources that can supply custom Prometheus labels. + """ + if not isinstance(standard_logging_payload, dict): + return {} + + standard_logging_metadata = standard_logging_payload.get("metadata") or {} + if not isinstance(standard_logging_metadata, dict): + return {} + + requester_metadata = standard_logging_metadata.get("requester_metadata") + user_api_key_auth_metadata = standard_logging_metadata.get( + "user_api_key_auth_metadata" + ) + spend_logs_metadata = standard_logging_metadata.get("spend_logs_metadata") + + return { + **(requester_metadata if isinstance(requester_metadata, dict) else {}), + **( + user_api_key_auth_metadata + if isinstance(user_api_key_auth_metadata, dict) + else {} + ), + **(spend_logs_metadata if isinstance(spend_logs_metadata, dict) else {}), + } + + def _tag_matches_wildcard_configured_pattern( tags: Sequence[str], configured_tag: str ) -> bool: diff --git a/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py b/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py new file mode 100644 index 00000000000..d834ae20142 --- /dev/null +++ b/litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import time +from collections import OrderedDict +from threading import RLock +from typing import Any, Dict, Optional + + +class BoundedPrometheusSeriesTracker: + """ + Tracks Prometheus child series and removes stale/excess labelsets. + + The tracker is label-agnostic: callers decide which series should be tracked + and pass the full label tuple used by the Prometheus metric. + """ + + def __init__(self) -> None: + self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {} + self._last_ttl_cleanup: Dict[str, float] = {} + self.lock = RLock() + + def track_series( + self, + metric: Any, + metric_name: str, + label_values: tuple[Optional[str], ...], + max_series: Optional[int], + ttl_seconds: Optional[float], + cleanup_interval_seconds: Optional[float], + ) -> None: + if max_series is None and ttl_seconds is None: + return + + now = time.monotonic() + + with self.lock: + series = self._series.setdefault(metric_name, OrderedDict()) + series[label_values] = now + series.move_to_end(label_values) + + if ttl_seconds is not None and self._should_run_ttl_cleanup( + metric_name=metric_name, + now=now, + cleanup_interval_seconds=cleanup_interval_seconds, + ): + expired_label_values = [ + tracked_label_values + for tracked_label_values, last_seen in series.items() + if now - last_seen > ttl_seconds + ] + for tracked_label_values in expired_label_values: + self._remove_metric_series(metric, series, tracked_label_values) + + # max_series <= 0 is treated as "unlimited" so a misconfigured zero + # value cannot silently drop every emission for this metric. + if max_series is not None and max_series > 0: + while len(series) > max_series: + tracked_label_values = next(iter(series)) + if not self._remove_metric_child(metric, tracked_label_values): + break + del series[tracked_label_values] + + def _should_run_ttl_cleanup( + self, + metric_name: str, + now: float, + cleanup_interval_seconds: Optional[float], + ) -> bool: + if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0: + self._last_ttl_cleanup[metric_name] = now + return True + + last_cleanup = self._last_ttl_cleanup.get(metric_name) + if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds: + self._last_ttl_cleanup[metric_name] = now + return True + return False + + def _remove_metric_series( + self, + metric: Any, + series: OrderedDict[tuple[Optional[str], ...], float], + label_values: tuple[Optional[str], ...], + ) -> None: + if self._remove_metric_child(metric, label_values): + series.pop(label_values, None) + + @staticmethod + def _remove_metric_child( + metric: Any, label_values: tuple[Optional[str], ...] + ) -> bool: + """ + Remove the Prometheus child for ``label_values`` and report whether the + tracker should commit the matching state change. + + Returns ``True`` when the child is no longer present in Prometheus + (either it was just removed or it was already gone), and ``False`` when + ``metric.remove()`` raised an unexpected error and the child likely + still exists. + """ + try: + metric.remove(*label_values) + return True + except KeyError: + return True + except (AttributeError, ValueError): + return False diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index 08ce7ed8947..332e84dd07d 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS from litellm.integrations.s3 import get_s3_object_key from litellm.litellm_core_utils.safe_json_dumps import safe_dumps +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, @@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_strip_base64_files: bool = False, s3_use_key_prefix: bool = False, s3_use_virtual_hosted_style: bool = False, + s3_callback_params_override: Optional[dict] = None, **kwargs, ): try: - verbose_logger.debug( - f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}" - ) + _masker = SensitiveDataMasker() + if s3_callback_params_override is not None: + verbose_logger.debug( + f"in init s3 logger (audit override) - " + f"{_masker.mask_dict(dict(s3_callback_params_override))}" + ) + else: + verbose_logger.debug( + f"in init s3 logger - s3_callback_params " + f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}" + ) # Initialize S3 params first to get the correct s3_verify value self._init_s3_params( + params_source=s3_callback_params_override, s3_bucket_name=s3_bucket_name, s3_region_name=s3_region_name, s3_api_version=s3_api_version, @@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_strip_base64_files: bool = False, s3_use_key_prefix: bool = False, s3_use_virtual_hosted_style: bool = False, + params_source: Optional[dict] = None, ): """ - Initialize the s3 params for this logging callback + Initialize the s3 params for this logging callback. Reads from + `params_source` if given (e.g. `s3_audit_callback_params` for the + audit-log instance), otherwise falls back to `litellm.s3_callback_params`. + Resolves `os.environ/X` markers into a local dict; never mutates the source. """ - litellm.s3_callback_params = litellm.s3_callback_params or {} - # read in .env variables - example os.environ/AWS_BUCKET_NAME - for key, value in litellm.s3_callback_params.items(): - if isinstance(value, str) and value.startswith("os.environ/"): - litellm.s3_callback_params[key] = litellm.get_secret(value) + if params_source is None: + params_source = litellm.s3_callback_params or {} + params: dict = { + key: ( + litellm.get_secret(value) + if isinstance(value, str) and value.startswith("os.environ/") + else value + ) + for key, value in params_source.items() + } - self.s3_bucket_name = ( - litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name - ) - self.s3_region_name = ( - litellm.s3_callback_params.get("s3_region_name") or s3_region_name - ) - self.s3_api_version = ( - litellm.s3_callback_params.get("s3_api_version") or s3_api_version - ) + self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name + self.s3_region_name = params.get("s3_region_name") or s3_region_name + self.s3_api_version = params.get("s3_api_version") or s3_api_version self.s3_use_ssl = ( - litellm.s3_callback_params.get("s3_use_ssl", True) - if litellm.s3_callback_params.get("s3_use_ssl") is not None + params.get("s3_use_ssl", True) + if params.get("s3_use_ssl") is not None else s3_use_ssl ) self.s3_verify = ( - litellm.s3_callback_params.get("s3_verify") - if litellm.s3_callback_params.get("s3_verify") is not None + params.get("s3_verify") + if params.get("s3_verify") is not None else s3_verify ) - self.s3_endpoint_url = ( - litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url - ) + self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url self.s3_aws_access_key_id = ( - litellm.s3_callback_params.get("s3_aws_access_key_id") - or s3_aws_access_key_id + params.get("s3_aws_access_key_id") or s3_aws_access_key_id ) self.s3_aws_secret_access_key = ( - litellm.s3_callback_params.get("s3_aws_secret_access_key") - or s3_aws_secret_access_key + params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key ) self.s3_aws_session_token = ( - litellm.s3_callback_params.get("s3_aws_session_token") - or s3_aws_session_token + params.get("s3_aws_session_token") or s3_aws_session_token ) self.s3_aws_session_name = ( - litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name + params.get("s3_aws_session_name") or s3_aws_session_name ) self.s3_aws_profile_name = ( - litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name + params.get("s3_aws_profile_name") or s3_aws_profile_name ) - self.s3_aws_role_name = ( - litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name - ) + self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name self.s3_aws_web_identity_token = ( - litellm.s3_callback_params.get("s3_aws_web_identity_token") - or s3_aws_web_identity_token + params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token ) self.s3_aws_sts_endpoint = ( - litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint + params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint ) - self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config - self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path - # done reading litellm.s3_callback_params + self.s3_config = params.get("s3_config") or s3_config + self.s3_path = params.get("s3_path") or s3_path self.s3_use_team_prefix = ( - bool(litellm.s3_callback_params.get("s3_use_team_prefix", False)) - or s3_use_team_prefix + bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix ) self.s3_use_key_prefix = ( - bool(litellm.s3_callback_params.get("s3_use_key_prefix", False)) - or s3_use_key_prefix + bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix ) self.s3_strip_base64_files = ( - bool(litellm.s3_callback_params.get("s3_strip_base64_files", False)) - or s3_strip_base64_files + bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files ) self.s3_use_virtual_hosted_style = ( - bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False)) + bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style ) diff --git a/litellm/litellm_core_utils/custom_logger_registry.py b/litellm/litellm_core_utils/custom_logger_registry.py index f873bfeece5..fd402b90d88 100644 --- a/litellm/litellm_core_utils/custom_logger_registry.py +++ b/litellm/litellm_core_utils/custom_logger_registry.py @@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal from litellm.integrations.agentops import AgentOps from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.integrations.argilla import ArgillaLogger +from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger from litellm.integrations.bitbucket import BitBucketPromptManager from litellm.integrations.braintrust_logging import BraintrustLogger @@ -73,6 +74,7 @@ class CustomLoggerRegistry: "opik": OpikLogger, "argilla": ArgillaLogger, "opentelemetry": OpenTelemetry, + "azure_sentinel": AzureSentinelLogger, "azure_storage": AzureBlobStorageLogger, "humanloop": HumanloopLogger, # OTEL compatible loggers diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index c0ca6835eee..ba6d438f16c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -621,6 +621,18 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or "https://integrate.api.nvidia.com/v1" ) # type: ignore dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY") + elif custom_llm_provider == "nvidia_riva": + # NVIDIA Riva is gRPC-based; api_base must be a host:port like + # `grpc.nvcf.nvidia.com:443` or `localhost:50051`. There is no + # public-default endpoint, so we do not fill one in here. + api_base = api_base or get_secret_str("NVIDIA_RIVA_API_BASE") # type: ignore + # Fall back to NVIDIA_NIM_API_KEY because users running both NVCF + # services typically reuse the same nvapi-* key. + dynamic_api_key = ( + api_key + or get_secret_str("NVIDIA_RIVA_API_KEY") + or get_secret_str("NVIDIA_NIM_API_KEY") + ) elif custom_llm_provider == "cerebras": api_base = ( api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1" diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b234e6c8f77..52269d705d0 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -436,12 +436,21 @@ def update_messages_with_model_file_ids( """ Updates messages with model file ids. + For managed files (unified file IDs), uses model_file_id_mapping if it + resolves the id, otherwise decodes the base64-encoded unified file ID + and extracts the llm_output_file_id directly. Mirrors the Responses-API + sibling `update_responses_input_with_model_file_ids`. + model_file_id_mapping: Dict[str, Dict[str, str]] = { "litellm_proxy/file_id": { "model_id": "provider_file_id" } } """ + from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, + convert_b64_uid_to_unified_uid, + ) for message in messages: if message.get("role") == "user": @@ -450,7 +459,13 @@ def update_messages_with_model_file_ids( if isinstance(content, str): continue for c in content: - if c["type"] == "file": + if not isinstance(c, dict): + # Content list items aren't always dicts. e.g. + # text_completion forwards a token-ids list/list-of- + # lists through this path. Skip non-dict items + # instead of indexing into them. + continue + if c.get("type") == "file": file_object = cast(ChatCompletionFileObject, c) file_object_file_field = file_object.get("file") if not isinstance(file_object_file_field, dict): @@ -468,9 +483,23 @@ def update_messages_with_model_file_ids( if file_id: provider_file_id = ( model_file_id_mapping.get(file_id, {}).get(model_id) - or file_id + if model_file_id_mapping + else None + ) + if ( + not provider_file_id + and _is_base64_encoded_unified_file_id(file_id) + ): + unified_file_id = convert_b64_uid_to_unified_uid( + file_id + ) + if "llm_output_file_id," in unified_file_id: + provider_file_id = unified_file_id.split( + "llm_output_file_id," + )[1].split(";")[0] + file_object_file_field["file_id"] = ( + provider_file_id or file_id ) - file_object_file_field["file_id"] = provider_file_id if format: file_object_file_field["format"] = format return messages diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ba840bc3d89..abe9e016e26 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2124,27 +2124,62 @@ def anthropic_process_openai_file_message( ) +_EMPTY_TEXT_PLACEHOLDER = ( + "[System: Empty message content sanitised to satisfy protocol]" +) + + def _sanitize_empty_text_content( message: AllMessageValues, ) -> AllMessageValues: """ Case C: Sanitize empty text content - Replace empty or whitespace-only text content with a placeholder message. + - Handles both string content and list-of-blocks content (rewriting only + the empty text blocks in place; non-text blocks like images are left + untouched). Returns: The message with sanitized content if needed, otherwise the original message """ - if message.get("role") in ["user", "assistant"]: - content = message.get("content") - if isinstance(content, str): - if not content or not content.strip(): - message = cast(AllMessageValues, dict(message)) # Make a copy - message["content"] = ( - "[System: Empty message content sanitised to satisfy protocol]" - ) - verbose_logger.debug( - f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message" - ) + if message.get("role") not in ["user", "assistant"]: + return message + + content = message.get("content") + + if isinstance(content, str): + if not content or not content.strip(): + message = cast(AllMessageValues, dict(message)) # Make a copy + message["content"] = _EMPTY_TEXT_PLACEHOLDER + verbose_logger.debug( + f"_sanitize_empty_text_content: Replaced empty text content in {message.get('role')} message" + ) + return message + + if isinstance(content, list): + # Walk the blocks and rewrite any empty text blocks. We rewrite (rather + # than drop) so callers don't end up with an entirely empty content + # list, which Anthropic also rejects. + new_blocks: List[Any] = [] + rewrote_any = False + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text") + if not isinstance(text, str) or not text or not text.strip(): + new_block = dict(block) + new_block["text"] = _EMPTY_TEXT_PLACEHOLDER + new_blocks.append(new_block) + rewrote_any = True + continue + new_blocks.append(block) + + if rewrote_any: + message = cast(AllMessageValues, dict(message)) # Make a copy + message["content"] = new_blocks # type: ignore + verbose_logger.debug( + f"_sanitize_empty_text_content: Replaced empty text block(s) in {message.get('role')} message" + ) + return message @@ -2427,6 +2462,18 @@ def anthropic_messages_pt( # noqa: PLR0915 # Sanitize messages for tool calling issues when modify_params=True messages = sanitize_messages_for_tool_calling(messages) + # Anthropic rejects empty text content blocks with: + # "messages: text content blocks must be non-empty" + # OpenAI/other providers silently tolerate `{"role": "user", "content": ""}`, + # so callers (and upstream agent frameworks like pydantic-ai) routinely + # send empty user/assistant turns. We always rewrite these to a placeholder + # for Anthropic-shaped requests, independent of `litellm.modify_params`, + # because there is no way to "pass through" an empty text block — the + # request will always 400 otherwise. The richer tool-call sanitization + # (Cases A/B/D in `sanitize_messages_for_tool_calling`) remains gated on + # `modify_params` because it actually mutates conversation structure. + messages = [_sanitize_empty_text_content(m) for m in messages] + # add role=tool support to allow function call result/error submission user_message_types = {"user", "tool", "function"} # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them. diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 4493a58f78b..c4528ff74e3 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -31,6 +31,8 @@ DefaultLoggedRealTimeEventTypes = [ "session.created", "response.create", "response.done", + "conversation.item.added", # GA + "conversation.item.done", # GA ] @@ -44,6 +46,7 @@ class RealTimeStreaming: model: str = "", user_api_key_dict: Optional[Any] = None, request_data: Optional[Dict] = None, + backend_uses_beta_protocol: Optional[bool] = None, ): self.websocket = websocket self.backend_ws = backend_ws @@ -54,6 +57,14 @@ class RealTimeStreaming: self.session_tools: List[Dict] = [] self.tool_calls: List[Dict] = [] + # Detect whether the client is explicitly opting into the beta protocol. + self._client_wants_beta = self._detect_beta_header(websocket) + self._backend_uses_beta_protocol = ( + self._client_wants_beta + if backend_uses_beta_protocol is None + else backend_uses_beta_protocol + ) + _logged_real_time_event_types = litellm.logged_real_time_event_types if _logged_real_time_event_types is None: @@ -76,6 +87,27 @@ class RealTimeStreaming: # response.create can be rewritten to include the failure context. self._pending_guardrail_message: Optional[str] = None + _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) + _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = { + "pcm16": {"type": "audio/pcm", "rate": 24000}, + "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, + "g711_alaw": {"type": "audio/G711-alaw", "rate": 8000}, + } + # GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1) + _GA_TO_BETA_EVENT_TYPES: Dict[str, str] = { + "conversation.item.added": "conversation.item.created", + "response.output_text.delta": "response.text.delta", + "response.output_audio.delta": "response.audio.delta", + "response.output_audio_transcript.delta": "response.audio_transcript.delta", + "response.output_text.done": "response.text.done", + "response.output_audio.done": "response.audio.done", + "response.output_audio_transcript.done": "response.audio_transcript.done", + } + _GA_TO_BETA_CONTENT_TYPES: Dict[str, str] = { + "output_text": "text", + "output_audio": "audio", + } + def _should_store_message( self, message_obj: Union[dict, OpenAIRealtimeEvents], @@ -92,24 +124,27 @@ class RealTimeStreaming: if isinstance(message, bytes): message = message.decode("utf-8") if isinstance(message, dict): - message_obj = message + # TypedDict union members do not narrow to plain dict for mypy. + message_obj: Dict[str, Any] = cast(Dict[str, Any], message) else: - message_obj = json.loads(message) + message_obj = cast(Dict[str, Any], json.loads(cast(str, message))) self._collect_tool_calls_from_response_done(cast(dict, message_obj)) try: - if ( - not isinstance(message, dict) - or message_obj.get("type") == "session.created" - or message_obj.get("type") == "session.updated" - ): - message_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore - elif not isinstance(message, dict): - message_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore + event_type = message_obj.get("type", "") + if event_type in self._SESSION_EVENT_TYPES: + typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj) # type: ignore + else: + # Use the base object as a safe catch-all for all other event types + # (both beta and GA), so unknown/new event names never raise here. + typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj) # type: ignore except Exception as e: verbose_logger.debug(f"Error parsing message for logging: {e}") - raise e - if self._should_store_message(message_obj): - self.messages.append(message_obj) + # Don't re-raise — a parse failure must not drop or delay the message + if self._should_store_message(message_obj): + self.messages.append(message_obj) # type: ignore[arg-type] + return + if self._should_store_message(typed_obj): + self.messages.append(typed_obj) def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None: """Extract user text content from client WebSocket events for spend logging.""" @@ -147,6 +182,8 @@ class RealTimeStreaming: tools = session.get("tools") if tools and isinstance(tools, list): self.session_tools = tools + # GA: session.type is required; log it for traceability but no action needed + verbose_logger.debug(f"Realtime session.type: {session.get('type')}") except (json.JSONDecodeError, AttributeError, TypeError): pass @@ -228,6 +265,23 @@ class RealTimeStreaming: else: await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] + def _make_disable_auto_response_message(self) -> str: + """Return a session.update that disables VAD auto-response.""" + if self._backend_uses_beta_protocol: + session: Dict[str, Any] = { + "turn_detection": {"create_response": False}, + } + else: + session = { + "type": "realtime", + "audio": { + "input": { + "turn_detection": {"create_response": False}, + } + }, + } + return json.dumps({"type": "session.update", "session": session}) + def _has_realtime_guardrails(self) -> bool: """Return True if any callback is registered for realtime guardrail event types.""" from litellm.integrations.custom_guardrail import CustomGuardrail @@ -435,14 +489,7 @@ class RealTimeStreaming: ): self.store_message(event_str) await self.websocket.send_text(event_str) - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": {"turn_detection": {"create_response": False}}, - } - ) - ) + await self._send_to_backend(self._make_disable_auto_response_message()) continue ## GUARDRAIL: run on transcription events in provider_config path too if ( @@ -484,14 +531,7 @@ class RealTimeStreaming: ): self.store_message(raw_response) await self.websocket.send_text(raw_response) - await self._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": {"turn_detection": {"create_response": False}}, - } - ) - ) + await self._send_to_backend(self._make_disable_auto_response_message()) return True if ( @@ -542,7 +582,20 @@ class RealTimeStreaming: continue ## LOGGING self.store_message(raw_response) - await self.websocket.send_text(raw_response) + + # If the client opted into beta protocol, translate GA event + # names/shapes back to the beta equivalents before forwarding. + if self._client_wants_beta: + try: + event_dict = json.loads(raw_response) + translated = self._translate_event_to_beta(event_dict) + if translated is None: + continue # drop GA-only events (e.g. conversation.item.done) + await self.websocket.send_text(json.dumps(translated)) + except Exception: + await self.websocket.send_text(raw_response) + else: + await self.websocket.send_text(raw_response) except websockets.exceptions.ConnectionClosed as e: # type: ignore verbose_logger.exception( @@ -553,6 +606,183 @@ class RealTimeStreaming: finally: await self.log_messages() + @staticmethod + def _detect_beta_header(websocket: Any) -> bool: + """Return True if the client sent 'OpenAI-Beta: realtime=v1'. + + Checks the raw ASGI scope headers so it works for both FastAPI WebSocket + objects and any test doubles that expose a .scope dict. + """ + try: + headers = websocket.scope.get("headers", []) + for name, value in headers: + if isinstance(name, bytes): + name = name.decode("latin-1") + if isinstance(value, bytes): + value = value.decode("latin-1") + if name.lower() == "openai-beta" and "realtime=v1" in value.lower(): + return True + except Exception: + pass + return False + + @staticmethod + def _remap_beta_session_to_ga(session: dict) -> dict: + """ + Convert a beta-style session.update payload to the GA nested schema. + + Beta → GA field mappings + ───────────────────────────────────────────────────────────────────── + session.type (inject "realtime" if absent) + session.modalities → session.output_modalities + session.voice → session.audio.output.voice + session.input_audio_format → session.audio.input.format (with type/rate) + session.output_audio_format → session.audio.output.format (with type/rate) + session.turn_detection → session.audio.input.turn_detection + session.input_audio_transcription → session.audio.input.transcription + ───────────────────────────────────────────────────────────────────── + Fields not in the mapping (instructions, tools, etc.) are passed through. + GA clients that already use the nested shape are unaffected. + """ + # Work on a shallow copy so we don't mutate the caller's dict + session = dict(session) + + # 1. Ensure session.type is present + if "type" not in session: + session["type"] = "realtime" + + # 2. Rename modalities → output_modalities and normalise combinations. + # Beta allowed ["audio", "text"] together; GA only supports ["audio"] or + # ["text"] as single-element lists. When both are present we prefer + # ["audio"] because audio mode already delivers transcripts via events. + if "modalities" in session: + mods = session.pop("modalities") + if "output_modalities" not in session: + mods_set = {m.lower() for m in (mods or [])} + if "audio" in mods_set: + session["output_modalities"] = ["audio"] + elif "text" in mods_set: + session["output_modalities"] = ["text"] + + # 3-7. Lift flat audio fields into the nested audio object + audio: Dict[str, Any] = {} + inp: Dict[str, Any] = {} + out: Dict[str, Any] = {} + + # voice → audio.output.voice + if "voice" in session: + out["voice"] = session.pop("voice") + + # input_audio_format → audio.input.format + if "input_audio_format" in session: + raw = session.pop("input_audio_format") + inp["format"] = ( + RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw) + if isinstance(raw, str) + else raw + ) + + # output_audio_format → audio.output.format + if "output_audio_format" in session: + raw = session.pop("output_audio_format") + out["format"] = ( + RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw, raw) + if isinstance(raw, str) + else raw + ) + + # turn_detection → audio.input.turn_detection + if "turn_detection" in session: + inp["turn_detection"] = session.pop("turn_detection") + + # input_audio_transcription → audio.input.transcription + if "input_audio_transcription" in session: + inp["transcription"] = session.pop("input_audio_transcription") + + if inp: + audio["input"] = inp + if out: + audio["output"] = out + + if audio: + # Merge with any existing GA-style `audio` block the client already set, + # letting the remapped values take precedence within each sub-key. + existing = session.get("audio") or {} + for sub_key, sub_val in audio.items(): + if ( + sub_key in existing + and isinstance(existing[sub_key], dict) + and isinstance(sub_val, dict) + ): + existing[sub_key] = {**existing[sub_key], **sub_val} + else: + existing[sub_key] = sub_val + session["audio"] = existing + + return session + + @staticmethod + def _translate_event_to_beta(event: dict) -> Optional[dict]: + """Translate a single GA event dict to its beta equivalent. + + Returns None if the event should be dropped entirely (e.g. the GA-only + conversation.item.done has no beta counterpart). + Returns the (possibly mutated copy of the) event otherwise. + """ + event_type = event.get("type", "") + + # conversation.item.done has no beta equivalent — the client already + # received conversation.item.created (translated from .added). + if event_type == "conversation.item.done": + return None + + # Shallow-copy so we don't mutate the stored message + translated = dict(event) + + # Rename the type field + if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES: + translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type] + + # Fix content block types inside items (response.done output list, + # conversation.item.created item content, etc.) + if "item" in translated and isinstance(translated["item"], dict): + translated["item"] = RealTimeStreaming._translate_item_content_types( + dict(translated["item"]) + ) + if "response" in translated and isinstance(translated["response"], dict): + resp = dict(translated["response"]) + if "output" in resp and isinstance(resp["output"], list): + resp["output"] = [ + ( + RealTimeStreaming._translate_item_content_types(dict(o)) + if isinstance(o, dict) + else o + ) + for o in resp["output"] + ] + translated["response"] = resp + + return translated + + @staticmethod + def _translate_item_content_types(item: dict) -> dict: + """Replace GA content type names with beta names inside a single item.""" + if "content" not in item or not isinstance(item["content"], list): + return item + new_content = [] + for block in item["content"]: + if ( + isinstance(block, dict) + and block.get("type") in RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES + ): + block = dict(block) + block["type"] = RealTimeStreaming._GA_TO_BETA_CONTENT_TYPES[ + block["type"] + ] + new_content.append(block) + item["content"] = new_content + return item + async def client_ack_messages(self): try: while True: @@ -594,6 +824,19 @@ class RealTimeStreaming: self._pending_guardrail_message = None continue + # GA compatibility: remap beta-style session fields only when + # the upstream is in GA mode. Beta upstreams expect the flat + # session shape unchanged. + if ( + msg_type == "session.update" + and not self._backend_uses_beta_protocol + ): + session = msg_obj.get("session", {}) + if isinstance(session, dict): + session = self._remap_beta_session_to_ga(session) + msg_obj["session"] = session + message = json.dumps(msg_obj) + except (json.JSONDecodeError, AttributeError): pass @@ -627,3 +870,8 @@ class RealTimeStreaming: await forward_task except asyncio.CancelledError: pass + + +def client_sent_openai_beta_realtime_header(websocket: Any) -> bool: + """True when the client WebSocket includes ``OpenAI-Beta: realtime=v1``.""" + return RealTimeStreaming._detect_beta_header(websocket) diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index f8e61d0166a..2fb29b32a61 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -65,7 +65,7 @@ from litellm.types.utils import ( from ...base import BaseLLM from ..common_utils import AnthropicError, process_anthropic_headers -from .transformation import AnthropicConfig +from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig if TYPE_CHECKING: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -83,6 +83,7 @@ async def make_call( timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, speed: Optional[str] = None, + tool_name_reverse_map: Optional[Dict[str, str]] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_aclient @@ -117,6 +118,7 @@ async def make_call( sync_stream=False, json_mode=json_mode, speed=speed, + tool_name_reverse_map=tool_name_reverse_map, ) # LOGGING @@ -141,6 +143,7 @@ def make_sync_call( timeout: Optional[Union[float, httpx.Timeout]], json_mode: bool, speed: Optional[str] = None, + tool_name_reverse_map: Optional[Dict[str, str]] = None, ) -> Tuple[Any, httpx.Headers]: if client is None: client = litellm.module_level_client # re-use a module level client @@ -183,6 +186,7 @@ def make_sync_call( sync_stream=True, json_mode=json_mode, speed=speed, + tool_name_reverse_map=tool_name_reverse_map, ) # LOGGING @@ -237,6 +241,11 @@ class AnthropicChatCompletion(BaseLLM): timeout=timeout, json_mode=json_mode, speed=optional_params.get("speed") if optional_params else None, + tool_name_reverse_map=( + litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY) + if isinstance(litellm_params, dict) + else None + ), ) streamwrapper = CustomStreamWrapper( completion_stream=completion_stream, @@ -462,6 +471,11 @@ class AnthropicChatCompletion(BaseLLM): timeout=timeout, json_mode=json_mode, speed=optional_params.get("speed") if optional_params else None, + tool_name_reverse_map=( + litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY) + if isinstance(litellm_params, dict) + else None + ), ) return CustomStreamWrapper( completion_stream=completion_stream, @@ -526,6 +540,7 @@ class ModelResponseIterator: sync_stream: bool, json_mode: Optional[bool] = False, speed: Optional[str] = None, + tool_name_reverse_map: Optional[Dict[str, str]] = None, ): self.streaming_response = streaming_response self.response_iterator = self.streaming_response @@ -533,6 +548,13 @@ class ModelResponseIterator: self.tool_index = -1 self.json_mode = json_mode self.speed = speed + # rewritten-name -> caller's original. Built per-request from the + # forward map in AnthropicConfig._build_request_tool_name_maps; only + # contains entries we actually rewrote, so a tool legitimately named + # `foo_bar` is *not* reverse-mapped just because some other tool was + # rewritten to `foo_bar` in a different request. Empty/None is the + # common case (no '/' or other invalid chars in any tool name). + self.tool_name_reverse_map: Dict[str, str] = tool_name_reverse_map or {} # Generate response ID once per stream to match OpenAI-compatible behavior self.response_id = _generate_id() @@ -557,6 +579,10 @@ class ModelResponseIterator: # Accumulate compaction blocks for multi-turn reconstruction self.compaction_blocks: List[Dict[str, Any]] = [] + # Accumulate streamed thinking text so final usage can split reasoning + # tokens from regular output tokens. + self.reasoning_content_chunks: List[str] = [] + # Track server tool use inputs and results for code_interpreter_results self._server_tool_inputs: Dict[str, Any] = {} self.tool_results: List[Dict[str, Any]] = [] @@ -587,9 +613,14 @@ class ModelResponseIterator: return False def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage: + reasoning_content = ( + "".join(self.reasoning_content_chunks) + if self.reasoning_content_chunks + else None + ) return AnthropicConfig().calculate_usage( usage_object=cast(dict, anthropic_usage_chunk), - reasoning_content=None, + reasoning_content=reasoning_content, speed=self.speed, ) @@ -636,10 +667,13 @@ class ModelResponseIterator: "thinking" in content_block["delta"] or "signature" in content_block["delta"] ): + thinking_content = content_block["delta"].get("thinking") + if isinstance(thinking_content, str) and thinking_content: + self.reasoning_content_chunks.append(thinking_content) thinking_blocks = [ ChatCompletionThinkingBlock( type="thinking", - thinking=content_block["delta"].get("thinking") or "", + thinking=thinking_content or "", signature=str(content_block["delta"].get("signature") or ""), ) ] @@ -792,6 +826,16 @@ class ModelResponseIterator: or content_block_start["content_block"]["type"] == "server_tool_use" ): self.tool_index += 1 + # Reverse-map the (sanitized) tool name back to the + # caller's original. No-op when the map is empty. + _stream_tool_name = content_block_start["content_block"]["name"] + if ( + self.tool_name_reverse_map + and _stream_tool_name in self.tool_name_reverse_map + ): + _stream_tool_name = self.tool_name_reverse_map[ + _stream_tool_name + ] # Use empty string for arguments in content_block_start - actual arguments # come in subsequent content_block_delta chunks and get accumulated. # Using str(input) here would prepend '{}' causing invalid JSON accumulation. @@ -799,7 +843,7 @@ class ModelResponseIterator: id=content_block_start["content_block"]["id"], type="function", function=ChatCompletionToolCallFunctionChunk( - name=content_block_start["content_block"]["name"], + name=_stream_tool_name, arguments="", ), index=self.tool_index, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 345558a69f7..2f11a3fccb5 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -105,6 +105,115 @@ else: LoggingClass = Any +# Anthropic requires tool names to match ^[a-zA-Z0-9_-]{1,128}$. Any other +# character (commonly '/' or '.' from OpenAPI-derived MCP tools, e.g. +# "actions/download-job-logs-for-workflow-run") must be replaced before +# the request is sent. +# +# A naive "replace [^a-zA-Z0-9_-] with _" is unsafe because it's lossy: +# `foo/bar` and `foo_bar` both collapse to `foo_bar`. Two tools with the +# same sanitized name would either 400 at Anthropic (duplicate) or, worse, +# cause the response side to mis-translate `foo_bar` (a name the caller +# really did register) back to `foo/bar`. +# +# Instead we build a *per-request* forward map (original -> sanitized) +# whose codomain is unique within the request: when two originals collapse +# to the same candidate, or when a sanitized name collides with an already- +# valid name elsewhere in the request, we append numeric suffixes +# (`_2`, `_3`, ...) until the result is free. +# +# The reverse map (sanitized -> original) only contains entries where the +# original was actually rewritten. So a tool whose name is already valid +# round-trips identically and is *never* mistakenly re-mapped on the +# response side. +_ANTHROPIC_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]") +_ANTHROPIC_TOOL_NAME_MAX_LEN = 128 +# Single, internal-only key on ``litellm_params`` used to thread the per- +# request reverse map (sanitized -> original) from request build to response +# parsing. ``litellm_params`` is never serialized to a provider; ``optional_ +# params`` IS (it becomes the JSON body via ``data = {**optional_params}``). +# Keep these two channels strictly separate -- never stash internal +# coordination state in ``optional_params``. +ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY = "_anthropic_tool_name_map" + + +def _basic_sanitize_anthropic_tool_name(name: str) -> str: + """Lossy: replace [^a-zA-Z0-9_-] with '_' and truncate to 128. + + Used as a candidate generator for the per-request forward map. + Callers should NOT use this directly for translation -- always go + through the forward map so collisions are resolved. + """ + if not isinstance(name, str) or not name: + return name + return _ANTHROPIC_TOOL_NAME_INVALID_CHARS.sub("_", name)[ + :_ANTHROPIC_TOOL_NAME_MAX_LEN + ] + + +def _build_anthropic_tool_name_maps( + original_names: List[str], +) -> Tuple[Dict[str, str], Dict[str, str]]: + """Build (forward, reverse) tool-name maps for a single request. + + forward[original] = sanitized -- only present when name was rewritten + reverse[sanitized] = original -- inverse of `forward` + + Properties: + - All sanitized names satisfy ^[a-zA-Z0-9_-]{1,128}$. + - Sanitized names are unique within the request (no two originals + collide on the wire). + - A name that's already valid AND doesn't collide with another tool's + sanitized form passes through untouched and is absent from the maps. + That's the key correctness property: response-side translation only + runs on entries we actually rewrote, so a tool legitimately named + `foo_bar` is never incorrectly retyped to `foo/bar` just because + some *other* request had that pair. + - Order-dependent: when two originals would clash, the *second* one + seen gets the disambiguating suffix. Callers should preserve the + caller's tool order (we do). + """ + forward: Dict[str, str] = {} + used: set = set() + + # First pass: reserve slots for names that are already valid so they + # always have priority regardless of input order. + for original in original_names: + if not isinstance(original, str) or not original: + continue + candidate = _basic_sanitize_anthropic_tool_name(original) + if candidate == original: + used.add(candidate) + + # Second pass: sanitize/disambiguate names that need rewriting. + for original in original_names: + if not isinstance(original, str) or not original: + continue + candidate = _basic_sanitize_anthropic_tool_name(original) + if candidate == original: + continue + # Skip duplicates of the same original name. Without this guard the + # second pass would assign a fresh suffix and overwrite the forward + # map entry, causing every reference to map to the suffixed name and + # leaving the original sanitized slot orphaned in `used` with no + # reverse mapping. + if original in forward: + continue + # Disambiguate against names already chosen this request. + unique = candidate + n = 1 + while unique in used: + n += 1 + suffix = f"_{n}" + # Keep within the 128-char cap. + head = candidate[: _ANTHROPIC_TOOL_NAME_MAX_LEN - len(suffix)] + unique = f"{head}{suffix}" + forward[original] = unique + used.add(unique) + reverse = {v: k for k, v in forward.items()} + return forward, reverse + + REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT: Dict[str, str] = { "low": "low", "minimal": "low", @@ -486,7 +595,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): } def _map_tool_choice( - self, tool_choice: Optional[str], parallel_tool_use: Optional[bool] + self, + tool_choice: Optional[str], + parallel_tool_use: Optional[bool], ) -> Optional[AnthropicMessagesToolChoice]: _tool_choice: Optional[AnthropicMessagesToolChoice] = None if tool_choice == "auto": @@ -527,7 +638,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return _tool_choice def _map_tool_helper( # noqa: PLR0915 - self, tool: ChatCompletionToolParam + self, + tool: ChatCompletionToolParam, ) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]: returned_tool: Optional[AllAnthropicToolsValues] = None mcp_server: Optional[AnthropicMcpServerTool] = None @@ -783,7 +895,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return initial_tool def _map_tools( - self, tools: List + self, + tools: List, ) -> Tuple[List[AllAnthropicToolsValues], List[AnthropicMcpServerTool]]: anthropic_tools = [] mcp_servers = [] @@ -799,6 +912,174 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_servers.append(mcp_server_tool) return anthropic_tools, mcp_servers + @staticmethod + def _rewrite_tool_names_in_messages( + messages: List[AllMessageValues], + name_forward_map: Dict[str, str], + ) -> List[AllMessageValues]: + """Return a copy of `messages` with tool_call/function_call names + rewritten using the per-request forward map. + + Only mutates messages whose tool_call/function_call name is *in* the + forward map. Names absent from the map (already valid, no collision) + round-trip untouched. We only deep-copy the entries we actually + change to keep this O(turns-with-rewritten-tools), not O(history). + """ + if not name_forward_map: + return messages + new_messages: List[AllMessageValues] = [] + for msg in messages: + if not isinstance(msg, dict): + new_messages.append(msg) + continue + tool_calls = msg.get("tool_calls") + function_call = msg.get("function_call") + if not tool_calls and not function_call: + new_messages.append(msg) + continue + new_msg = dict(msg) + if isinstance(tool_calls, list): + new_calls = [] + for tc in tool_calls: + if not isinstance(tc, dict): + new_calls.append(tc) + continue + fn = tc.get("function") + fn_name = fn.get("name") if isinstance(fn, dict) else None + if ( + isinstance(fn, dict) + and isinstance(fn_name, str) + and fn_name in name_forward_map + ): + new_fn = dict(fn) + new_fn["name"] = name_forward_map[fn_name] + new_tc = dict(tc) + new_tc["function"] = new_fn + new_calls.append(new_tc) + else: + new_calls.append(tc) + new_msg["tool_calls"] = new_calls + fc_name = ( + function_call.get("name") if isinstance(function_call, dict) else None + ) + if ( + isinstance(function_call, dict) + and isinstance(fc_name, str) + and fc_name in name_forward_map + ): + new_fc = dict(function_call) + new_fc["name"] = name_forward_map[fc_name] + new_msg["function_call"] = new_fc + new_messages.append(cast(AllMessageValues, new_msg)) + return new_messages + + @staticmethod + def _build_request_tool_name_maps( + tools: List, + ) -> Tuple[Dict[str, str], Dict[str, str]]: + """Build the (forward, reverse) tool-name maps for an OpenAI tools list. + + Operates on **OpenAI-format** tool dicts (pre-``_map_tools``). The + production sanitization path uses ``_sanitize_tool_names_in_request`` + instead, which operates on **Anthropic-format** tools (post- + ``_map_tools``, where ``type == "custom"``). This helper exists for + callers that need to compute the maps from the raw OpenAI shape -- + e.g. test setup or future pre-mapping consumers. + + See _build_anthropic_tool_name_maps for the collision rules. Pulls + the original name out of either ``{"function": {"name": ...}}`` + (legacy OpenAI shape) or ``{"name": ...}`` (rare top-level shape). + """ + original_names: List[str] = [] + for tool in tools or []: + if not isinstance(tool, dict): + continue + original = ( + tool.get("function", {}).get("name") + if isinstance(tool.get("function"), dict) + else None + ) + if original is None: + original = tool.get("name") + if isinstance(original, str) and original: + original_names.append(original) + return _build_anthropic_tool_name_maps(original_names) + + @staticmethod + def _sanitize_tool_names_in_request( + optional_params: Dict[str, Any], + ) -> Tuple[Dict[str, str], Dict[str, str]]: + """Sanitize ``optional_params['tools']`` and ``optional_params['tool_choice']`` + in place so every name matches Anthropic's ``^[a-zA-Z0-9_-]{1,128}$``. + + Returns ``(forward, reverse)`` for use by message-history rewriting + and response translation. ``forward[original] = sanitized`` is only + populated for names that were actually rewritten -- i.e. either + contained an invalid character or collided with another tool's + sanitized form. Names already valid AND unique pass through and are + absent from both maps. + + Only ``type == "custom"`` tools (the OpenAI function-tool shape) are + considered. Hosted tools (``web_search``, ``bash``, ``code_execution``, + ``computer_*``, ``mcp``, ...) own reserved names defined by Anthropic + and must not be touched. + """ + tools = optional_params.get("tools") + if not isinstance(tools, list) or not tools: + return {}, {} + + # 1. Collect originals from the Anthropic-shaped custom-tool entries. + # Order matters: the first occurrence wins the canonical slot; + # later collisions get numeric suffixes (see + # ``_build_anthropic_tool_name_maps``). + original_names: List[str] = [] + for t in tools: + if not isinstance(t, dict): + continue + if t.get("type") != "custom": + continue + name = t.get("name") + if isinstance(name, str) and name: + original_names.append(name) + + if not original_names: + return {}, {} + + forward, reverse = _build_anthropic_tool_name_maps(original_names) + if not forward: + # Every name was already valid -- nothing to do. + return forward, reverse + + # 2. Apply forward map. Build a new list with copy-on-change entries + # so a caller reusing the same tool list/dicts across requests + # doesn't see its inputs permanently rewritten (which would also + # drop the original key from `forward` on the next request). + new_tools: List[Any] = [] + for t in tools: + if ( + isinstance(t, dict) + and t.get("type") == "custom" + and isinstance(t.get("name"), str) + and t["name"] in forward + ): + new_tools.append({**t, "name": forward[t["name"]]}) + else: + new_tools.append(t) + optional_params["tools"] = new_tools + + # 3. Same for ``tool_choice`` when it targets a named tool. Copy + # rather than mutate for the same reason as above. + tool_choice = optional_params.get("tool_choice") + if isinstance(tool_choice, dict) and tool_choice.get("type") == "tool": + tc_name = tool_choice.get("name") + if isinstance(tc_name, str) and tc_name in forward: + optional_params["tool_choice"] = { + **tool_choice, + "name": forward[tc_name], + } + + return forward, reverse + def _detect_tool_search_tools(self, tools: Optional[List]) -> bool: """Check if tool search tools are present in the tools list.""" if not tools: @@ -1125,6 +1406,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): non_default_params=non_default_params ) + # NB: ``map_openai_params`` deliberately does NOT sanitize tool names + # here. Names are the *original* OpenAI names at this stage, and must + # remain so until ``transform_request`` -- which is the single + # chokepoint where Anthropic, Bedrock-Anthropic, and Vertex-Anthropic + # all pass through. Doing it there guarantees: + # 1. one source of truth for the per-request forward/reverse maps, + # 2. the maps land on ``litellm_params`` (internal), never on + # ``optional_params`` (which is serialized into the request body + # via ``data = {**optional_params}`` and would 400 with + # ``Extra inputs are not permitted``). + for param, value in non_default_params.items(): if param == "max_tokens": optional_params["max_tokens"] = ( @@ -1135,7 +1427,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): value if isinstance(value, int) else max(1, int(round(value))) ) elif param == "tools": - # check if optional params already has tools anthropic_tools, mcp_servers = self._map_tools(value) optional_params = self._add_tools_to_optional_params( optional_params=optional_params, tools=anthropic_tools @@ -1565,6 +1856,34 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers=headers, optional_params=optional_params ) + # === Tool-name sanitization (single chokepoint) === + # Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We + # sanitize *here* -- not in map_openai_params -- because: + # + # - This function is the single boundary shared by AnthropicConfig, + # AmazonAnthropicConfig (Bedrock invoke), VertexAIAnthropicConfig, + # and AzureAnthropicConfig (all call ``super().transform_request`` + # or ``AnthropicConfig.transform_request(self, ...)``). Sanitizing + # once here covers every Anthropic-shaped request. + # - The forward/reverse maps are coordination state; they belong on + # ``litellm_params`` (internal-only), never on ``optional_params`` + # (which becomes the JSON body via ``{**optional_params}``). + # - It keeps ``map_openai_params`` a pure param translator with no + # side-channel state. + # + # The reverse map only contains entries for names that were actually + # rewritten -- so a tool legitimately named ``foo_bar`` is never + # incorrectly retyped to ``foo/bar`` on the response side. + # See _build_anthropic_tool_name_maps for the collision-handling + # rules and rationale. + _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( + optional_params=optional_params, + ) + if _name_forward_map: + messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map) + if _name_reverse_map and isinstance(litellm_params, dict): + litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map + # Separate system prompt from rest of message anthropic_system_message_list = self.translate_system_message(messages=messages) # Handling anthropic API Prompt Caching @@ -1837,8 +2156,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): speed: Optional[str] = None, ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this - prompt_tokens = usage_object.get("input_tokens", 0) or 0 - completion_tokens = usage_object.get("output_tokens", 0) or 0 + raw_prompt_tokens = usage_object.get("input_tokens", 0) or 0 + prompt_tokens: int = ( + int(raw_prompt_tokens) if isinstance(raw_prompt_tokens, (int, float)) else 0 + ) + raw_completion_tokens = usage_object.get("output_tokens", 0) or 0 + completion_tokens: int = ( + int(raw_completion_tokens) + if isinstance(raw_completion_tokens, (int, float)) + else 0 + ) _usage = usage_object cache_creation_input_tokens: int = 0 cache_read_input_tokens: int = 0 @@ -1907,11 +2234,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text_tokens=raw_input_tokens, ) # Always populate completion_token_details, not just when there's reasoning_content - reasoning_tokens = ( + estimated_reasoning_tokens = ( token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 ) + reasoning_tokens = min(estimated_reasoning_tokens, completion_tokens) completion_token_details = CompletionTokensDetailsWrapper( reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0, text_tokens=( @@ -2041,6 +2369,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_mode: Optional[bool] = None, prefix_prompt: Optional[str] = None, speed: Optional[str] = None, + tool_name_reverse_map: Optional[Dict[str, str]] = None, ): _hidden_params: Dict = {} _hidden_params["additional_headers"] = process_anthropic_headers( @@ -2065,6 +2394,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): compaction_blocks, ) = self.extract_response_content(completion_response=completion_response) + # Reverse-map rewritten tool names back to caller's originals so a + # downstream OpenAI-style dispatcher can match on the registered name. + # See _build_anthropic_tool_name_maps for why this is keyed on the + # per-request reverse map (so a tool legitimately named `foo_bar` is + # never incorrectly retyped to `foo/bar`). No-op when the map is + # empty (the common case). + if tool_name_reverse_map and tool_calls: + for tc in tool_calls: + fn = tc.get("function") if isinstance(tc, dict) else None + if fn is None: + continue + _name = fn.get("name") + if isinstance(_name, str) and _name in tool_name_reverse_map: + fn["name"] = tool_name_reverse_map[_name] + if ( prefix_prompt is not None and not text_content.startswith(prefix_prompt) @@ -2191,6 +2535,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): prefix_prompt = self.get_prefix_prompt(messages=messages) speed = optional_params.get("speed") + tool_name_reverse_map: Optional[Dict[str, str]] = None + if isinstance(litellm_params, dict): + _candidate = litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY) + if isinstance(_candidate, dict): + tool_name_reverse_map = _candidate model_response = self.transform_parsed_response( completion_response=completion_response, @@ -2199,6 +2548,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): json_mode=json_mode, prefix_prompt=prefix_prompt, speed=speed, + tool_name_reverse_map=tool_name_reverse_map, ) return model_response diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 1f3428f2ca5..1f3357fd788 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -89,9 +89,10 @@ class AzureOpenAIRealtime(AzureChatCompletion): if api_base is None: raise ValueError("api_base is required for Azure OpenAI calls") - if api_version is None and ( + backend_uses_beta_protocol = ( realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1") - ): + ) + if api_version is None and backend_uses_beta_protocol: raise ValueError("api_version is required for Azure OpenAI calls") url = self._construct_url( @@ -114,6 +115,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): logging_obj, user_api_key_dict=user_api_key_dict, request_data={"litellm_metadata": litellm_metadata or {}}, + backend_uses_beta_protocol=backend_uses_beta_protocol, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index dae60948a58..0885775932c 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -7,6 +7,8 @@ from datetime import datetime from typing import ( TYPE_CHECKING, Any, + Callable, + ClassVar, Dict, List, Literal, @@ -63,8 +65,16 @@ class AwsAuthError(Exception): class BaseAWSLLM: + # Process-wide IAM credential cache (shared across instances — Bedrock passthrough is per-request). + # Storage is in-process memory only: default ``DualCache()`` has no Redis backend unless attached + # elsewhere. Entry TTL: static access-key + secret + region use ``_get_default_ttl_for_boto3_credentials`` + # (~59 minutes); ambient env (``_auth_with_env_vars`` returns ``ttl=None``) uses ``InMemoryCache``'s + # ``default_ttl`` (600 seconds / 10 minutes). AssumeRole, web identity, profiles, and explicit + # session-token tuples are not cached — see ``get_credentials`` and ``_get_or_set_cached_credentials``. + _shared_iam_cache: ClassVar[DualCache] = DualCache() + def __init__(self) -> None: - self.iam_cache = DualCache() + self.iam_cache = BaseAWSLLM._shared_iam_cache super().__init__() self.aws_authentication_params = [ "aws_access_key_id", @@ -103,6 +113,79 @@ class BaseAWSLLM: credential_str = json.dumps(credential_args, sort_keys=True) return hashlib.sha256(credential_str.encode()).hexdigest() + def _get_or_set_cached_credentials( + self, + credential_args: Dict[str, Optional[str]], + credential_fetcher: Callable[[], Tuple[Any, Optional[int]]], + ) -> Any: + """ + Read-through IAM cache on the process-wide ``DualCache``. + + Only the in-memory layer is used by default (no Redis on ``_shared_iam_cache`` unless + configured globally). TTL on write: static access-key fetches pass + ``_get_default_ttl_for_boto3_credentials()`` (~59 minutes); ambient env passes ``ttl=None``, + which ``InMemoryCache.set_cache`` resolves to ``default_ttl`` (600 seconds / 10 minutes by + default). + + Used only for static access-key credentials and ambient credentials from + ``_auth_with_env_vars`` (including when skipping AssumeRole because the runtime identity + already matches ``aws_role_name``). + + AssumeRole, web identity exchange, profiles, and explicit session-token tuples are not + cached here — shared ``Credentials`` / refresh state must not span logical sessions. + """ + cache_key = self.get_cache_key(credential_args) + _cached = self.iam_cache.get_cache(cache_key) + if _cached: + return _cached + credentials, ttl = credential_fetcher() + self.iam_cache.set_cache(cache_key, credentials, ttl=ttl) + return credentials + + @staticmethod + def _is_auth_with_web_identity_token( + aws_web_identity_token: Optional[str], + aws_role_name: Optional[str], + aws_session_name: Optional[str], + ) -> bool: + return ( + aws_web_identity_token is not None + and aws_role_name is not None + and aws_session_name is not None + ) + + @staticmethod + def _is_auth_with_aws_role(aws_role_name: Optional[str]) -> bool: + return aws_role_name is not None + + @staticmethod + def _is_auth_with_aws_profile(aws_profile_name: Optional[str]) -> bool: + return aws_profile_name is not None + + @staticmethod + def _is_auth_with_aws_session_token_tuple( + aws_access_key_id: Optional[str], + aws_secret_access_key: Optional[str], + aws_session_token: Optional[str], + ) -> bool: + return ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_session_token is not None + ) + + @staticmethod + def _is_auth_with_access_key_and_secret_key( + aws_access_key_id: Optional[str], + aws_secret_access_key: Optional[str], + aws_region_name: Optional[str], + ) -> bool: + return ( + aws_access_key_id is not None + and aws_secret_access_key is not None + and aws_region_name is not None + ) + @tracer.wrap() def get_credentials( self, @@ -184,95 +267,97 @@ class BaseAWSLLM: aws_external_id, ) - # create cache key for non-expiring auth flows args = { k: v for k, v in locals().items() if k.startswith("aws_") or k == "ssl_verify" } - cache_key = self.get_cache_key(args) - _cached_credentials = self.iam_cache.get_cache(cache_key) - if _cached_credentials: - return _cached_credentials - ######################################################### # Handle diff boto3 auth flows # for each helper # Return: # Credentials - boto3.Credentials # cache ttl - Optional[int]. If None, the credentials are not cached. Some auth flows have no expiry time. + # + # iam_cache: static keys and ambient env only (including skip-AssumeRole path). + # Do not cache AssumeRole / web identity / profile / explicit session-token paths here. ######################################################### - if ( - aws_web_identity_token is not None - and aws_role_name is not None - and aws_session_name is not None + if self._is_auth_with_web_identity_token( + aws_web_identity_token, + aws_role_name, + aws_session_name, ): credentials, _cache_ttl = self._auth_with_web_identity_token( - aws_web_identity_token=aws_web_identity_token, - aws_role_name=aws_role_name, - aws_session_name=aws_session_name, + aws_web_identity_token=cast(str, aws_web_identity_token), + aws_role_name=cast(str, aws_role_name), + aws_session_name=cast(str, aws_session_name), aws_region_name=aws_region_name, aws_sts_endpoint=aws_sts_endpoint, aws_external_id=aws_external_id, ) - elif aws_role_name is not None: - # Check if we're already running as the target role and can skip assumption - # This handles IRSA (EKS), ECS task roles, and EC2 instance profiles - if self._is_already_running_as_role(aws_role_name, ssl_verify=ssl_verify): + return credentials + elif self._is_auth_with_aws_role(aws_role_name): + # Same role (IRSA/ECS/EC2): ambient creds via _get_or_set_cached_credentials like the + # default env branch; never pre-read cache (must run _is_already_running_as_role first). + if self._is_already_running_as_role( + cast(str, aws_role_name), ssl_verify=ssl_verify + ): verbose_logger.debug( "Already running as target role %s, using ambient credentials", aws_role_name, ) - credentials, _cache_ttl = self._auth_with_env_vars() - else: - verbose_logger.debug( - "Using role assumption: calling _auth_with_aws_role" + return self._get_or_set_cached_credentials( + args, self._auth_with_env_vars ) - # If aws_session_name is not provided, generate a default one - if aws_session_name is None: - aws_session_name = ( - f"litellm-session-{int(datetime.now().timestamp())}" - ) - credentials, _cache_ttl = self._auth_with_aws_role( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_session_token=aws_session_token, - aws_role_name=aws_role_name, - aws_session_name=aws_session_name, - aws_region_name=aws_region_name, - aws_sts_endpoint=aws_sts_endpoint, - aws_external_id=aws_external_id, - ssl_verify=ssl_verify, - ) - - elif aws_profile_name is not None: ### CHECK SESSION ### - credentials, _cache_ttl = self._auth_with_aws_profile(aws_profile_name) - elif ( - aws_access_key_id is not None - and aws_secret_access_key is not None - and aws_session_token is not None - ): - credentials, _cache_ttl = self._auth_with_aws_session_token( + verbose_logger.debug("Using role assumption: calling _auth_with_aws_role") + # If aws_session_name is not provided, generate a default one + if aws_session_name is None: + aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}" + credentials, _assume_ttl = self._auth_with_aws_role( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, - ) - elif ( - aws_access_key_id is not None - and aws_secret_access_key is not None - and aws_region_name is not None - ): - credentials, _cache_ttl = self._auth_with_access_key_and_secret_key( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + aws_role_name=cast(str, aws_role_name), + aws_session_name=aws_session_name, aws_region_name=aws_region_name, + aws_sts_endpoint=aws_sts_endpoint, + aws_external_id=aws_external_id, + ssl_verify=ssl_verify, + ) + return credentials + + elif self._is_auth_with_aws_profile(aws_profile_name): + credentials, _cache_ttl = self._auth_with_aws_profile( + cast(str, aws_profile_name) + ) + return credentials + elif self._is_auth_with_aws_session_token_tuple( + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + ): + credentials, _cache_ttl = self._auth_with_aws_session_token( + aws_access_key_id=cast(str, aws_access_key_id), + aws_secret_access_key=cast(str, aws_secret_access_key), + aws_session_token=cast(str, aws_session_token), + ) + return credentials + elif self._is_auth_with_access_key_and_secret_key( + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + ): + return self._get_or_set_cached_credentials( + args, + lambda: self._auth_with_access_key_and_secret_key( + aws_access_key_id=cast(str, aws_access_key_id), + aws_secret_access_key=cast(str, aws_secret_access_key), + aws_region_name=cast(str, aws_region_name), + ), ) else: - credentials, _cache_ttl = self._auth_with_env_vars() - - self.iam_cache.set_cache(cache_key, credentials, ttl=_cache_ttl) - return credentials + return self._get_or_set_cached_credentials(args, self._auth_with_env_vars) def _get_aws_region_from_model_arn(self, model: Optional[str]) -> Optional[str]: try: diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index 4c667b0ce39..e4072c24557 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -299,29 +299,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): ) def _get_response_stream_shape(self): - """Get the response stream shape for parsing, reusing existing logic.""" - try: - # Try to reuse the cached shape from the existing decoder - from litellm.llms.bedrock.chat.invoke_handler import ( - get_response_stream_shape, - ) + from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE - return get_response_stream_shape() - except ImportError: - # Fallback: create our own shape - try: - from botocore.loaders import Loader - from botocore.model import ServiceModel - - loader = Loader() - bedrock_service_dict = loader.load_service_model( - "bedrock-runtime", "service-2" - ) - bedrock_service_model = ServiceModel(bedrock_service_dict) - return bedrock_service_model.shape_for("ResponseStream") - except Exception as e: - verbose_logger.warning(f"Could not load response stream shape: {e}") - return None + return BEDROCK_RESPONSE_STREAM_SHAPE def _extract_response_content(self, events: InvokeAgentEventList) -> str: """Extract the final response content from parsed events.""" diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 9dfada7c418..92ca75db95b 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -67,9 +67,13 @@ from litellm.types.utils import ( from litellm.utils import CustomStreamWrapper, get_secret from ..base_aws_llm import BaseAWSLLM -from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name +from ..common_utils import ( + BEDROCK_RESPONSE_STREAM_SHAPE, + BedrockError, + ModelResponseIterator, + get_bedrock_tool_name, +) -_response_stream_shape_cache = None bedrock_tool_name_mappings: InMemoryCache = InMemoryCache( max_size_in_memory=50, default_ttl=600 ) @@ -1391,20 +1395,6 @@ class BedrockLLM(BaseAWSLLM): return None -def get_response_stream_shape(): - global _response_stream_shape_cache - if _response_stream_shape_cache is None: - from botocore.loaders import Loader - from botocore.model import ServiceModel - - loader = Loader() - bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2") - bedrock_service_model = ServiceModel(bedrock_service_dict) - _response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream") - - return _response_stream_shape_cache - - class AWSEventStreamDecoder: def __init__(self, model: str, json_mode: Optional[bool] = False) -> None: from botocore.parsers import EventStreamJSONParser @@ -1838,8 +1828,18 @@ class AWSEventStreamDecoder: yield self._chunk_parser(chunk_data=_data) def _parse_message_from_event(self, event) -> Optional[str]: + if BEDROCK_RESPONSE_STREAM_SHAPE is None: + raise BedrockError( + status_code=500, + message=( + "Bedrock event-stream shape could not be loaded from botocore. " + "Ensure botocore is correctly installed." + ), + ) response_dict = event.to_response_dict() - parsed_response = self.parser.parse(response_dict, get_response_stream_shape()) + parsed_response = self.parser.parse( + response_dict, BEDROCK_RESPONSE_STREAM_SHAPE + ) if response_dict["status_code"] != 200: decoded_body = response_dict["body"].decode() diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 9a97a134cc4..856a525f773 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: import httpx import litellm +from litellm import verbose_logger from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -917,38 +918,57 @@ def get_bedrock_chat_config(model: str): return litellm.AmazonInvokeConfig() +def _load_bedrock_response_stream_shape(): + """ + Load the ResponseStream shape from botocore's bundled bedrock-runtime schema. + + Called once at module import time; the result is stored in + ``BEDROCK_RESPONSE_STREAM_SHAPE`` and reused for the process lifetime. + Returns ``None`` if botocore is unavailable or the service model cannot be + loaded, so the module still imports cleanly. + """ + try: + from botocore.loaders import Loader + from botocore.model import ServiceModel + + loader = Loader() + service_dict = loader.load_service_model("bedrock-runtime", "service-2") + return ServiceModel(service_dict).shape_for("ResponseStream") + except Exception as e: + verbose_logger.warning( + "litellm: could not pre-load bedrock-runtime response stream shape " + "— Bedrock event-stream decoding will be unavailable. Error: %s", + e, + ) + return None + + +# Eagerly resolved once per process — avoids per-instance or per-request disk I/O. +BEDROCK_RESPONSE_STREAM_SHAPE = _load_bedrock_response_stream_shape() + + class BedrockEventStreamDecoderBase: """ Base class for event stream decoding for Bedrock """ - _response_stream_shape_cache = None - def __init__(self): from botocore.parsers import EventStreamJSONParser self.parser = EventStreamJSONParser() - def get_response_stream_shape(self): - if self._response_stream_shape_cache is None: - from botocore.loaders import Loader - from botocore.model import ServiceModel - - loader = Loader() - bedrock_service_dict = loader.load_service_model( - "bedrock-runtime", "service-2" - ) - bedrock_service_model = ServiceModel(bedrock_service_dict) - self._response_stream_shape_cache = bedrock_service_model.shape_for( - "ResponseStream" - ) - - return self._response_stream_shape_cache - def _parse_message_from_event(self, event) -> Optional[str]: + if BEDROCK_RESPONSE_STREAM_SHAPE is None: + raise BedrockError( + status_code=500, + message=( + "Bedrock event-stream shape could not be loaded from botocore. " + "Ensure botocore is correctly installed." + ), + ) response_dict = event.to_response_dict() parsed_response = self.parser.parse( - response_dict, self.get_response_stream_shape() + response_dict, BEDROCK_RESPONSE_STREAM_SHAPE ) if response_dict["status_code"] != 200: diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index 274b0282acc..846af65c0f9 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -10,14 +10,12 @@ from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockEventStreamDecoderBase, BedrockModelInfo if TYPE_CHECKING: + from httpx import URL + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import CostResponseTypes -if TYPE_CHECKING: - from httpx import URL - - class BedrockPassthroughConfig( BaseAWSLLM, BedrockModelInfo, BedrockEventStreamDecoderBase, BasePassthroughConfig ): diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index dd955c23d23..af18c666679 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1,4 +1,5 @@ import asyncio +import concurrent.futures import inspect import os import socket @@ -133,6 +134,11 @@ _DEFAULT_TIMEOUT = httpx.Timeout( timeout=COMPLETION_HTTP_FALLBACK_SECONDS, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS, ) +_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0 +_STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor( + max_workers=50, + thread_name_prefix="litellm-streaming-error-body-read", +) def _prepare_request_data_and_content( @@ -386,17 +392,30 @@ def _safe_get_response_text(response: httpx.Response) -> str: return "" -async def _safe_aread_response(response: httpx.Response) -> bytes: +async def _safe_aread_response( + response: httpx.Response, timeout: Optional[float] = None +) -> bytes: """Safely read async response body, falling back to empty bytes on errors.""" try: + if timeout is not None: + return await asyncio.wait_for(response.aread(), timeout=timeout) return await response.aread() except Exception: return b"" -def _safe_read_response(response: httpx.Response) -> bytes: +def _safe_read_response( + response: httpx.Response, timeout: Optional[float] = None +) -> bytes: """Safely read sync response body, falling back to empty bytes on errors.""" try: + if timeout is not None: + future = _STREAMING_ERROR_BODY_READ_EXECUTOR.submit(response.read) + try: + return future.result(timeout=timeout) + except Exception: + response.close() + return b"" return response.read() except Exception: return b"" @@ -405,8 +424,19 @@ def _safe_read_response(response: httpx.Response) -> bytes: def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None: """Raise a MaskedHTTPStatusError for sync HTTP handlers.""" if stream: - _body = mask_sensitive_info(_safe_read_response(e.response)) - raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + try: + _body = mask_sensitive_info( + _safe_read_response( + e.response, + timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS, + ) + ) + raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + finally: + try: + e.response.close() + except Exception: + pass _text = mask_sensitive_info(_safe_get_response_text(e.response)) raise MaskedHTTPStatusError(e, message=_text, text=_text) from None @@ -414,8 +444,19 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None: async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None: """Raise a MaskedHTTPStatusError for async HTTP handlers.""" if stream: - _body = mask_sensitive_info(await _safe_aread_response(e.response)) - raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + try: + _body = mask_sensitive_info( + await _safe_aread_response( + e.response, + timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS, + ) + ) + raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + finally: + try: + await e.response.aclose() + except Exception: + pass _text = mask_sensitive_info(_safe_get_response_text(e.response)) raise MaskedHTTPStatusError(e, message=_text, text=_text) from None diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 2ffc7acbfb1..fa1253d9005 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5255,7 +5255,6 @@ class BaseLLMHTTPHandler: headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", - "OpenAI-Beta": "realtime=v1", } if extra_headers: diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index b5a8b25beba..1824314865c 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,7 +2,18 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import ( + Any, + Coroutine, + Dict, + List, + Literal, + Optional, + Tuple, + Union, + cast, + overload, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, @@ -21,6 +32,61 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig class HostedVLLMChatConfig(OpenAIGPTConfig): + def _convert_custom_tools_to_function_tools( + self, tools: List[Dict[str, Any]] + ) -> List[Dict[str, Any]]: + """ + vLLM chat completions currently accepts only OpenAI function tools. + Convert custom tools into function tools so request validation does not fail. + """ + converted_tools: List[Dict[str, Any]] = [] + for idx, tool in enumerate(tools): + if not isinstance(tool, dict): + converted_tools.append(tool) + continue + + if tool.get("type") != "custom": + converted_tools.append(tool) + continue + + custom_tool = tool.get("custom", {}) + if not isinstance(custom_tool, dict): + custom_tool = {} + + tool_name = ( + custom_tool.get("name") or tool.get("name") or f"custom_tool_{idx}" + ) + tool_description = custom_tool.get("description") or tool.get("description") + tool_parameters = custom_tool.get("input_schema") or tool.get( + "input_schema" + ) + + if not isinstance(tool_parameters, dict): + tool_parameters = { + "type": "object", + "properties": { + "input": { + "type": "string", + "description": "Raw tool input payload.", + } + }, + "required": ["input"], + } + + function_tool: Dict[str, Any] = { + "type": "function", + "function": { + "name": str(tool_name), + "parameters": tool_parameters, + }, + } + if isinstance(tool_description, str): + function_tool["function"]["description"] = tool_description + + converted_tools.append(function_tool) + + return converted_tools + def get_supported_openai_params(self, model: str) -> List[str]: params = super().get_supported_openai_params(model) params.extend(["reasoning_effort", "thinking"]) @@ -39,6 +105,8 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): _tools = _remove_additional_properties(_tools) # remove 'strict' from tools _tools = _remove_strict_from_schema(_tools) + if isinstance(_tools, list): + _tools = self._convert_custom_tools_to_function_tools(_tools) if _tools is not None: non_default_params["tools"] = _tools diff --git a/litellm/llms/nvidia_riva/__init__.py b/litellm/llms/nvidia_riva/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/nvidia_riva/audio_transcription/__init__.py b/litellm/llms/nvidia_riva/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/nvidia_riva/audio_transcription/audio_utils.py b/litellm/llms/nvidia_riva/audio_transcription/audio_utils.py new file mode 100644 index 00000000000..253d6d2f73f --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/audio_utils.py @@ -0,0 +1,232 @@ +""" +Audio resampling utilities for the NVIDIA Riva STT provider. + +We intentionally avoid a hard dependency on ``ffmpeg`` so this works in +slim Python environments. Format coverage: + +- ``soundfile`` handles wav / flac / ogg out of the box (libsndfile). +- ``audioread`` is tried for everything ``soundfile`` cannot decode (mp3, + m4a, mp4, webm, ...). This is a soft optional dependency. + +If neither library can decode the input we raise a clear error instructing +the caller to convert the audio upstream. +""" + +import io +import os +import tempfile +from dataclasses import dataclass +from typing import Any, Tuple, cast + +from litellm.llms.nvidia_riva.audio_transcription.transformation import ( + RIVA_TARGET_NUM_CHANNELS, + RIVA_TARGET_SAMPLE_RATE_HZ, +) +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException + +# Keep this as Any: the module intentionally avoids importing numpy at module +# import time (optional dependency), and project-wide mypy config evaluates this +# file in contexts where conditional type aliases can degrade to "FloatArray?". +FloatArray = Any + + +_INSTALL_HINT = ( + "Install Riva STT extras to enable automatic audio resampling: " + "`pip install 'litellm[stt-nvidia-riva]'`" +) + + +@dataclass +class ResampledAudio: + pcm_bytes: bytes + duration_seconds: float + sample_rate_hz: int + num_channels: int + + +def resample_to_riva_pcm(file_bytes: bytes) -> ResampledAudio: + """ + Decode ``file_bytes`` and produce 16 kHz mono LINEAR_PCM (int16 little + endian) suitable for streaming to Riva, plus the audio duration in + seconds (used for cost calculation when Riva does not return usage). + """ + try: + import numpy as np # type: ignore + except ImportError as e: + raise NvidiaRivaException( + status_code=500, + message=f"numpy is required for Riva audio resampling. {_INSTALL_HINT}", + ) from e + + samples_float, source_rate = _decode_to_float32(file_bytes) + + # Downmix to mono by averaging channels. + if samples_float.ndim == 2 and samples_float.shape[1] > 1: + samples_float = samples_float.mean(axis=1) + elif samples_float.ndim == 2: + samples_float = samples_float[:, 0] + + samples_float = np.asarray(samples_float, dtype=np.float32).ravel() + + if source_rate != RIVA_TARGET_SAMPLE_RATE_HZ: + samples_float = _resample( + samples_float, source_rate, RIVA_TARGET_SAMPLE_RATE_HZ + ) + + # Clip + convert float [-1, 1] to int16 little-endian PCM. + np.clip(samples_float, -1.0, 1.0, out=samples_float) + pcm_int16 = (samples_float * 32767.0).astype(" Tuple["FloatArray", int]: + """ + Decode arbitrary audio bytes into a float32 array shaped either + ``(n_samples,)`` (mono) or ``(n_samples, n_channels)`` plus the source + sample rate. + + Tries ``soundfile`` first (wav/flac/ogg), then falls back to + ``audioread`` for compressed formats. Raises a clear error if neither + works. + """ + import numpy as np # type: ignore + + sf_error: Exception | None = None + try: + import soundfile as sf # type: ignore + + with io.BytesIO(file_bytes) as buf: + data, source_rate = sf.read(buf, dtype="float32", always_2d=False) + return cast("FloatArray", data), int(source_rate) + except ImportError as e: + sf_error = e + except Exception as e: + # soundfile raises RuntimeError / LibsndfileError for formats it + # cannot decode (mp3 on older libsndfile, m4a, webm, ...). + sf_error = e + + try: + import audioread # type: ignore + except ImportError as e: + raise NvidiaRivaException( + status_code=400, + message=( + "Could not decode audio for Riva STT. Install audio extras " + f"(`pip install 'litellm[stt-nvidia-riva]'`) or convert your " + f"audio to wav/flac/ogg before calling the API. " + f"Underlying error: {sf_error}" + ), + ) from e + + # audioread backends (FFmpeg subprocess, GStreamer, Core Audio) require a + # filesystem path, so spill the bytes to a temp file. mkstemp is portable + # to Windows where re-opening a NamedTemporaryFile is not allowed. + fd, tmp_path = tempfile.mkstemp(suffix=".audio") + try: + with os.fdopen(fd, "wb") as tmp_file: + tmp_file.write(file_bytes) + try: + with audioread.audio_open(tmp_path) as src: + source_rate = int(src.samplerate) + channels = int(src.channels) + chunks = [] + for buf in src: + chunks.append(np.frombuffer(buf, dtype=np.int16)) + if not chunks: + raise NvidiaRivaException( + status_code=400, + message="Audio decode produced no samples.", + ) + interleaved = np.concatenate(chunks).astype(np.float32) / 32768.0 + if channels > 1: + interleaved = interleaved.reshape(-1, channels) + return cast("FloatArray", interleaved), source_rate + except NvidiaRivaException: + raise + except Exception as e: + raise NvidiaRivaException( + status_code=400, + message=( + "Could not decode audio for Riva STT. Convert your audio to " + f"wav/flac/ogg before calling the API. Underlying error: {e}" + ), + ) from e + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _resample( + samples: "FloatArray", source_rate: int, target_rate: int +) -> "FloatArray": + """ + Resample mono float32 ``samples`` from ``source_rate`` to ``target_rate``. + + Prefers high-quality polyphase resampling when ``soxr`` or ``scipy`` is + available (anti-aliased, important for downsampling 44.1/48 kHz -> 16 kHz + where naive interpolation folds high frequencies back into the speech + band). Falls back to linear interpolation if neither is installed — + acceptable for speech-only mono input but lossy for wideband content. + """ + import numpy as np # type: ignore + + if source_rate == target_rate or samples.size == 0: + return samples + + try: + import soxr # type: ignore + + return cast( + "FloatArray", + np.asarray( + soxr.resample(samples, source_rate, target_rate), dtype=np.float32 + ), + ) + except ImportError: + pass + + try: + from math import gcd + + from scipy.signal import resample_poly # type: ignore + + g = gcd(int(source_rate), int(target_rate)) + up = int(target_rate) // g + down = int(source_rate) // g + return cast( + "FloatArray", np.asarray(resample_poly(samples, up, down), dtype=np.float32) + ) + except ImportError: + pass + + return _linear_resample(samples, source_rate, target_rate) + + +def _linear_resample( + samples: "FloatArray", source_rate: int, target_rate: int +) -> "FloatArray": + """Linear-interpolation fallback. See :func:`_resample` for caveats.""" + import numpy as np # type: ignore + + duration = samples.size / float(source_rate) + target_length = int(round(duration * target_rate)) + if target_length <= 1: + return samples.astype(np.float32) + + src_indices = np.linspace(0, samples.size - 1, num=target_length, dtype=np.float64) + left = np.floor(src_indices).astype(np.int64) + right = np.minimum(left + 1, samples.size - 1) + frac = (src_indices - left).astype(np.float32) + + return ((1.0 - frac) * samples[left] + frac * samples[right]).astype(np.float32) diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py new file mode 100644 index 00000000000..9740162ba1c --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -0,0 +1,444 @@ +""" +NVIDIA Riva STT handler. + +This module bridges litellm's transcription dispatch to NVIDIA Riva's gRPC +streaming ASR API. We do *not* go through ``base_llm_http_handler`` because +Riva is gRPC-only: HTTP-shaped abstractions (``httpx.Response``, +``api_base/v1/...`` URLs, multipart bodies) do not apply. + +The handler is intentionally a thin orchestration layer: + +1. Resample the inbound audio to 16 kHz mono LINEAR_PCM (Riva's required + wire format). +2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs + from the structured dict produced by + :class:`NvidiaRivaAudioTranscriptionConfig`. +3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS) + vs self-hosted (any host:port, optional TLS) modes. +4. Stream the audio through Riva's ``streaming_response_generator`` and + aggregate ``is_final`` results into a single transcript. +5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed + on ``_hidden_params`` so cost calculation works. + +``riva-client`` is imported lazily so ``litellm`` core remains usable +without the optional STT extras installed. +""" + +import asyncio +import inspect +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from litellm.litellm_core_utils.audio_utils.utils import ( + get_audio_file_name, + process_audio_file, +) +from litellm.llms.nvidia_riva.audio_transcription.audio_utils import ( + resample_to_riva_pcm, +) +from litellm.llms.nvidia_riva.audio_transcription.transformation import ( + NvidiaRivaAudioTranscriptionConfig, + RIVA_TARGET_NUM_CHANNELS, + RIVA_TARGET_SAMPLE_RATE_HZ, +) +from litellm.llms.nvidia_riva.common_utils import ( + NvidiaRivaException, + grpc_error_to_litellm_exception, +) +from litellm.types.utils import FileTypes, TranscriptionResponse +from litellm.utils import convert_to_model_response_object + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) + +# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches +# NVIDIA's recommended chunk size for streaming ASR — small enough for +# responsive endpointing, large enough to keep per-RPC overhead low. +_DEFAULT_CHUNK_SAMPLES = 1600 +_DEFAULT_CHUNK_BYTES = _DEFAULT_CHUNK_SAMPLES * 2 # int16 = 2 bytes/sample + + +_RIVA_INSTALL_HINT = ( + "NVIDIA Riva client is not installed. " + "Install with `pip install 'litellm[stt-nvidia-riva]'`." +) + + +class NvidiaRivaAudioTranscription: + """Sync + async entry point for Riva ASR.""" + + def audio_transcriptions( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: "LiteLLMLoggingObj", + api_key: Optional[str], + api_base: Optional[str], + atranscription: bool = False, + provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None, + ): + if provider_config is None: + provider_config = NvidiaRivaAudioTranscriptionConfig() + + if atranscription: + return self.async_audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + ) + + return self._run_sync( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + atranscription=atranscription, + ) + + async def async_audio_transcriptions( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: "LiteLLMLoggingObj", + api_key: Optional[str], + api_base: Optional[str], + provider_config: Optional[NvidiaRivaAudioTranscriptionConfig] = None, + ) -> TranscriptionResponse: + # ``riva-client`` exposes a sync streaming generator, so we offload + # the blocking call to a worker thread to keep the event loop free. + return await asyncio.to_thread( + self._run_sync, + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + provider_config=provider_config or NvidiaRivaAudioTranscriptionConfig(), + atranscription=True, + ) + + def _run_sync( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: "LiteLLMLoggingObj", + api_key: Optional[str], + api_base: Optional[str], + provider_config: NvidiaRivaAudioTranscriptionConfig, + atranscription: bool = False, + ) -> TranscriptionResponse: + if not api_base: + raise NvidiaRivaException( + status_code=400, + message=( + "NVIDIA Riva requires `api_base` (host:port for the gRPC " + "endpoint, e.g. `grpc.nvcf.nvidia.com:443` or " + "`localhost:50051`). Set it in litellm_params or via " + "NVIDIA_RIVA_API_BASE." + ), + ) + + processed = process_audio_file(audio_file) + resampled = resample_to_riva_pcm(processed.file_content) + + request_payload = provider_config.transform_audio_transcription_request( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params={ + **litellm_params, + "api_base": api_base, + "api_key": api_key, + }, + ).data + if not isinstance(request_payload, dict): + raise NvidiaRivaException( + status_code=500, + message="NvidiaRivaAudioTranscriptionConfig produced an unexpected request payload type.", + ) + + recognition_config_dict: Dict[str, Any] = request_payload["recognition_config"] + # The wire format is fixed by our resampler; override anything stale + # the caller passed in so the gRPC config matches the bytes we send. + recognition_config_dict["sample_rate_hertz"] = RIVA_TARGET_SAMPLE_RATE_HZ + recognition_config_dict["audio_channel_count"] = RIVA_TARGET_NUM_CHANNELS + recognition_config_dict["encoding"] = "LINEAR_PCM" + + response_format = request_payload.get("response_format") or "json" + timestamp_granularities = request_payload.get("timestamp_granularities") + + riva_module, riva_asr_module = _import_riva() + auth_obj = self._construct_auth( + riva_module=riva_module, + api_base=api_base, + api_key=api_key, + optional_params=optional_params, + ) + + recognition_config = self._build_recognition_config_proto( + riva_asr_module=riva_asr_module, + recognition_config_dict=recognition_config_dict, + ) + streaming_config = riva_asr_module.StreamingRecognitionConfig( + config=recognition_config, interim_results=False + ) + + logging_obj.pre_call( + input=None, + api_key=api_key, + additional_args={ + "api_base": api_base, + "atranscription": atranscription, + "complete_input_dict": { + "recognition_config": recognition_config_dict, + "nvcf_function_id_set": bool( + optional_params.get("nvcf_function_id") + ), + "use_ssl": optional_params.get("use_ssl"), + }, + }, + ) + + try: + asr_service = riva_module.ASRService(auth_obj) + audio_chunks = self._iter_audio_chunks(resampled.pcm_bytes) + stream_kwargs: Dict[str, Any] = { + "audio_chunks": audio_chunks, + "streaming_config": streaming_config, + } + # Forward the deadline so the stream cannot block forever if the + # server stalls. Older riva-client versions do not accept a + # ``timeout`` kwarg, so pass it only when supported. + if timeout is not None and self._supports_timeout_kwarg( + asr_service.streaming_response_generator + ): + stream_kwargs["timeout"] = float(timeout) + stream = asr_service.streaming_response_generator(**stream_kwargs) + final_results = self._collect_final_results(stream) + except NvidiaRivaException: + raise + except Exception as e: + raise grpc_error_to_litellm_exception(e) from e + + transcription = NvidiaRivaAudioTranscriptionConfig.build_transcription_response( + final_results=final_results, + response_format=response_format, + duration_seconds=resampled.duration_seconds, + timestamp_granularities=timestamp_granularities, + ) + + stringified_response = dict(transcription) + + logging_obj.post_call( + input=get_audio_file_name(audio_file), + api_key=api_key, + additional_args={"complete_input_dict": recognition_config_dict}, + original_response=stringified_response, + ) + + hidden_params = { + "model": model, + "custom_llm_provider": "nvidia_riva", + "audio_transcription_duration": resampled.duration_seconds, + } + + final_response: TranscriptionResponse = convert_to_model_response_object( # type: ignore + response_object=stringified_response, + model_response_object=model_response, + hidden_params=hidden_params, + response_type="audio_transcription", + ) + + return final_response + + def _construct_auth( + self, + riva_module: Any, + api_base: str, + api_key: Optional[str], + optional_params: dict, + ) -> Any: + """ + Build a ``riva.client.Auth`` object. + + - When ``nvcf_function_id`` is provided we attach the NVCF + ``function-id`` and bearer ``authorization`` metadata, and default + ``use_ssl`` to True (NVCF endpoints are TLS-only). + - Otherwise (self-hosted) we default ``use_ssl`` to False but still + honor an explicit override — self-hosted Riva behind an ingress + with TLS termination is a real deployment topology. + """ + nvcf_function_id = optional_params.get("nvcf_function_id") + use_ssl_override = optional_params.get("use_ssl") + use_ssl = ( + bool(use_ssl_override) + if use_ssl_override is not None + else bool(nvcf_function_id) + ) + + metadata: List[Tuple[str, str]] = [] + if nvcf_function_id: + metadata.append(("function-id", str(nvcf_function_id))) + if api_key: + metadata.append(("authorization", f"Bearer {api_key}")) + + try: + return riva_module.Auth( + uri=api_base, use_ssl=use_ssl, metadata_args=metadata + ) + except TypeError: + # Older riva-client signatures used positional-only args. + return riva_module.Auth(None, use_ssl, api_base, metadata) + + def _build_recognition_config_proto( + self, riva_asr_module: Any, recognition_config_dict: Dict[str, Any] + ): + encoding_name = ( + recognition_config_dict.get("encoding") or "LINEAR_PCM" + ).upper() + encoding_enum = getattr( + riva_asr_module.AudioEncoding, + encoding_name, + riva_asr_module.AudioEncoding.LINEAR_PCM, + ) + + config = riva_asr_module.RecognitionConfig( + encoding=encoding_enum, + sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]), + language_code=recognition_config_dict["language_code"], + audio_channel_count=int(recognition_config_dict["audio_channel_count"]), + enable_automatic_punctuation=bool( + recognition_config_dict.get("enable_automatic_punctuation", True) + ), + enable_word_time_offsets=bool( + recognition_config_dict.get("enable_word_time_offsets", False) + ), + max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)), + model=recognition_config_dict.get("model", "") or "", + verbatim_transcripts=bool( + recognition_config_dict.get("verbatim_transcripts", False) + ), + profanity_filter=bool( + recognition_config_dict.get("profanity_filter", False) + ), + ) + + endpointing = recognition_config_dict.get("endpointing_config") + if isinstance(endpointing, dict) and endpointing: + try: + ep = riva_asr_module.EndpointingConfig(**endpointing) + config.endpointing_config.CopyFrom(ep) + except Exception: + # If the user supplied an unknown EndpointingConfig field + # (older Riva server), fall back to Riva's defaults rather + # than failing the whole request. + pass + + return config + + @staticmethod + def _supports_timeout_kwarg(callable_obj: Any) -> bool: + try: + sig = inspect.signature(callable_obj) + except (TypeError, ValueError): + return False + params = sig.parameters + if "timeout" in params: + return True + return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) + + @staticmethod + def _iter_audio_chunks(pcm_bytes: bytes): + for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES): + chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES] + if not chunk: + continue + yield chunk + + @staticmethod + def _collect_final_results(stream) -> List[Dict[str, Any]]: + """ + Walk the gRPC stream, ignore empty / non-final chunks, and return a + list of normalized final-result dicts. Matching the user's note: the + ``id`` blocks with no ``results`` are streaming heartbeats and must + be skipped. + """ + final_results: List[Dict[str, Any]] = [] + for response in stream: + results = getattr(response, "results", None) or [] + for result in results: + if not getattr(result, "is_final", False): + continue + alternatives = getattr(result, "alternatives", None) or [] + if not alternatives: + continue + top = alternatives[0] + transcript = getattr(top, "transcript", "") or "" + words_proto = getattr(top, "words", None) or [] + words = [] + for word in words_proto: + words.append( + { + "word": getattr(word, "word", ""), + "start_time_ms": int(getattr(word, "start_time", 0) or 0), + "end_time_ms": int(getattr(word, "end_time", 0) or 0), + } + ) + final_results.append({"transcript": transcript, "words": words}) + return final_results + + +def _import_riva(): + """ + Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``. + + We try the SDK first (preferred) and fall back to importing the proto + module separately when the SDK packaging changes between versions. + """ + try: + import riva.client as riva_client # type: ignore + except ImportError as e: + raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e + + riva_asr_module = riva_client + if not hasattr(riva_asr_module, "RecognitionConfig"): + try: + import riva.client.proto.riva_asr_pb2 as riva_asr_pb2 # type: ignore + + riva_asr_module = riva_asr_pb2 + except ImportError as e: + raise NvidiaRivaException( + status_code=500, message=_RIVA_INSTALL_HINT + ) from e + + return riva_client, riva_asr_module diff --git a/litellm/llms/nvidia_riva/audio_transcription/transformation.py b/litellm/llms/nvidia_riva/audio_transcription/transformation.py new file mode 100644 index 00000000000..c2dfc25d945 --- /dev/null +++ b/litellm/llms/nvidia_riva/audio_transcription/transformation.py @@ -0,0 +1,284 @@ +""" +Translates from OpenAI's `/v1/audio/transcriptions` to NVIDIA Riva's gRPC +streaming recognize API. + +Riva is gRPC-only, so unlike most providers in this directory the request +"transformation" produced here is a structured dict consumed directly by the +gRPC handler (rather than HTTP form-data). The handler builds Riva +``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs from this +dict at call time. + +Reference: https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-overview.html +""" + +from typing import Any, Dict, List, Optional, Union + +from httpx import Headers, Response + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +from ...base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from ..common_utils import NvidiaRivaException + +# Riva expects a fixed wire format for the audio chunks we stream in. +RIVA_TARGET_SAMPLE_RATE_HZ = 16000 +RIVA_TARGET_NUM_CHANNELS = 1 +RIVA_TARGET_ENCODING = "LINEAR_PCM" + + +class NvidiaRivaAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + """ + Config for NVIDIA Riva ASR (gRPC). + + Supports both NVCF-hosted (``api_base=grpc.nvcf.nvidia.com:443`` + + ``nvcf_function_id``) and self-hosted deployments (any ``host:port``, + optional TLS via ``use_ssl``). + """ + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + # Riva natively understands language + word timestamps. + # `response_format` is honored at response-shaping time in the handler. + return ["language", "response_format", "timestamp_granularities"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for key, value in non_default_params.items(): + if value is None: + continue + + if key == "language": + optional_params["language_code"] = self._normalize_language_code(value) + elif key == "timestamp_granularities": + # OpenAI accepts ["word"], ["segment"], or both. Riva only + # natively exposes word timing, so we toggle it on whenever + # "word" is requested. Segment timing is reconstructed in the + # response transformer. + if isinstance(value, list) and "word" in value: + optional_params["enable_word_time_offsets"] = True + optional_params["timestamp_granularities"] = value + elif key == "response_format": + # Stored verbatim; consumed by transform_audio_transcription_response. + optional_params["response_format"] = value + else: + optional_params[key] = value + + return optional_params + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return NvidiaRivaException( + message=error_message, status_code=status_code, headers=headers + ) + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + """ + Build a structured dict that the gRPC handler consumes. We do *not* + construct protobufs here, so this module remains importable without + ``nvidia-riva-client`` being installed (matching how other providers + defer SDK imports to handler-call time). + """ + recognition_config = self._build_recognition_config_dict( + model=model, + optional_params=optional_params, + ) + + endpointing_config = self._build_endpointing_config_dict(optional_params) + if endpointing_config is not None: + recognition_config["endpointing_config"] = endpointing_config + + request_payload: Dict[str, Any] = { + "recognition_config": recognition_config, + "response_format": optional_params.get("response_format") or "json", + "timestamp_granularities": optional_params.get("timestamp_granularities"), + } + + return AudioTranscriptionRequestData(data=request_payload, files=None) + + def transform_audio_transcription_response( + self, + raw_response: Response, + ) -> TranscriptionResponse: + # Not used: Riva responses come from a gRPC stream, not an httpx + # response. The handler calls _build_transcription_response directly. + raise NotImplementedError( + "NvidiaRivaAudioTranscriptionConfig.transform_audio_transcription_response " + "is not used. The handler builds the TranscriptionResponse directly " + "from Riva's gRPC streaming results." + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + # gRPC auth is constructed in the handler, not via HTTP headers. + return headers + + def _build_recognition_config_dict( + self, model: str, optional_params: dict + ) -> Dict[str, Any]: + """ + Build the Riva ``RecognitionConfig`` shape as a plain dict. + + ``model`` is intentionally left empty when the user has not supplied + ``riva_model_name``. Riva auto-selects the right deployment from + ``language_code`` + ``sample_rate_hertz``. NVIDIA's internal + deployment names (e.g. ``parakeet-1.1b-en-US-asr-streaming-...``) + change across NIM versions, regions, and self-hosted builds, so + hardcoding any name here would break unpredictably. + """ + return { + "language_code": optional_params.get("language_code", "en-US"), + "sample_rate_hertz": optional_params.get( + "sample_rate_hertz", RIVA_TARGET_SAMPLE_RATE_HZ + ), + "encoding": optional_params.get("encoding", RIVA_TARGET_ENCODING), + "audio_channel_count": optional_params.get( + "audio_channel_count", RIVA_TARGET_NUM_CHANNELS + ), + "enable_automatic_punctuation": optional_params.get( + "enable_automatic_punctuation", True + ), + "enable_word_time_offsets": bool( + optional_params.get("enable_word_time_offsets", False) + ), + "max_alternatives": optional_params.get("max_alternatives", 1), + "model": optional_params.get("riva_model_name", ""), + "verbatim_transcripts": optional_params.get("verbatim_transcripts", False), + "profanity_filter": optional_params.get("profanity_filter", False), + } + + def _build_endpointing_config_dict( + self, optional_params: dict + ) -> Optional[Dict[str, Any]]: + """ + Translate an OpenAI-style ``chunking_strategy`` into Riva's + ``EndpointingConfig`` shape, or pass through an explicit + ``endpointing_config`` dict. + + Returns ``None`` when neither is provided so Riva uses its built-in + VAD defaults. + """ + explicit = optional_params.get("endpointing_config") + if isinstance(explicit, dict): + return dict(explicit) + + chunking = optional_params.get("chunking_strategy") + if chunking in (None, "auto"): + return None + + if isinstance(chunking, dict) and chunking.get("type") == "server_vad": + config: Dict[str, Any] = {} + if "threshold" in chunking: + threshold = float(chunking["threshold"]) + config["start_threshold"] = threshold + config["stop_threshold"] = threshold + if "silence_duration_ms" in chunking: + config["stop_history"] = int(chunking["silence_duration_ms"]) + if "prefix_padding_ms" in chunking: + config["stop_history_eou"] = int(chunking["prefix_padding_ms"]) + return config or None + + return None + + @staticmethod + def _normalize_language_code(language: str) -> str: + """ + OpenAI accepts bare ISO-639 codes like ``en``; Riva requires BCP-47 + like ``en-US``. Normalize the most common bare codes; pass through + anything that already looks like BCP-47. + """ + if not isinstance(language, str) or not language: + return "en-US" + if "-" in language: + return language + bare_to_bcp47 = { + "en": "en-US", + "es": "es-ES", + "de": "de-DE", + "fr": "fr-FR", + "it": "it-IT", + "pt": "pt-BR", + "ja": "ja-JP", + "ko": "ko-KR", + "zh": "zh-CN", + "ru": "ru-RU", + "hi": "hi-IN", + "ar": "ar-SA", + } + return bare_to_bcp47.get(language.lower(), language) + + @staticmethod + def build_transcription_response( + final_results: List[Dict[str, Any]], + response_format: str, + duration_seconds: Optional[float], + timestamp_granularities: Optional[List[str]], + ) -> TranscriptionResponse: + """ + Aggregate a list of normalized "final result" dicts into a + ``TranscriptionResponse`` shaped for the requested ``response_format``. + + Each entry in ``final_results`` is expected to look like:: + + { + "transcript": str, + "words": [{"word": str, "start_time_ms": int, "end_time_ms": int}, ...], + } + + which the handler produces by walking the gRPC stream and keeping + only ``result.is_final`` entries (empty/non-final chunks are + ignored). + """ + full_transcript = "".join( + (item.get("transcript") or "") for item in final_results + ).strip() + + response = TranscriptionResponse(text=full_transcript) + response["task"] = "transcribe" + + if response_format == "verbose_json": + words: List[Dict[str, Any]] = [] + if timestamp_granularities and "word" in timestamp_granularities: + for item in final_results: + for word in item.get("words", []) or []: + words.append( + { + "word": word.get("word", ""), + "start": (float(word.get("start_time_ms", 0)) / 1000.0), + "end": float(word.get("end_time_ms", 0)) / 1000.0, + } + ) + if words: + response["words"] = words + if duration_seconds is not None: + response["duration"] = duration_seconds + + return response diff --git a/litellm/llms/nvidia_riva/common_utils.py b/litellm/llms/nvidia_riva/common_utils.py new file mode 100644 index 00000000000..a3071cf7060 --- /dev/null +++ b/litellm/llms/nvidia_riva/common_utils.py @@ -0,0 +1,92 @@ +""" +Common utilities and exceptions for the NVIDIA Riva STT provider +""" + +from typing import Any, Optional + +from litellm.llms.base_llm.chat.transformation import BaseLLMException + + +class NvidiaRivaException(BaseLLMException): + """ + Exception raised for NVIDIA Riva (gRPC) errors. + + ``status_code`` is an HTTP-equivalent code derived from the underlying + gRPC ``StatusCode`` (when available) so that litellm's existing error + classifiers (RateLimitError, AuthenticationError, etc.) keep working. + """ + + pass + + +# Mapping from grpc.StatusCode.name -> equivalent HTTP status code. +# Kept as a plain dict (rather than importing grpc enums) so this module is +# importable without grpc installed. +_GRPC_STATUS_CODE_TO_HTTP: dict = { + "OK": 200, + "CANCELLED": 499, + "UNKNOWN": 500, + "INVALID_ARGUMENT": 400, + "DEADLINE_EXCEEDED": 504, + "NOT_FOUND": 404, + "ALREADY_EXISTS": 409, + "PERMISSION_DENIED": 403, + "RESOURCE_EXHAUSTED": 429, + "FAILED_PRECONDITION": 400, + "ABORTED": 409, + "OUT_OF_RANGE": 400, + "UNIMPLEMENTED": 501, + "INTERNAL": 500, + "UNAVAILABLE": 503, + "DATA_LOSS": 500, + "UNAUTHENTICATED": 401, +} + + +def _extract_grpc_status_name(error: Any) -> Optional[str]: + """ + Best-effort extraction of a gRPC StatusCode name from an arbitrary error. + + Works for ``grpc.RpcError`` instances (which expose ``.code()``) as well + as plain exceptions whose string representation contains a status name. + """ + code_fn = getattr(error, "code", None) + if callable(code_fn): + try: + code = code_fn() + except Exception: + code = None + name = getattr(code, "name", None) + if isinstance(name, str): + return name + return None + + +def _extract_grpc_details(error: Any) -> Optional[str]: + """Best-effort extraction of a human-readable detail string from a gRPC error.""" + details_fn = getattr(error, "details", None) + if callable(details_fn): + try: + details = details_fn() + except Exception: + details = None + if isinstance(details, str) and details: + return details + return None + + +def grpc_error_to_litellm_exception(error: Exception) -> NvidiaRivaException: + """ + Convert a gRPC error (or any exception raised from the Riva client) into + a ``NvidiaRivaException`` with an appropriate HTTP-equivalent status code. + """ + status_name = _extract_grpc_status_name(error) + http_status = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500) + + detail = _extract_grpc_details(error) or str(error) + message = ( + f"NVIDIA Riva gRPC error ({status_name}): {detail}" + if status_name + else f"NVIDIA Riva error: {detail}" + ) + return NvidiaRivaException(status_code=http_status, message=message) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 5ca0a3186f7..f34dae2df09 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -6,12 +6,15 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast -from litellm._logging import _redact_string +from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from ....litellm_core_utils.realtime_streaming import RealTimeStreaming +from ....litellm_core_utils.realtime_streaming import ( + RealTimeStreaming, + client_sent_openai_beta_realtime_header, +) from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion @@ -33,21 +36,24 @@ class OpenAIRealtime(OpenAIChatCompletion): """ return "https://api.openai.com/" - def _get_additional_headers(self, api_key: str) -> dict: + def _get_additional_headers( + self, + api_key: str, + *, + openai_beta_realtime: bool = False, + ) -> dict: """ - Get additional headers beyond Authorization. - Override this in subclasses to customize headers (e.g., remove OpenAI-Beta). + Headers for the upstream OpenAI Realtime WebSocket. - Args: - api_key: API key for authentication - - Returns: - Dictionary of additional headers + When the client sent ``OpenAI-Beta: realtime=v1`` on the proxy WebSocket, + ``openai_beta_realtime`` is True and the same header is forwarded upstream + so the legacy beta API is used. GA clients omit that header on the client + connection and must send GA-shaped ``session.update`` payloads. """ - return { - "Authorization": f"Bearer {api_key}", - "OpenAI-Beta": "realtime=v1", - } + headers: dict = {"Authorization": f"Bearer {api_key}"} + if openai_beta_realtime: + headers["OpenAI-Beta"] = "realtime=v1" + return headers def _get_ssl_config(self, url: str) -> Any: """ @@ -120,8 +126,16 @@ class OpenAIRealtime(OpenAIChatCompletion): # Get provider-specific SSL configuration ssl_config = self._get_ssl_config(url) - # Get provider-specific headers - headers = self._get_additional_headers(api_key) + openai_beta_realtime = client_sent_openai_beta_realtime_header(websocket) + if not openai_beta_realtime: + verbose_logger.debug( + "OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). " + "If your client expects beta event names, add 'OpenAI-Beta: realtime=v1' " + "to the WebSocket headers sent to the LiteLLM proxy." + ) + headers = self._get_additional_headers( + api_key, openai_beta_realtime=openai_beta_realtime + ) # Log a masked request preview consistent with other endpoints. logging_obj.pre_call( diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index ad6b24d85a3..50c8ee4220e 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -9,7 +9,27 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import StreamingChatCompletionChunk -_response_stream_shape_cache = None + +def _load_sagemaker_response_stream_shape(): + try: + from botocore.loaders import Loader + from botocore.model import ServiceModel + + loader = Loader() + service_dict = loader.load_service_model("sagemaker-runtime", "service-2") + return ServiceModel(service_dict).shape_for( + "InvokeEndpointWithResponseStreamOutput" + ) + except Exception as e: + verbose_logger.warning( + "litellm: could not pre-load sagemaker-runtime response stream shape " + "— SageMaker event-stream decoding will be unavailable. Error: %s", + e, + ) + return None + + +SAGEMAKER_RESPONSE_STREAM_SHAPE = _load_sagemaker_response_stream_shape() class SagemakerError(BaseLLMException): @@ -187,8 +207,18 @@ class AWSEventStreamDecoder: verbose_logger.error(f"Final error parsing accumulated JSON: {e}") def _parse_message_from_event(self, event) -> Optional[str]: + if SAGEMAKER_RESPONSE_STREAM_SHAPE is None: + raise SagemakerError( + status_code=500, + message=( + "SageMaker event-stream shape could not be loaded from botocore. " + "Ensure botocore is correctly installed." + ), + ) response_dict = event.to_response_dict() - parsed_response = self.parser.parse(response_dict, get_response_stream_shape()) + parsed_response = self.parser.parse( + response_dict, SAGEMAKER_RESPONSE_STREAM_SHAPE + ) if response_dict["status_code"] != 200: raise ValueError(f"Bad response code, expected 200: {response_dict}") @@ -204,20 +234,3 @@ class AWSEventStreamDecoder: return None return chunk.decode() # type: ignore[no-any-return] - - -def get_response_stream_shape(): - global _response_stream_shape_cache - if _response_stream_shape_cache is None: - from botocore.loaders import Loader - from botocore.model import ServiceModel - - loader = Loader() - sagemaker_service_dict = loader.load_service_model( - "sagemaker-runtime", "service-2" - ) - sagemaker_service_model = ServiceModel(sagemaker_service_dict) - _response_stream_shape_cache = sagemaker_service_model.shape_for( - "InvokeEndpointWithResponseStreamOutput" - ) - return _response_stream_shape_cache diff --git a/litellm/llms/xai/realtime/handler.py b/litellm/llms/xai/realtime/handler.py index 805cce5a264..eab19f4a6c8 100644 --- a/litellm/llms/xai/realtime/handler.py +++ b/litellm/llms/xai/realtime/handler.py @@ -28,7 +28,12 @@ class XAIRealtime(OpenAIRealtime): """xAI uses a different API base URL.""" return XAI_API_BASE - def _get_additional_headers(self, api_key: str) -> dict: + def _get_additional_headers( + self, + api_key: str, + *, + openai_beta_realtime: bool = False, + ) -> dict: """ xAI does NOT require the OpenAI-Beta header. Only send Authorization header. diff --git a/litellm/main.py b/litellm/main.py index 0553cf9d422..051a82fdd19 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -211,6 +211,12 @@ from .llms.oobabooga.chat import oobabooga from .llms.openai.completion.handler import OpenAITextCompletion from .llms.openai.image_variations.handler import OpenAIImageVariationsHandler from .llms.openai.openai import OpenAIChatCompletion +from .llms.nvidia_riva.audio_transcription.handler import ( + NvidiaRivaAudioTranscription, +) +from .llms.nvidia_riva.audio_transcription.transformation import ( + NvidiaRivaAudioTranscriptionConfig, +) from .llms.openai.transcriptions.handler import OpenAIAudioTranscription from .llms.openai_like.chat.handler import OpenAILikeChatHandler from .llms.openai_like.embedding.handler import OpenAILikeEmbeddingHandler @@ -266,6 +272,7 @@ from .types.utils import ( openai_chat_completions = OpenAIChatCompletion() openai_text_completions = OpenAITextCompletion() openai_audio_transcriptions = OpenAIAudioTranscription() +nvidia_riva_audio_transcriptions = NvidiaRivaAudioTranscription() openai_image_variations = OpenAIImageVariationsHandler() groq_chat_completions = GroqChatCompletion() sap_gen_ai_hub_chat_completions = GenAIHubOrchestration() @@ -1452,14 +1459,14 @@ def completion( # type: ignore # noqa: PLR0915 if eos_token: custom_prompt_dict[model]["eos_token"] = eos_token - if kwargs.get("model_file_id_mapping"): - messages = update_messages_with_model_file_ids( - messages=messages, - model_id=kwargs.get("model_info", {}).get("id", None), - model_file_id_mapping=cast( - Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping") - ), - ) + messages = update_messages_with_model_file_ids( + messages=messages, + model_id=kwargs.get("model_info", {}).get("id", None), + model_file_id_mapping=cast( + Dict[str, Dict[str, str]], + kwargs.get("model_file_id_mapping") or {}, + ), + ) provider_config: Optional[BaseConfig] = None if custom_llm_provider is not None and custom_llm_provider in [ @@ -6605,6 +6612,26 @@ def transcription( litellm_params=litellm_params_dict, shared_session=shared_session, ) + elif custom_llm_provider == "nvidia_riva": + # NVIDIA Riva is gRPC-based, not HTTP. It has its own dedicated handler + # rather than going through base_llm_http_handler. + response = nvidia_riva_audio_transcriptions.audio_transcriptions( + model=model, + audio_file=file, + optional_params=optional_params, + litellm_params=litellm_params_dict, + model_response=model_response, + atranscription=atranscription, + timeout=timeout, + logging_obj=litellm_logging_obj, + api_base=api_base, + api_key=api_key, + provider_config=( + provider_config + if isinstance(provider_config, NvidiaRivaAudioTranscriptionConfig) + else None + ), + ) elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a83df726ed..4fba1980103 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -28874,6 +28874,19 @@ "mode": "chat", "output_cost_per_token": 0.0 }, + "sambanova/MiniMax-M2.7": { + "input_cost_per_token": 3e-07, + "litellm_provider": "sambanova", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.sambanova.ai/plans/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "sambanova/DeepSeek-R1": { "input_cost_per_token": 5e-06, "litellm_provider": "sambanova", diff --git a/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py b/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py new file mode 100644 index 00000000000..97a16ad3e15 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/auth/token_exchange.py @@ -0,0 +1,196 @@ +""" +OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers. + +Exchanges a user's incoming JWT (subject_token) for a scoped access token +at an IDP's token exchange endpoint. The exchanged token is then used to +authenticate requests to the upstream MCP server. + +See: https://datatracker.ietf.org/doc/html/rfc8693 +""" + +import asyncio +import hashlib +import weakref +from typing import TYPE_CHECKING, Dict, Tuple + +import httpx + +from litellm._logging import verbose_logger +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import ( + MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, + MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, + MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, + MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, +) +from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.llms.custom_http import httpxSpecialProvider + +if TYPE_CHECKING: + from litellm.types.mcp_server.mcp_server_manager import MCPServer + +# RFC 8693 grant type constant +TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" + +DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" + + +class TokenExchangeHandler: + """Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers. + + Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so + repeated calls with the same user token skip the IDP round-trip. + """ + + def __init__(self) -> None: + self._cache = InMemoryCache( + max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE, + default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL, + ) + # WeakValueDictionary so locks are GC'd once no coroutine holds a reference, + # preventing unbounded growth with many rotating user tokens. + self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = ( + weakref.WeakValueDictionary() + ) + + def _get_lock(self, cache_key: str) -> asyncio.Lock: + lock = self._locks.get(cache_key) + if lock is None: + lock = asyncio.Lock() + self._locks[cache_key] = lock + return lock + + @staticmethod + def _cache_key(subject_token: str, server_id: str) -> str: + raw = f"{subject_token}:{server_id}" + return hashlib.sha256(raw.encode()).hexdigest() + + async def exchange_token( + self, + subject_token: str, + server: "MCPServer", + ) -> str: + """Exchange *subject_token* for a scoped access token. + + Returns the exchanged ``access_token`` string (suitable for a + ``Bearer`` header). + + Raises ``ValueError`` on configuration or IDP errors. + """ + cache_key = self._cache_key(subject_token, server.server_id) + + # Fast path + cached = self._cache.get_cache(cache_key) + if cached is not None: + return cached + + # Slow path — one exchange at a time per (user, server) pair + async with self._get_lock(cache_key): + cached = self._cache.get_cache(cache_key) + if cached is not None: + return cached + + token, ttl = await self._do_exchange(subject_token, server) + self._cache.set_cache(cache_key, token, ttl=ttl) + return token + + async def _do_exchange( + self, + subject_token: str, + server: "MCPServer", + ) -> Tuple[str, int]: + """POST to the token exchange endpoint with RFC 8693 parameters. + + Returns ``(access_token, ttl_seconds)``. + """ + endpoint = server.token_exchange_endpoint or server.token_url + if not endpoint: + raise ValueError( + f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange " + f"but no token_exchange_endpoint or token_url configured" + ) + if not server.client_id or not server.client_secret: + raise ValueError( + f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange " + f"but missing client_id or client_secret" + ) + + data: Dict[str, str] = { + "grant_type": TOKEN_EXCHANGE_GRANT_TYPE, + "subject_token": subject_token, + "subject_token_type": server.subject_token_type + or DEFAULT_SUBJECT_TOKEN_TYPE, + "client_id": server.client_id, + "client_secret": server.client_secret, + } + if server.audience: + data["audience"] = server.audience + if server.scopes: + data["scope"] = " ".join(server.scopes) + + verbose_logger.debug( + "Exchanging token for MCP server %s at %s (audience=%s)", + server.server_id, + endpoint, + server.audience, + ) + + client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP) + try: + response = await client.post(endpoint, data=data) + response.raise_for_status() + except httpx.HTTPStatusError as exc: + verbose_logger.debug( + "Token exchange IDP error for MCP server %s (status %d)", + server.server_id, + exc.response.status_code, + ) + raise ValueError( + f"Token exchange for MCP server '{server.server_id}' " + f"failed with status {exc.response.status_code}" + ) from exc + + body = response.json() + if not isinstance(body, dict): + raise ValueError( + f"Token exchange response for MCP server '{server.server_id}' " + f"returned non-object JSON (got {type(body).__name__})" + ) + + access_token = body.get("access_token") + if not access_token: + raise ValueError( + f"Token exchange response for MCP server '{server.server_id}' " + f"missing 'access_token'" + ) + + raw_expires_in = body.get("expires_in") + try: + expires_in = ( + int(raw_expires_in) + if raw_expires_in is not None + else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL + ) + except (TypeError, ValueError): + expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL + + ttl = max( + expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS, + MCP_OAUTH2_TOKEN_CACHE_MIN_TTL, + ) + + verbose_logger.info( + "Token exchange succeeded for MCP server %s (expires in %ds)", + server.server_id, + expires_in, + ) + return access_token, ttl + + def invalidate(self, subject_token: str, server_id: str) -> None: + """Remove a cached exchanged token (e.g. after a 401).""" + cache_key = self._cache_key(subject_token, server_id) + self._cache.delete_cache(cache_key) + + +# Module-level singleton +mcp_token_exchange_handler = TokenExchangeHandler() diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 9923c3ce4bf..55d5e4409e8 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -411,6 +411,15 @@ class MCPServerManager: aws_role_name=server_config.get("aws_role_name", None), aws_session_name=server_config.get("aws_session_name", None), instructions=server_config.get("instructions", None), + # Token Exchange (OBO) fields + token_exchange_endpoint=server_config.get( + "token_exchange_endpoint", None + ), + audience=server_config.get("audience", None), + subject_token_type=server_config.get( + "subject_token_type", + "urn:ietf:params:oauth:token-type:access_token", + ), ) self._assign_unique_short_prefix(new_server) self.config_mcp_servers[server_id] = new_server @@ -765,10 +774,23 @@ class MCPServerManager: aws_role_name=aws_creds.get("aws_role_name"), aws_session_name=aws_creds.get("aws_session_name"), instructions=mcp_server.instructions, + # Token Exchange (OBO) fields — read from credentials JSON blob + token_exchange_endpoint=( + credentials_dict.get("token_exchange_endpoint") + if credentials_dict + else None + ), + audience=(credentials_dict.get("audience") if credentials_dict else None), + subject_token_type=( + credentials_dict.get("subject_token_type") if credentials_dict else None + ) + or "urn:ietf:params:oauth:token-type:access_token", ) return new_server - async def _maybe_register_openapi_tools(self, server: MCPServer): + async def _maybe_register_openapi_tools( + self, server: MCPServer, *, initialize_mapping: bool = True + ): """Register OpenAPI tools if the server has a spec_path configured.""" if server.spec_path: verbose_logger.info( @@ -779,7 +801,8 @@ class MCPServerManager: server=server, base_url=server.url or "", ) - self.initialize_tool_name_to_mcp_server_name_mapping() + if initialize_mapping: + self.initialize_tool_name_to_mcp_server_name_mapping() async def add_server(self, mcp_server: LiteLLM_MCPServerTable): try: @@ -1136,6 +1159,29 @@ class MCPServerManager: ######################################################### # Methods that call the upstream MCP servers ######################################################### + @staticmethod + def _extract_bearer_token( + oauth2_headers: Optional[Dict[str, str]], + raw_headers: Optional[Dict[str, str]], + ) -> Optional[str]: + """Extract the bare Bearer token from oauth2_headers or raw_headers. + + Returns the token string without the ``Bearer `` prefix, or ``None`` + if no Authorization header is found. + """ + auth_value: Optional[str] = None + if oauth2_headers and "Authorization" in oauth2_headers: + auth_value = oauth2_headers["Authorization"] + elif raw_headers: + # raw_headers may have lowercase keys depending on the ASGI server + normalized = {k.lower(): v for k, v in raw_headers.items()} + auth_value = normalized.get("authorization") + if auth_value: + if auth_value.startswith("Bearer "): + return auth_value[len("Bearer ") :] + return auth_value + return None + def _build_stdio_env( self, server: MCPServer, @@ -1169,25 +1215,30 @@ class MCPServerManager: mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, stdio_env: Optional[Dict[str, str]] = None, + subject_token: Optional[str] = None, ) -> MCPClient: """ Create an MCPClient instance for the given server. Auth resolution (single place for all auth logic): 1. ``mcp_auth_header`` — per-request/per-user override - 2. OAuth2 client_credentials token — auto-fetched and cached - 3. ``server.authentication_token`` — static token from config/DB + 2. OAuth2 Token Exchange (OBO) — exchange user token for scoped token + 3. OAuth2 client_credentials token — auto-fetched and cached + 4. ``server.authentication_token`` — static token from config/DB Args: server: The server configuration. mcp_auth_header: Optional per-request auth override. extra_headers: Additional headers to forward. stdio_env: Environment variables for stdio transport. + subject_token: Optional user JWT for token exchange (OBO) flow. Returns: Configured MCP client instance. """ - auth_value = await resolve_mcp_auth(server, mcp_auth_header) + auth_value = await resolve_mcp_auth( + server, mcp_auth_header, subject_token=subject_token + ) transport = server.transport or MCPTransport.sse @@ -1978,7 +2029,11 @@ class MCPServerManager: _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024 - def _assign_unique_short_prefix(self, server: MCPServer) -> None: + def _assign_unique_short_prefix( + self, + server: MCPServer, + registry: Optional[Dict[str, MCPServer]] = None, + ) -> None: """Resolve and cache a collision-free short tool prefix on ``server``. Called at registration time for every MCP server entering the @@ -2002,7 +2057,8 @@ class MCPServerManager: return used: Dict[str, str] = {} - for other in self.get_registry().values(): + registry_for_collision_check = registry or self.get_registry() + for other in registry_for_collision_check.values(): if other.server_id == server.server_id: continue if other.short_prefix: @@ -2534,9 +2590,12 @@ class MCPServerManager: if server_auth_header is None: server_auth_header = mcp_auth_header - # oauth2 headers + # Extract subject token for OAuth2 Token Exchange (OBO) flow + subject_token: Optional[str] = None extra_headers: Optional[Dict[str, str]] = None - if mcp_server.auth_type == MCPAuth.oauth2: + if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: + subject_token = self._extract_bearer_token(oauth2_headers, raw_headers) + elif mcp_server.auth_type == MCPAuth.oauth2: if mcp_server.has_client_credentials: # For M2M OAuth servers, Authorization must come from token fetch. extra_headers = None @@ -2604,6 +2663,7 @@ class MCPServerManager: mcp_auth_header=server_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + subject_token=subject_token, ) call_tool_params = MCPCallToolRequestParams( @@ -2916,46 +2976,72 @@ class MCPServerManager: # against the *full* set so dedup is deterministic regardless of # iteration order. for server in db_mcp_servers: - existing_server = previous_registry.get(server.server_id) + try: + existing_server = previous_registry.get(server.server_id) - if ( - existing_server is not None - and existing_server.updated_at is not None - and server.updated_at is not None - and existing_server.updated_at == server.updated_at - ): - # Re-use existing server instance to avoid re-running build_mcp_server_from_table() - # which can perform network discovery for OAuth2 servers. - new_registry[server.server_id] = existing_server - continue + if ( + existing_server is not None + and existing_server.updated_at is not None + and server.updated_at is not None + and existing_server.updated_at == server.updated_at + ): + # Re-use existing server instance to avoid re-running build_mcp_server_from_table() + # which can perform network discovery for OAuth2 servers. + new_registry[server.server_id] = existing_server + continue - _warn_on_server_name_fields( - server_id=server.server_id, - alias=getattr(server, "alias", None), - server_name=getattr(server, "server_name", None), - ) - verbose_logger.debug( - f"Building server from DB: {server.server_id} ({server.server_name})" - ) - new_server = await self.build_mcp_server_from_table(server) - # Carry the cached short_prefix from the previous registry entry - # (if any) so the prefix is stable across reloads. - if existing_server is not None and existing_server.short_prefix: - new_server.short_prefix = existing_server.short_prefix - new_registry[server.server_id] = new_server + _warn_on_server_name_fields( + server_id=server.server_id, + alias=getattr(server, "alias", None), + server_name=getattr(server, "server_name", None), + ) + verbose_logger.debug( + f"Building server from DB: {server.server_id} ({server.server_name})" + ) + new_server = await self.build_mcp_server_from_table(server) + # Carry the cached short_prefix from the previous registry entry + # (if any) so the prefix is stable across reloads. + if existing_server is not None and existing_server.short_prefix: + new_server.short_prefix = existing_server.short_prefix + new_registry[server.server_id] = new_server + except Exception as e: + verbose_logger.exception( + "Skipping MCP server %s (%s) during DB reload: %s", + server.server_id, + getattr(server, "alias", None), + e, + ) - # Swap in the new registry first so _assign_unique_short_prefix - # sees the complete set when checking for collisions. - self.registry = new_registry - for new_server in new_registry.values(): - self._assign_unique_short_prefix(new_server) - # Register OpenAPI tools *after* the final short prefix is assigned - # so the tools are stored in the global registry under the same - # prefix that lookups will use. - await self._maybe_register_openapi_tools(new_server) + # Assign short prefixes against the full candidate set without + # publishing the staged registry to concurrent callers. + registered_registry: Dict[str, MCPServer] = {} + registered_openapi_tools = False + for server_id, new_server in new_registry.items(): + try: + self._assign_unique_short_prefix(new_server, registry=new_registry) + # Register OpenAPI tools *after* the final short prefix is assigned + # so the tools are stored in the global registry under the same + # prefix that lookups will use. + await self._maybe_register_openapi_tools( + new_server, initialize_mapping=False + ) + registered_registry[server_id] = new_server + if new_server.spec_path: + registered_openapi_tools = True + except Exception as e: + verbose_logger.exception( + "Skipping MCP server %s (%s) during DB reload: %s", + new_server.server_id, + getattr(new_server, "alias", None), + e, + ) + + self.registry = registered_registry + if registered_openapi_tools: + self.initialize_tool_name_to_mcp_server_name_mapping() verbose_logger.debug( - "MCP registry refreshed (%s servers in registry)", len(new_registry) + "MCP registry refreshed (%s servers in registry)", len(registered_registry) ) def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]: diff --git a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py index 476e215666e..92ef57d8cd5 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py +++ b/litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py @@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy._experimental.mcp_server.auth import token_exchange from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: @@ -50,12 +51,23 @@ class MCPOAuth2TokenCache(InMemoryCache): def _get_lock(self, server_id: str) -> asyncio.Lock: return self._locks.setdefault(server_id, asyncio.Lock()) - async def async_get_token(self, server: "MCPServer") -> Optional[str]: + @staticmethod + def _has_client_credentials_config(server: "MCPServer") -> bool: + return bool(server.client_id and server.client_secret and server.token_url) + + async def async_get_token( + self, + server: "MCPServer", + *, + require_client_credentials_flow: bool = True, + ) -> Optional[str]: """Return a valid access token, fetching or refreshing as needed. Returns ``None`` when the server lacks client credentials config. """ - if not server.has_client_credentials: + if require_client_credentials_flow and not server.has_client_credentials: + return None + if not self._has_client_credentials_config(server): return None server_id = server.server_id @@ -263,16 +275,38 @@ mcp_per_user_token_cache = MCPPerUserTokenCache() async def resolve_mcp_auth( server: "MCPServer", mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, + subject_token: Optional[str] = None, ) -> Optional[Union[str, Dict[str, str]]]: """Resolve the auth value for an MCP server. Priority: 1. ``mcp_auth_header`` — per-request/per-user override - 2. OAuth2 client_credentials token — auto-fetched and cached - 3. ``server.authentication_token`` — static token from config/DB + 2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token + 3. OAuth2 client_credentials token — auto-fetched and cached + 4. ``server.authentication_token`` — static token from config/DB """ if mcp_auth_header: return mcp_auth_header + if server.has_token_exchange_config: + if subject_token: + return await token_exchange.mcp_token_exchange_handler.exchange_token( + subject_token, server + ) + # No subject_token — fall back to client_credentials using the same client + # credentials and token_url so M2M scenarios still work. + if server.client_id and server.client_secret and server.token_url: + return await mcp_oauth2_token_cache.async_get_token( + server, + require_client_credentials_flow=False, + ) + # OBO configured but no subject_token and missing client credentials — warn + # rather than silently proceeding unauthenticated. + verbose_logger.warning( + "MCP server '%s' is configured for token exchange (OBO) but no subject_token " + "was provided and client credentials (client_id/client_secret/token_url) are " + "incomplete. The request will proceed without authentication.", + server.server_id, + ) if server.has_client_credentials: return await mcp_oauth2_token_cache.async_get_token(server) return server.authentication_token diff --git a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py index 3b2fa097b70..718435cce6f 100644 --- a/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py +++ b/litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py @@ -6,10 +6,34 @@ import asyncio import contextvars import json import os +import re from pathlib import PurePosixPath from typing import Any, Dict, List, Optional from urllib.parse import quote +# Tool names emitted from OpenAPI specs must work across all major LLM providers. +# OpenAI/Anthropic/Bedrock all enforce a character class roughly equivalent to +# ^[a-zA-Z0-9_-]+$ on tool names. Many specs (notably GitHub's REST API) use +# tag-namespaced operationIds like "actions/download-job-logs-for-workflow-run" +# which include '/'. Sanitize here so the same regex passes everywhere downstream. +_OPENAPI_TOOL_NAME_INVALID_CHARS = re.compile(r"[^a-zA-Z0-9_-]") +_OPENAPI_TOOL_NAME_MAX_LEN = 128 + + +def sanitize_openapi_tool_name(raw_name: str) -> str: + """Map an OpenAPI operationId / fallback to a provider-safe tool name. + + Replaces any character outside ``[a-zA-Z0-9_-]`` with ``_`` and caps the + result at 128 chars (the most restrictive of the major providers). + Lowercased to match the existing convention in + ``register_tools_from_openapi``. + """ + if not raw_name: + return raw_name + sanitized = _OPENAPI_TOOL_NAME_INVALID_CHARS.sub("_", raw_name).lower() + return sanitized[:_OPENAPI_TOOL_NAME_MAX_LEN] + + from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -399,17 +423,36 @@ def create_tool_function( def register_tools_from_openapi(spec: Dict[str, Any], base_url: str): """Register MCP tools from OpenAPI specification.""" paths = spec.get("paths", {}) + used_names: set = set() for path, path_item in paths.items(): for method in ["get", "post", "put", "delete", "patch"]: if method in path_item: operation = path_item[method] - # Generate tool name - operation_id = operation.get( - "operationId", f"{method}_{path.replace('/', '_')}" - ) - tool_name = operation_id.replace(" ", "_").lower() + # Generate tool name. Sanitize to ^[a-zA-Z0-9_-]+$ (lowercase) + # so the resulting name is valid across OpenAI/Anthropic/Bedrock. + # Many specs (e.g. GitHub REST) use tag-namespaced operationIds + # like "actions/download-job-logs-for-workflow-run" which + # contain '/' and would 400 at the LLM provider boundary. + operation_id = operation.get("operationId", f"{method}_{path}") + tool_name = sanitize_openapi_tool_name(operation_id) + + # Disambiguate collisions: two operationIds that differ only + # by sanitized characters (e.g. "foo/list" and "foo.list") + # would both become "foo_list". Append _2, _3, … to keep + # every tool reachable, mirroring the Anthropic-side logic + # in _build_anthropic_tool_name_maps. + unique = tool_name + n = 1 + while unique in used_names: + n += 1 + suffix = f"_{n}" + unique = ( + tool_name[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix + ) + tool_name = unique + used_names.add(tool_name) # Get description description = operation.get( diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 3047fb73325..829863d2dbb 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -857,6 +857,7 @@ if MCP_AVAILABLE: ######################################################## from litellm.proxy.management_endpoints.mcp_management_endpoints import ( NewMCPServerRequest, + _inherit_credentials_from_existing_server, ) def _extract_credentials( @@ -975,9 +976,11 @@ if MCP_AVAILABLE: async def _preview_openapi_tools(spec_path: str) -> dict: """Generate tool previews from an OpenAPI spec without creating a server.""" from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + _OPENAPI_TOOL_NAME_MAX_LEN, build_input_schema, load_openapi_spec_async, resolve_operation_params, + sanitize_openapi_tool_name, ) try: @@ -985,8 +988,9 @@ if MCP_AVAILABLE: paths = spec.get("paths", {}) components = spec.get("components", {}) tools: List[dict] = [] + used_names: set = set() for path, path_item in paths.items(): - for method in ("get", "post", "put", "patch", "delete"): + for method in ("get", "post", "put", "delete", "patch"): operation = path_item.get(method) if operation is None: continue @@ -995,7 +999,23 @@ if MCP_AVAILABLE: operation, path_item, components ) - op_id = operation.get("operationId", f"{method}_{path}") + raw_op_id = operation.get("operationId", f"{method}_{path}") + # Match what register_tools_from_openapi does so the preview + # the user sees in the dashboard equals the names that get + # registered (and shipped to LLM providers, which enforce + # ^[a-zA-Z0-9_-]+$). See sanitize_openapi_tool_name docstring. + op_id = sanitize_openapi_tool_name(raw_op_id) + + unique = op_id + n = 1 + while unique in used_names: + n += 1 + suffix = f"_{n}" + unique = ( + op_id[: _OPENAPI_TOOL_NAME_MAX_LEN - len(suffix)] + suffix + ) + op_id = unique + used_names.add(op_id) summary = operation.get("summary", "") description = operation.get("description", summary) input_schema = build_input_schema(resolved_op) @@ -1068,6 +1088,10 @@ if MCP_AVAILABLE: }, ) + new_mcp_server_request = _inherit_credentials_from_existing_server( + new_mcp_server_request + ) + # For OpenAPI spec servers, generate tools from the spec directly if new_mcp_server_request.spec_path: return await _preview_openapi_tools(new_mcp_server_request.spec_path) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c6653a722d6..ed20fe86cdc 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -656,6 +656,13 @@ class LiteLLMRoutes(enum.Enum): "/health/services", ] + info_routes + # Stateless validators on caller-supplied log data; source logs are + # already accessible via spend_tracking_routes, so no scope expansion. + compliance_check_routes = [ + "/compliance/eu-ai-act", + "/compliance/gdpr", + ] + # Routes in `global_spend_tracking_routes` return proxy-wide spend across # every team, customer, and api_key. They are intentionally NOT included # here — non-admin roles must not see other tenants' spend. Admin roles go @@ -675,6 +682,7 @@ class LiteLLMRoutes(enum.Enum): ] + spend_tracking_routes + key_management_routes + + compliance_check_routes ) internal_user_view_only_routes = spend_tracking_routes @@ -3348,6 +3356,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase): ], ) + azure_sentinel: CallbackOnUI = CallbackOnUI( + litellm_callback_name="azure_sentinel", + ui_callback_name="Azure Sentinel", + litellm_callback_params=[ + "AZURE_SENTINEL_DCR_IMMUTABLE_ID", + "AZURE_SENTINEL_ENDPOINT", + "AZURE_SENTINEL_TENANT_ID", + "AZURE_SENTINEL_CLIENT_ID", + "AZURE_SENTINEL_CLIENT_SECRET", + "AZURE_SENTINEL_STREAM_NAME", + ], + ) + openmeter: CallbackOnUI = CallbackOnUI( litellm_callback_name="openmeter", ui_callback_name="OpenMeter", diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 1b6bd3aff21..f6f99eb62c8 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3516,7 +3516,6 @@ async def _check_team_member_budget( if ( team_object is not None and team_object.team_id is not None - and user_object is not None and valid_token is not None and valid_token.user_id is not None ): @@ -3619,6 +3618,7 @@ async def _check_team_member_model_access( llm_router=llm_router, models=member_allowed_models, object_type="team", + team_id=team_object.team_id, ) except ProxyException: raise ProxyException( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index baa08537003..038d2d81277 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1512,7 +1512,7 @@ class ProxyBaseLLMRequestProcessing: status_code=result.status_code, headers=HttpPassThroughEndpointHelpers.get_response_headers( headers=result.headers, - custom_headers=None, + custom_headers=dict(fastapi_response.headers), ), ) diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index bc9efb52b0f..9475779cfdf 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -5,8 +5,10 @@ from typing import Optional from litellm._logging import verbose_proxy_logger from litellm.caching import RedisCache from litellm.constants import ( + SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS, SPEND_LOG_CLEANUP_BATCH_SIZE, SPEND_LOG_CLEANUP_JOB_NAME, + SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES, SPEND_LOG_RUN_LOOPS, ) from litellm.litellm_core_utils.duration_parser import duration_in_seconds @@ -74,6 +76,7 @@ class SpendLogCleanup: """ total_deleted = 0 run_count = 0 + consecutive_failures = 0 while True: if run_count > SPEND_LOG_RUN_LOOPS: verbose_proxy_logger.info( @@ -82,18 +85,50 @@ class SpendLogCleanup: break # Step 1: Find logs and delete them in one go without fetching to application # Delete in batches, limited by self.batch_size - deleted_result = await prisma_client.db.execute_raw( - """ - DELETE FROM "LiteLLM_SpendLogs" - WHERE "request_id" IN ( - SELECT "request_id" FROM "LiteLLM_SpendLogs" - WHERE "startTime" < $1::timestamptz - LIMIT $2 + try: + deleted_result = await prisma_client.db.execute_raw( + """ + DELETE FROM "LiteLLM_SpendLogs" + WHERE "request_id" IN ( + SELECT "request_id" FROM "LiteLLM_SpendLogs" + WHERE "startTime" < $1::timestamptz + LIMIT $2 + ) + """, + cutoff_date, + self.batch_size, ) - """, - cutoff_date, - self.batch_size, - ) + except Exception as batch_exc: + # A single batch failure (e.g. Prisma/DB timeout) must not abort + # the whole run — subsequent batches may still succeed. + consecutive_failures += 1 + verbose_proxy_logger.exception( + "Spend log cleanup batch failed " + "(run_count=%d, consecutive_failures=%d, batch_size=%d, " + "cutoff=%s, total_deleted_so_far=%d): %s: %s", + run_count, + consecutive_failures, + self.batch_size, + cutoff_date.isoformat(), + total_deleted, + type(batch_exc).__name__, + batch_exc, + ) + if ( + consecutive_failures + >= SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES + ): + verbose_proxy_logger.error( + "Aborting spend log cleanup after %d consecutive batch " + "failures; total deleted before abort: %d", + consecutive_failures, + total_deleted, + ) + break + await asyncio.sleep(SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS) + continue + + consecutive_failures = 0 deleted_count = 0 if isinstance(deleted_result, int): @@ -168,7 +203,13 @@ class SpendLogCleanup: verbose_proxy_logger.info(f"Deleted {total_deleted} logs") except Exception as e: - verbose_proxy_logger.error(f"Error during cleanup: {str(e)}") + # .exception() captures the traceback; str(e) alone on a Prisma/DB + # timeout is often empty and gives operators no signal to diagnose. + verbose_proxy_logger.exception( + "Error during spend log cleanup: %s: %s", + type(e).__name__, + e, + ) return # Return after error handling finally: # Only release the lock if it was actually acquired diff --git a/litellm/proxy/health_endpoints/health_app_factory.py b/litellm/proxy/health_endpoints/health_app_factory.py deleted file mode 100644 index c4fe3833650..00000000000 --- a/litellm/proxy/health_endpoints/health_app_factory.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI -from litellm.proxy.health_endpoints._health_endpoints import router as health_router - - -def build_health_app(): - health_app = FastAPI(title="LiteLLM Health Endpoints") - health_app.include_router(health_router) - return health_app diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index f7c0592992f..861083e7dfa 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -498,6 +498,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): "error": f"Model capacity reached for {model}. " f"Priority: {priority}, " f"Rate limit type: {status['rate_limit_type']}, " + f"Model TPM: {model_group_info.tpm if model_group_info.tpm is not None else 'not configured'}, " + f"Model RPM: {model_group_info.rpm if model_group_info.rpm is not None else 'not configured'}, " f"Remaining: {status['limit_remaining']}" }, headers={ @@ -515,8 +517,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): status_code=429, detail={ "error": f"Priority-based rate limit exceeded. " + f"Model: {model}, " f"Priority: {priority}, " f"Rate limit type: {status['rate_limit_type']}, " + f"Model TPM: {model_group_info.tpm if model_group_info.tpm is not None else 'not configured'}, " + f"Model RPM: {model_group_info.rpm if model_group_info.rpm is not None else 'not configured'}, " f"Remaining: {status['limit_remaining']}, " f"Model saturation: {saturation:.1%}" }, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 4497e64c17f..cd797483b29 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -16,6 +16,8 @@ from typing import ( List, Literal, Optional, + Set, + Tuple, TypedDict, Union, cast, @@ -27,8 +29,12 @@ from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_str_from_messages, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata +from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject from litellm.types.utils import ModelResponse, Usage @@ -196,6 +202,29 @@ return results REDIS_CLUSTER_SLOTS = 16384 REDIS_NODE_HASHTAG_NAME = "all_keys" +# TPM token reservation tuning constants. +# When max_tokens is not specified in the request we still need to reserve +# *some* output budget; these define that fallback estimate. +DEFAULT_MAX_TOKENS_ESTIMATE = 4096 +DEFAULT_CHARS_PER_TOKEN = 4 +# Stash for the reserved-token count on the request data dict so success/ +# failure callbacks can reconcile against the upfront reservation. +TPM_RESERVED_TOKENS_KEY = "_litellm_tpm_reserved_tokens" +# Stash for the model identifier the reservation was charged against. +# Reconciliation must target the same key that was incremented at reservation +TPM_RESERVED_MODEL_KEY = "_litellm_tpm_reserved_model" +# Stash for the (scope_key, scope_value) pairs whose :tokens counter the +# upfront reservation incremented. Reconciliation applies the delta to these +# scopes only; scopes without a configured TPM limit were never charged at +# pre-call and must receive the full actual usage instead of the delta — +# otherwise their counters drift negative whenever actual < reserved. +TPM_RESERVED_SCOPES_KEY = "_litellm_tpm_reserved_scopes" +# Idempotency marker for the reservation refund path. Set when any failure +# callback releases the reservation so the next callback in the same flow +# (e.g. async_log_failure_event firing after async_post_call_failure_hook) +# does not double-refund. +TPM_RESERVATION_RELEASED_KEY = "_litellm_tpm_reservation_released" + class RateLimitDescriptorRateLimitObject(TypedDict, total=False): requests_per_unit: Optional[int] @@ -300,6 +329,76 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """Return the current time for rate limiting calculations.""" return self._time_provider() + def _estimate_tokens_for_request( + self, + data: dict, + model: Optional[str] = None, + ) -> int: + """ + Estimate total tokens this request will consume so we can reserve them + upfront (input + output budget): + estimated = input_tokens + max_tokens. + + Supports chat (messages), completions (prompt), and embeddings (input). + """ + messages = data.get("messages") + prompt = data.get("prompt") + input_text = data.get("input") # embeddings + + match (messages, prompt, input_text): + case (messages, _, _) if messages: + total_chars = len(get_str_from_messages(messages)) + case (_, str() as p, _): + total_chars = len(p) + case (_, list() as p, _): + total_chars = sum(len(str(item)) for item in p) + case (_, _, str() as t): + total_chars = len(t) + case (_, _, list() as t): + total_chars = sum(len(str(item)) for item in t) + case _: + total_chars = 0 + + estimated_input_tokens = ( + max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 + ) + + explicit_max_tokens = data.get("max_tokens") or data.get( + "max_completion_tokens" + ) + + match (explicit_max_tokens, input_text): + case (mt, _) if mt is not None: + max_tokens_estimate = int(mt) + case (_, embeddings_input) if embeddings_input: + # Embeddings have no output tokens + max_tokens_estimate = 0 + case _ if total_chars == 0: + # Fully contentless request (no messages, prompt, or input). + # Don't apply the conservative output-budget floor here — it + # would over-reserve and could push small TPM limits into a + # false 429. The caller floors at 1 so backpressure still + # applies once the counter is at limit. + max_tokens_estimate = 0 + case _: + # No max_tokens specified — reserve at least the input size with a + # conservative floor so a stream of small concurrent requests can't + # collectively bypass the limit. + max_tokens_estimate = max( + estimated_input_tokens, + DEFAULT_MAX_TOKENS_ESTIMATE // 4, + ) + + total_estimated = estimated_input_tokens + max_tokens_estimate + + verbose_proxy_logger.debug( + f"TPM reservation estimate: input={estimated_input_tokens}, " + f"max_tokens={max_tokens_estimate} (explicit={explicit_max_tokens is not None}), " + f"total={total_estimated}" + ) + + return total_estimated + def _is_redis_cluster(self) -> bool: """ Check if the dual cache is using Redis cluster. @@ -557,6 +656,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors: List[RateLimitDescriptor], parent_otel_span: Optional[Span] = None, read_only: bool = False, + skip_tpm_check: bool = False, ) -> RateLimitResponse: """ Check if any of the rate limit descriptors should be rate limited. @@ -567,6 +667,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptors: List of rate limit descriptors to check parent_otel_span: Optional OpenTelemetry span for tracing read_only: If True, only check limits without incrementing counters + skip_tpm_check: If True, ignore each descriptor's ``tokens_per_unit`` + — the :tokens counter is neither read nor incremented by this + pass. Callers that handle TPM via the atomic + ``reserve_tpm_tokens`` reservation path should set this to + avoid the +1-per-key Lua / in-memory increment double-charging + the tokens counter. """ current_time = self._get_current_time() @@ -583,7 +689,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject() ) requests_limit = rate_limit.get("requests_per_unit") - tokens_limit = rate_limit.get("tokens_per_unit") + tokens_limit = None if skip_tpm_check else rate_limit.get("tokens_per_unit") max_parallel_requests_limit = rate_limit.get("max_parallel_requests") window_size = rate_limit.get("window_size") or self.window_size @@ -710,6 +816,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): read and increment in both single-process and multi-process (Redis) deployments. + Cluster-safety: each descriptor's keys all share a `{key:value}` hash + tag, so the Redis Lua path issues one Lua call per descriptor — every + call's keys co-locate on a single Redis Cluster slot, avoiding + CROSSSLOT errors. Cross-descriptor atomicity is preserved via + refund-on-rollback: if descriptor i is OVER_LIMIT, descriptors 0..i-1 + get a direct INCRBY refund (refunds need no atomicity guarantee). + Args: descriptors: rate-limit descriptors to check increments: per-descriptor increment amounts, indexed parallel to @@ -726,104 +839,181 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): "must have the same length" ) - keys: List[str] = [] - per_counter_meta: List[Dict[str, Any]] = [] - script_args: List[Any] = [] - + # Build per-descriptor (keys, args, meta) groups. All keys within a + # group share the descriptor's {key:value} hash tag, so a single Lua + # call per group never triggers CROSSSLOT on Redis Cluster. + descriptor_groups: List[Tuple[List[str], List[Any], List[Dict[str, Any]]]] = [] for descriptor, increment_amounts in zip(descriptors, increments): - descriptor_key = descriptor["key"] - descriptor_value = descriptor["value"] - rate_limit: RateLimitDescriptorRateLimitObject = ( - descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject() + keys, args, meta = self._build_descriptor_atomic_payload( + descriptor=descriptor, + increment_amounts=increment_amounts, ) - window_size = rate_limit.get("window_size") or self.window_size - window_key = f"{{{descriptor_key}:{descriptor_value}}}:window" + if keys: + descriptor_groups.append((keys, args, meta)) - for rate_limit_type in ("requests", "tokens"): - rlt: Literal["requests", "tokens"] = cast( - Literal["requests", "tokens"], rate_limit_type - ) - if rlt == "requests": - limit_value = rate_limit.get("requests_per_unit") - inc_amount = int(increment_amounts.get("requests", 0) or 0) - else: - limit_value = rate_limit.get("tokens_per_unit") - inc_amount = int(increment_amounts.get("tokens", 0) or 0) - if limit_value is None or inc_amount <= 0: - continue - counter_key = self.create_rate_limit_keys( - descriptor_key, descriptor_value, rlt - ) - # Counter-key TTL and window_size are conceptually distinct - # ("how long the counter Redis key lives" vs "how long the - # sliding window is"). They happen to be equal today because - # we have no descriptor type that needs them apart, but they - # are kept as separate variables here so a future custom-TTL - # descriptor doesn't reintroduce a silent expiry bug. Both - # the Lua script and the in-memory fallback read these from - # their respective ARGV / meta slots. - ttl_seconds = int(window_size) - window_size_seconds = int(window_size) - keys.extend([window_key, counter_key]) - # Per-counter 4-tuple matches the Lua ARGV layout exactly: - # [limit, increment, ttl_seconds, window_size_seconds]. - script_args.extend( - [ - int(limit_value), - inc_amount, - ttl_seconds, - window_size_seconds, - ] - ) - per_counter_meta.append( - { - "descriptor_key": descriptor_key, - "current_limit": int(limit_value), - "rate_limit_type": rlt, - "window_key": window_key, - "counter_key": counter_key, - "increment": inc_amount, - "ttl": ttl_seconds, - "window_size": window_size_seconds, - } - ) - - if not keys: + if not descriptor_groups: return RateLimitResponse(overall_code="OK", statuses=[]) - # Multi-process atomicity via Redis Lua. Single-process atomicity - # falls back to the asyncio.Lock + in-memory sliding window below. - # Note: in-memory state diverges from Redis state — if Lua fails - # mid-write, retrying via in-memory may double-count. See fallback - # warning below. + # Multi-process atomicity via Redis Lua, per descriptor for slot + # co-location. Single-process atomicity falls back to the + # asyncio.Lock + in-memory sliding window below — there are no + # cluster slot concerns locally, so we keep the batched 2-phase + # critical section for true cross-descriptor atomicity. if self.check_and_increment_by_n_script is not None: + return await self._atomic_lua_per_descriptor( + descriptor_groups=descriptor_groups, + parent_otel_span=parent_otel_span, + ) + + flat_meta: List[Dict[str, Any]] = [ + m for _keys, _args, group_meta in descriptor_groups for m in group_meta + ] + async with self._check_and_increment_lock: + return await self._atomic_check_and_increment_in_memory( + per_counter_meta=flat_meta, + parent_otel_span=parent_otel_span, + ) + + def _build_descriptor_atomic_payload( + self, + descriptor: RateLimitDescriptor, + increment_amounts: Dict[Literal["requests", "tokens"], int], + ) -> Tuple[List[str], List[Any], List[Dict[str, Any]]]: + """ + Build (KEYS, ARGV, per-counter meta) for a single descriptor's Lua + call. All keys returned share the descriptor's {key:value} hash tag. + """ + descriptor_key = descriptor["key"] + descriptor_value = descriptor["value"] + rate_limit: RateLimitDescriptorRateLimitObject = ( + descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject() + ) + window_size = rate_limit.get("window_size") or self.window_size + window_key = f"{{{descriptor_key}:{descriptor_value}}}:window" + + keys: List[str] = [] + args: List[Any] = [] + meta: List[Dict[str, Any]] = [] + + for rate_limit_type in ("requests", "tokens"): + rlt: Literal["requests", "tokens"] = cast( + Literal["requests", "tokens"], rate_limit_type + ) + if rlt == "requests": + limit_value = rate_limit.get("requests_per_unit") + inc_amount = int(increment_amounts.get("requests", 0) or 0) + else: + limit_value = rate_limit.get("tokens_per_unit") + inc_amount = int(increment_amounts.get("tokens", 0) or 0) + if limit_value is None or inc_amount <= 0: + continue + counter_key = self.create_rate_limit_keys( + descriptor_key, descriptor_value, rlt + ) + # Counter-key TTL and window_size are conceptually distinct + # ("how long the counter Redis key lives" vs "how long the + # sliding window is"). Kept as separate values so a future + # custom-TTL descriptor doesn't reintroduce a silent expiry bug. + ttl_seconds = int(window_size) + window_size_seconds = int(window_size) + keys.extend([window_key, counter_key]) + # 4-tuple matches the Lua ARGV layout: + # [limit, increment, ttl_seconds, window_size_seconds]. + args.extend( + [int(limit_value), inc_amount, ttl_seconds, window_size_seconds] + ) + meta.append( + { + "descriptor_key": descriptor_key, + "current_limit": int(limit_value), + "rate_limit_type": rlt, + "window_key": window_key, + "counter_key": counter_key, + "increment": inc_amount, + "ttl": ttl_seconds, + "window_size": window_size_seconds, + } + ) + return keys, args, meta + + async def _atomic_lua_per_descriptor( + self, + descriptor_groups: List[Tuple[List[str], List[Any], List[Dict[str, Any]]]], + parent_otel_span: Optional[Span] = None, + ) -> RateLimitResponse: + """ + Run Lua check-and-increment one descriptor at a time so each call's + keys co-locate on a single Redis Cluster slot. On OVER_LIMIT for + descriptor i, refund descriptors 0..i-1's increments. On Lua failure + mid-loop, refund applied increments and fall back to in-memory. + """ + applied: List[List[Dict[str, Any]]] = [] + statuses: List[RateLimitStatus] = [] + + for _idx, (keys, args, meta) in enumerate(descriptor_groups): try: raw = await self.check_and_increment_by_n_script( keys=keys, - args=script_args, + args=args, ) - return self._build_atomic_response(raw, per_counter_meta) except Exception as e: - # Escalated from warning to error: Lua failures (script timeout, - # Redis OOM, network partition) leave counter state ambiguous. - # The fallback path below uses LOCAL DualCache, which is a - # different store from Redis — counters here will diverge from - # Redis until that key's window expires (TTL bounds divergence). - # Operators should alert on this log line; sustained occurrences - # indicate Redis health degradation that may erode rate-limit - # accuracy. + # Lua failure (timeout, OOM, network partition) leaves Redis + # state ambiguous. Refund any prior groups so Redis returns + # to its pre-call state, then fall back to in-memory for the + # whole call (counters there are independent of Redis). verbose_proxy_logger.error( f"atomic_check_and_increment_by_n: Redis Lua execution " - f"failed ({type(e).__name__}: {e}). Falling back to " - f"in-memory enforcement — counters will diverge from Redis " - f"state until window expires (window_size={self.window_size}s)." + f"failed ({type(e).__name__}: {e}). Refunding " + f"{len(applied)} prior descriptors and falling back to " + f"in-memory enforcement — counters will diverge from " + f"Redis until window expires (window_size=" + f"{self.window_size}s)." ) + await self._refund_applied_descriptor_groups(applied) + flat_meta: List[Dict[str, Any]] = [ + m for _k, _a, group_meta in descriptor_groups for m in group_meta + ] + async with self._check_and_increment_lock: + return await self._atomic_check_and_increment_in_memory( + per_counter_meta=flat_meta, + parent_otel_span=parent_otel_span, + ) - async with self._check_and_increment_lock: - return await self._atomic_check_and_increment_in_memory( - per_counter_meta=per_counter_meta, - parent_otel_span=parent_otel_span, - ) + response = self._build_atomic_response(raw, meta) + if response["overall_code"] == "OVER_LIMIT": + await self._refund_applied_descriptor_groups(applied) + return response + applied.append(meta) + statuses.extend(response["statuses"]) + + return RateLimitResponse(overall_code="OK", statuses=statuses) + + async def _refund_applied_descriptor_groups( + self, + applied: List[List[Dict[str, Any]]], + ) -> None: + """ + Decrement counters for descriptor groups already applied via Lua. + Best-effort: refund failures are logged but not raised — the original + OVER_LIMIT / fallback decision is what matters to the caller. + """ + if not applied: + return + redis_cache = self.internal_usage_cache.dual_cache.redis_cache + if redis_cache is None: + return + for group_meta in applied: + for entry in group_meta: + try: + await redis_cache.async_increment( + key=entry["counter_key"], + value=-entry["increment"], + ) + except Exception as e: + verbose_proxy_logger.warning( + f"Failed to refund {entry['counter_key']} on " + f"cross-descriptor rollback: {e}" + ) def _build_atomic_response( self, @@ -970,6 +1160,39 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) return RateLimitResponse(overall_code="OK", statuses=statuses) + async def reserve_tpm_tokens( + self, + descriptors: List[RateLimitDescriptor], + estimated_tokens: int, + parent_otel_span: Optional[Span] = None, + ) -> RateLimitResponse: + """ + Reserve ``estimated_tokens`` against every TPM-bearing descriptor + BEFORE the upstream call, so concurrent requests cannot all observe + "under limit" before any of them increments the counter. + + Thin wrapper around ``atomic_check_and_increment_by_n``: builds a + TPM-only descriptor/increment list and delegates the all-or-nothing + atomicity (Lua on Redis, asyncio-locked DualCache otherwise) to the + shared primitive. + """ + tpm_descriptors: List[RateLimitDescriptor] = [ + d + for d in descriptors + if (d.get("rate_limit") or {}).get("tokens_per_unit") is not None + ] + if not tpm_descriptors: + return RateLimitResponse(overall_code="OK", statuses=[]) + + increments: List[Dict[Literal["requests", "tokens"], int]] = [ + {"tokens": estimated_tokens} for _ in tpm_descriptors + ] + return await self.atomic_check_and_increment_by_n( + descriptors=tpm_descriptors, + increments=increments, + parent_otel_span=parent_otel_span, + ) + def create_organization_rate_limit_descriptor( self, user_api_key_dict: UserAPIKeyAuth, requested_model: Optional[str] = None ) -> List[RateLimitDescriptor]: @@ -1736,9 +1959,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) # Only check rate limits if we have descriptors with actual limits if descriptors: + # First pass: RPM and max_parallel_requests sliding-window check. + # `skip_tpm_check=True` tells should_rate_limit to ignore each + # descriptor's tokens_per_unit so its +1-per-key Lua / in-memory + # increment never touches the :tokens counters — those are owned + # exclusively by the atomic reserve_tpm_tokens path below. Without + # this, every concurrent in-flight request would pre-inflate the + # :tokens counter by 1, shrinking the effective TPM budget by N + # and causing false-positive 429s under bursts. response = await self.should_rate_limit( descriptors=descriptors, parent_otel_span=user_api_key_dict.parent_otel_span, + skip_tpm_check=True, ) if response["overall_code"] == "OVER_LIMIT": @@ -1750,6 +1982,83 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # add descriptors to request headers data["litellm_proxy_rate_limit_response"] = response + # ---------------------------------------------------------------- + # TPM token reservation + # Atomically reserve estimated tokens upfront so concurrent + # requests cannot all observe "under limit" before any of them + # has incremented the counter. atomic_check_and_increment_by_n + # uses Redis Lua when available and falls back to an asyncio-locked + # in-memory check otherwise — single-worker protection still holds + # even without Redis. + # ---------------------------------------------------------------- + has_tpm_limits = any( + (d.get("rate_limit") or {}).get("tokens_per_unit") is not None + for d in descriptors + ) + + if has_tpm_limits: + # Floor at 1 token so contentless requests (/responses, + # tool-call continuations, empty messages) still flow + # through the atomic counter and get backpressure when at + # limit. Without this floor, N concurrent contentless + # requests would all pass pre-call with no enforcement. + # Post-call reconciliation refunds the over-reservation + # delta when actual usage comes in below the floor. + estimated_tokens = max( + self._estimate_tokens_for_request( + data=data, + model=requested_model, + ), + 1, + ) + + tpm_response = await self.reserve_tpm_tokens( + descriptors=descriptors, + estimated_tokens=estimated_tokens, + parent_otel_span=user_api_key_dict.parent_otel_span, + ) + + if tpm_response["overall_code"] == "OVER_LIMIT": + self._handle_rate_limit_error( + response=tpm_response, + descriptors=descriptors, + ) + else: + data["_litellm_rate_limit_descriptors"] = descriptors + # Capture the exact (key, value) scopes the reservation + # incremented so post-call reconciliation only applies + # the (actual - reserved) delta to those — unreserved + # scopes get charged the full actual usage instead. + reserved_scopes: List[Tuple[str, str]] = [ + (d["key"], d["value"]) + for d in descriptors + if (d.get("rate_limit") or {}).get("tokens_per_unit") + is not None + ] + self._stash_reservation_in_data( + data=data, + estimated_tokens=estimated_tokens, + reserved_model=requested_model, + reserved_scopes=reserved_scopes, + ) + + # Merge TPM statuses into the stored rate-limit response + # so x-ratelimit-{key}-remaining-tokens / -limit-tokens + # headers reach the client. Without this, the RPM-only + # response from should_rate_limit (skip_tpm_check=True) + # silently drops all token headers. + stored_response = data.get("litellm_proxy_rate_limit_response") + if isinstance(stored_response, dict): + stored_response.setdefault("statuses", []).extend( + tpm_response["statuses"] + ) + elif tpm_response["statuses"]: + data["litellm_proxy_rate_limit_response"] = tpm_response + + verbose_proxy_logger.debug( + f"TPM tokens reserved: {estimated_tokens} for model {requested_model}" + ) + def _create_pipeline_operations( self, key: str, @@ -1760,8 +2069,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): """ Create pipeline operations for TPM increments """ - from litellm.types.caching import RedisPipelineIncrementOperation - pipeline_operations: List[RedisPipelineIncrementOperation] = [] counter_key = self.create_rate_limit_keys( key=key, @@ -1925,24 +2232,193 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return "total" # default to total return specified_rate_limit_type - def _build_success_event_pipeline_operations( - self, - kwargs: Any, - response_obj: Any, - rate_limit_type: Literal["output", "input", "total"], - ) -> List["RedisPipelineIncrementOperation"]: - """Build Redis pipeline increment ops for TPM / parallel-request counters.""" - from litellm.proxy.common_utils.callback_utils import ( - get_model_group_from_litellm_kwargs, + @staticmethod + def _stash_reservation_in_data( + data: Dict[str, Any], + estimated_tokens: int, + reserved_model: Optional[str], + reserved_scopes: Optional[List[Tuple[str, str]]] = None, + ) -> None: + """ + Persist the reservation amount, model, and reserved scopes into every + channel a callback might read from: top-level kwargs (via ``**data``), + request metadata, and litellm_metadata. Keeps reservation and + reconciliation in sync. + + ``reserved_scopes`` is serialized as a list of [key, value] pairs so + it round-trips through JSON-based metadata transports. + """ + scopes_payload: Optional[List[List[str]]] = ( + [[k, v] for k, v in reserved_scopes] if reserved_scopes else None ) - from litellm.types.caching import RedisPipelineIncrementOperation - # Get metadata from standard_logging_object - this correctly handles both - # 'metadata' and 'litellm_metadata' fields from litellm_params - standard_logging_object = kwargs.get("standard_logging_object") or {} - standard_logging_metadata = standard_logging_object.get("metadata") or {} + data[TPM_RESERVED_TOKENS_KEY] = estimated_tokens + if reserved_model: + data[TPM_RESERVED_MODEL_KEY] = reserved_model + if scopes_payload is not None: + data[TPM_RESERVED_SCOPES_KEY] = scopes_payload - # user_api_key_hash is the same as user_api_key (it's the hash) + for channel in ("metadata", "litellm_metadata"): + existing = data.get(channel) + if isinstance(existing, dict): + existing[TPM_RESERVED_TOKENS_KEY] = estimated_tokens + if reserved_model: + existing[TPM_RESERVED_MODEL_KEY] = reserved_model + if scopes_payload is not None: + existing[TPM_RESERVED_SCOPES_KEY] = scopes_payload + elif channel == "metadata": + # Only auto-create ``metadata`` (preserves prior behavior); + # ``litellm_metadata`` is set by the router and shouldn't be + # conjured here. + stash: Dict[str, Any] = {TPM_RESERVED_TOKENS_KEY: estimated_tokens} + if reserved_model: + stash[TPM_RESERVED_MODEL_KEY] = reserved_model + if scopes_payload is not None: + stash[TPM_RESERVED_SCOPES_KEY] = scopes_payload + data[channel] = stash + + @staticmethod + def _lookup_stashed_value( + kwargs: Any, + standard_logging_metadata: Optional[Dict[str, Any]], + key: str, + ) -> Any: + """ + Resolve a stashed value from any of the channels the request data can + flow through to a callback. + + Checks (in priority order): + 1. kwargs (top-level data fields propagate via **data) + 2. kwargs["litellm_params"]["metadata"] (request metadata channel) + 3. standard_logging_metadata (covers tests that mock the SLO directly) + """ + candidate = kwargs.get(key) if isinstance(kwargs, dict) else None + if candidate is None: + litellm_params = ( + kwargs.get("litellm_params") if isinstance(kwargs, dict) else None + ) + if isinstance(litellm_params, dict): + lp_metadata = litellm_params.get("metadata") + if isinstance(lp_metadata, dict): + candidate = lp_metadata.get(key) + if candidate is None and isinstance(standard_logging_metadata, dict): + candidate = standard_logging_metadata.get(key) + return candidate + + @classmethod + def _get_reserved_tokens_from_kwargs( + cls, + kwargs: Any, + standard_logging_metadata: Optional[Dict[str, Any]] = None, + ) -> int: + candidate = cls._lookup_stashed_value( + kwargs, standard_logging_metadata, TPM_RESERVED_TOKENS_KEY + ) + try: + return int(candidate or 0) + except (TypeError, ValueError): + return 0 + + @classmethod + def _get_reserved_model_from_kwargs( + cls, + kwargs: Any, + standard_logging_metadata: Optional[Dict[str, Any]] = None, + ) -> Optional[str]: + """ + Resolve the model the upfront reservation was charged against. Used to + target reconciliation at the same key that was incremented, regardless + of whether the router later set a different ``model_group`` in + ``litellm_params.metadata``. + """ + candidate = cls._lookup_stashed_value( + kwargs, standard_logging_metadata, TPM_RESERVED_MODEL_KEY + ) + return candidate if isinstance(candidate, str) and candidate else None + + @classmethod + def _get_reserved_scopes_from_kwargs( + cls, + kwargs: Any, + standard_logging_metadata: Optional[Dict[str, Any]] = None, + ) -> Set[Tuple[str, str]]: + """ + Resolve the (scope_key, scope_value) pairs the upfront reservation + actually charged. Reconciliation distinguishes these from + unreserved scopes — applying the delta to reserved scopes (which + already carry +reserved on the counter) and the full actual to + unreserved ones (which were never charged). + """ + candidate = cls._lookup_stashed_value( + kwargs, standard_logging_metadata, TPM_RESERVED_SCOPES_KEY + ) + if not isinstance(candidate, list): + return set() + scopes: Set[Tuple[str, str]] = set() + for entry in candidate: + if ( + isinstance(entry, (list, tuple)) + and len(entry) == 2 + and isinstance(entry[0], str) + and isinstance(entry[1], str) + ): + scopes.add((entry[0], entry[1])) + return scopes + + @classmethod + def _is_reservation_released( + cls, + kwargs: Any, + standard_logging_metadata: Optional[Dict[str, Any]] = None, + ) -> bool: + """True if a prior callback already refunded this request's reservation.""" + return bool( + cls._lookup_stashed_value( + kwargs, standard_logging_metadata, TPM_RESERVATION_RELEASED_KEY + ) + ) + + @staticmethod + def _mark_reservation_released(data: Any) -> None: + """ + Stamp the released flag into every metadata channel a sibling + callback might read from. async_post_call_failure_hook receives the + request data dict; async_log_failure_event reads kwargs + + standard_logging_object.metadata. Same dict identity across + ``request_data["metadata"]`` and ``kwargs["litellm_params"]["metadata"]`` + means writes here propagate to the other hook. + """ + if not isinstance(data, dict): + return + data[TPM_RESERVATION_RELEASED_KEY] = True + for channel in ("metadata", "litellm_metadata"): + existing = data.get(channel) + if isinstance(existing, dict): + existing[TPM_RESERVATION_RELEASED_KEY] = True + litellm_params = data.get("litellm_params") + if isinstance(litellm_params, dict): + lp_metadata = litellm_params.get("metadata") + if isinstance(lp_metadata, dict): + lp_metadata[TPM_RESERVATION_RELEASED_KEY] = True + slo = data.get("standard_logging_object") + if isinstance(slo, dict): + slo_meta = slo.get("metadata") + if isinstance(slo_meta, dict): + slo_meta[TPM_RESERVATION_RELEASED_KEY] = True + + def _collect_tpm_scope_targets( + self, + standard_logging_metadata: Dict[str, Any], + kwargs: Any, + model_group: Optional[str], + ) -> List[Tuple[str, str]]: + """ + Enumerate every (scope_key, scope_value) pair that *might* carry a + TPM counter for this request — independent of whether each scope had + a configured TPM limit at pre-call. Reservation awareness happens at + the emitter; this helper just lists the candidate scopes so callers + can split reserved-vs-unreserved. + """ user_api_key = standard_logging_metadata.get("user_api_key_hash") user_api_key_user_id = standard_logging_metadata.get("user_api_key_user_id") user_api_key_team_id = standard_logging_metadata.get("user_api_key_team_id") @@ -1952,9 +2428,109 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): user_api_key_project_id = standard_logging_metadata.get( "user_api_key_project_id" ) - user_api_key_end_user_id = kwargs.get("user") or standard_logging_metadata.get( - "user_api_key_end_user_id" + user_api_key_end_user_id = ( + kwargs.get("user") if isinstance(kwargs, dict) else None + ) or standard_logging_metadata.get("user_api_key_end_user_id") + agent_id = standard_logging_metadata.get("agent_id") + session_id = standard_logging_metadata.get( + "session_id" + ) or standard_logging_metadata.get("trace_id") + + targets: List[Tuple[str, str]] = [] + if user_api_key: + targets.append(("api_key", user_api_key)) + if user_api_key_user_id: + targets.append(("user", user_api_key_user_id)) + if user_api_key_team_id: + targets.append(("team", user_api_key_team_id)) + if user_api_key_team_id and user_api_key_user_id: + targets.append( + ("team_member", f"{user_api_key_team_id}:{user_api_key_user_id}") + ) + if user_api_key_end_user_id: + targets.append(("end_user", user_api_key_end_user_id)) + if user_api_key_organization_id: + targets.append(("organization", user_api_key_organization_id)) + if model_group: + if user_api_key: + targets.append(("model_per_key", f"{user_api_key}:{model_group}")) + if user_api_key_team_id: + targets.append( + ("model_per_team", f"{user_api_key_team_id}:{model_group}") + ) + if user_api_key_organization_id: + targets.append( + ( + "model_per_organization", + f"{user_api_key_organization_id}:{model_group}", + ) + ) + if user_api_key_project_id: + targets.append( + ( + "model_per_project", + f"{user_api_key_project_id}:{model_group}", + ) + ) + if agent_id: + targets.append(("agent", agent_id)) + if session_id: + targets.append(("agent_session", f"{agent_id}:{session_id}")) + return targets + + def _build_reservation_aware_tpm_ops( + self, + targets: List[Tuple[str, str]], + reserved_scopes: Set[Tuple[str, str]], + actual_tokens: int, + reserved_tokens: int, + ) -> List[RedisPipelineIncrementOperation]: + """ + Emit per-scope TPM increment ops with reservation awareness. + + - Reserved scope (counter already at +reserved from pre-call): + reconcile to actual via ``actual - reserved``. + - Unreserved scope (counter never touched at pre-call): + charge the full ``actual``. + + Same primitive serves success reconciliation, over-reservation + release, and failure refund — pass ``actual_tokens=0`` for the pure + refund case (reserved scopes get -reserved, unreserved get 0/skip). + """ + ops: List[RedisPipelineIncrementOperation] = [] + for scope_key, scope_value in targets: + if (scope_key, scope_value) in reserved_scopes: + increment = actual_tokens - reserved_tokens + else: + increment = actual_tokens + if increment == 0: + continue + ops.append( + RedisPipelineIncrementOperation( + key=self.create_rate_limit_keys(scope_key, scope_value, "tokens"), + increment_value=increment, + ttl=self.window_size, + ) + ) + return ops + + def _build_success_event_pipeline_operations( + self, + kwargs: Any, + response_obj: Any, + rate_limit_type: Literal["output", "input", "total"], + ) -> List[RedisPipelineIncrementOperation]: + """Build Redis pipeline increment ops for TPM / parallel-request counters.""" + from litellm.proxy.common_utils.callback_utils import ( + get_model_group_from_litellm_kwargs, ) + + # Get metadata from standard_logging_object - this correctly handles both + # 'metadata' and 'litellm_metadata' fields from litellm_params + standard_logging_object = kwargs.get("standard_logging_object") or {} + standard_logging_metadata = standard_logging_object.get("metadata") or {} + + user_api_key = standard_logging_metadata.get("user_api_key_hash") model_group = get_model_group_from_litellm_kwargs(kwargs) # Get total tokens from response @@ -1968,140 +2544,70 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): usage=_usage, rate_limit_type=rate_limit_type ) + reserved_tokens = self._get_reserved_tokens_from_kwargs( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + reserved_model = self._get_reserved_model_from_kwargs( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + reserved_scopes = self._get_reserved_scopes_from_kwargs( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + # Reconciliation must target the same model-scoped counter that the + # pre-call reservation incremented. If a reservation was made, + # ``reserved_model`` is authoritative; otherwise fall back to the + # router's ``model_group`` (covers the no-reservation charge path). + reconcile_model = reserved_model or model_group + pipeline_operations: List[RedisPipelineIncrementOperation] = [] - # API Key TPM + # max_parallel_requests is its own counter (api-key only) — always decrement. if user_api_key: - # MAX PARALLEL REQUESTS - only support for API Key, just decrement the counter - counter_key = self.create_rate_limit_keys( - key="api_key", - value=user_api_key, - rate_limit_type="max_parallel_requests", - ) pipeline_operations.append( RedisPipelineIncrementOperation( - key=counter_key, + key=self.create_rate_limit_keys( + key="api_key", + value=user_api_key, + rate_limit_type="max_parallel_requests", + ), increment_value=-1, ttl=self.window_size, ) ) - pipeline_operations.extend( - self._create_pipeline_operations( - key="api_key", - value=user_api_key, - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - # User TPM - if user_api_key_user_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="user", - value=user_api_key_user_id, - rate_limit_type="tokens", - total_tokens=total_tokens, - ) + # ---------------------------------------------------------------- + # TPM reconciliation + # Per-scope behavior: + # reserved scope -> apply (actual - reserved) delta to settle + # the counter at +actual. + # unreserved scope -> charge the full actual usage (the + # reservation never incremented this scope). + # When no reservation was made, reserved_tokens=0 and reserved_scopes + # is empty, so every scope falls through the unreserved branch and + # gets the full actual charge — matching pre-PR behavior. + # ---------------------------------------------------------------- + targets = self._collect_tpm_scope_targets( + standard_logging_metadata=standard_logging_metadata, + kwargs=kwargs, + model_group=reconcile_model, + ) + if reserved_tokens > 0 and total_tokens < reserved_tokens: + verbose_proxy_logger.debug( + f"Releasing unused TPM budget on success: " + f"reserved={reserved_tokens}, actual={total_tokens}, " + f"release={reserved_tokens - total_tokens}" ) - - # Team TPM - if user_api_key_team_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="team", - value=user_api_key_team_id, - rate_limit_type="tokens", - total_tokens=total_tokens, - ) + pipeline_operations.extend( + self._build_reservation_aware_tpm_ops( + targets=targets, + reserved_scopes=reserved_scopes, + actual_tokens=total_tokens, + reserved_tokens=reserved_tokens, ) - # Team Member TPM - if user_api_key_team_id and user_api_key_user_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="team_member", - value=f"{user_api_key_team_id}:{user_api_key_user_id}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - # End User TPM - if user_api_key_end_user_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="end_user", - value=user_api_key_end_user_id, - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - # Model-specific TPM - if model_group and user_api_key: - pipeline_operations.extend( - self._create_pipeline_operations( - key="model_per_key", - value=f"{user_api_key}:{model_group}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - if model_group and user_api_key_team_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="model_per_team", - value=f"{user_api_key_team_id}:{model_group}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - if model_group and user_api_key_organization_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="model_per_organization", - value=f"{user_api_key_organization_id}:{model_group}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - if model_group and user_api_key_project_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="model_per_project", - value=f"{user_api_key_project_id}:{model_group}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - # Agent TPM - agent_id = standard_logging_metadata.get("agent_id") - if agent_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="agent", - value=agent_id, - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) - - # Agent Session TPM - session_id = standard_logging_metadata.get( - "session_id" - ) or standard_logging_metadata.get("trace_id") - if session_id: - pipeline_operations.extend( - self._create_pipeline_operations( - key="agent_session", - value=f"{agent_id}:{session_id}", - rate_limit_type="tokens", - total_tokens=total_tokens, - ) - ) + ) return pipeline_operations @@ -2142,19 +2648,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): """ - Decrement max parallel requests counter for the API Key + On failure: decrement max_parallel_requests and refund the upfront + TPM reservation only against the scopes the reservation actually + charged. Unreserved scopes were never incremented at pre-call, so + refunding them would drive their counter negative. """ from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, ) - from litellm.types.caching import RedisPipelineIncrementOperation try: litellm_parent_otel_span: Union[Span, None] = ( _get_parent_otel_span_from_kwargs(kwargs) ) - # Get metadata from standard_logging_object - this correctly handles both - # 'metadata' and 'litellm_metadata' fields from litellm_params standard_logging_object = kwargs.get("standard_logging_object") or {} standard_logging_metadata = standard_logging_object.get("metadata") or {} user_api_key = standard_logging_metadata.get("user_api_key_hash") @@ -2162,26 +2668,65 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations: List[RedisPipelineIncrementOperation] = [] if user_api_key: - # MAX PARALLEL REQUESTS - only support for API Key, just decrement the counter - counter_key = self.create_rate_limit_keys( - key="api_key", - value=user_api_key, - rate_limit_type="max_parallel_requests", - ) pipeline_operations.append( RedisPipelineIncrementOperation( - key=counter_key, + key=self.create_rate_limit_keys( + key="api_key", + value=user_api_key, + rate_limit_type="max_parallel_requests", + ), increment_value=-1, ttl=self.window_size, ) ) - # Execute all increments in a single pipeline + # Skip the reservation refund if async_post_call_failure_hook + # already released it (proxy-level rejection that also bubbles up + # here as an LLM-error callback). max_parallel_requests is its + # own counter and is always decremented per call. + already_released = self._is_reservation_released( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + reserved_tokens = ( + 0 + if already_released + else self._get_reserved_tokens_from_kwargs( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + ) + if reserved_tokens > 0: + verbose_proxy_logger.debug( + f"Releasing reserved TPM tokens on failure: {reserved_tokens}" + ) + # Refund only against the scopes the reservation actually + # charged. _build_reservation_aware_tpm_ops with + # actual_tokens=0 emits -reserved on reserved scopes and 0 + # on unreserved (skipped), so unreserved scopes can't drift + # negative. Targets are derived purely from the reserved + # set so we don't even need to re-collect them from + # metadata. + reserved_scopes = self._get_reserved_scopes_from_kwargs( + kwargs=kwargs, + standard_logging_metadata=standard_logging_metadata, + ) + pipeline_operations.extend( + self._build_reservation_aware_tpm_ops( + targets=list(reserved_scopes), + reserved_scopes=reserved_scopes, + actual_tokens=0, + reserved_tokens=reserved_tokens, + ) + ) + if pipeline_operations: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=pipeline_operations, litellm_parent_otel_span=litellm_parent_otel_span, ) + if reserved_tokens > 0: + self._mark_reservation_released(kwargs) except Exception as e: verbose_proxy_logger.exception( f"Error in rate limit failure event: {str(e)}" @@ -2239,3 +2784,67 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): verbose_proxy_logger.exception( f"Error in rate limit post-call hook: {str(e)}" ) + + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: Optional[str] = None, + ) -> None: + """ + Release any TPM reservation when the request is rejected after the + pre-call hook reserved tokens but before the LLM call ran (e.g. a + downstream guardrail/auth hook raised). Without this, those + reservations are stranded — async_log_failure_event is a litellm + completion-level callback and never fires for proxy-side rejections. + + Idempotent via TPM_RESERVATION_RELEASED_KEY: if both this hook and + async_log_failure_event end up running in the same flow, only the + first refund applies. + """ + try: + if self._is_reservation_released(kwargs=request_data): + return + reserved_tokens = self._get_reserved_tokens_from_kwargs(kwargs=request_data) + if reserved_tokens <= 0: + return + + # Refund directly against the descriptors we reserved against — + # the pre-call hook stashes them on the request data before + # success/failure callbacks run. + stashed = request_data.get("_litellm_rate_limit_descriptors") + descriptors: List[RateLimitDescriptor] = ( + stashed if isinstance(stashed, list) else [] + ) + ops: List[RedisPipelineIncrementOperation] = [] + for descriptor in descriptors: + rate_limit = descriptor.get("rate_limit") or {} + if rate_limit.get("tokens_per_unit") is None: + continue + ops.append( + RedisPipelineIncrementOperation( + key=self.create_rate_limit_keys( + descriptor["key"], + descriptor["value"], + "tokens", + ), + increment_value=-reserved_tokens, + ttl=self.window_size, + ) + ) + if ops: + verbose_proxy_logger.debug( + f"Releasing reserved TPM tokens on proxy-level " + f"rejection: {reserved_tokens}" + ) + await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( + increment_list=ops, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + ) + self._mark_reservation_released(request_data) + except Exception as e: + verbose_proxy_logger.exception( + f"Error releasing TPM reservation on post-call failure: {e}" + ) + return None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 49230c65ec2..a63613c5836 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1667,7 +1667,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) if tags is not None and _admin_allow_client_tags: - data[_metadata_variable_name]["tags"] = tags + data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( + request_tags=data[_metadata_variable_name].get("tags"), + tags_to_add=tags, + ) elif tags is not None: verbose_proxy_logger.warning( "Ignored caller-supplied tags from header/root body: this " diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index ac66adc26f3..d173cd745ba 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -1,3 +1,4 @@ +import asyncio from datetime import datetime, timedelta from types import SimpleNamespace from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union @@ -543,6 +544,13 @@ def _build_aggregated_sql_query( where_clause = " AND ".join(sql_conditions) + # Postgres computes every rollup level the response needs — per-date + # totals, per-(date, model), per-(date, model, api_key), per-provider, + # etc. — in a single pass via GROUPING SETS. The GROUPING() bitmask + # encodes which level a row belongs to so Python can dispatch rows + # straight into their buckets without re-summing. The leaf grouping + # is omitted on purpose: nothing in the response shape needs it once + # all the rollups are present. sql_query = f""" SELECT date, @@ -552,6 +560,9 @@ def _build_aggregated_sql_query( custom_llm_provider, mcp_namespaced_tool_name, endpoint, + GROUPING(date, api_key, model, model_group, + custom_llm_provider, mcp_namespaced_tool_name, + endpoint) AS group_level, SUM(spend)::float AS spend, SUM(prompt_tokens)::bigint AS prompt_tokens, SUM(completion_tokens)::bigint AS completion_tokens, @@ -562,32 +573,35 @@ def _build_aggregated_sql_query( SUM(failed_requests)::bigint AS failed_requests FROM "{pg_table}" WHERE {where_clause} - GROUP BY date, api_key, model, model_group, custom_llm_provider, - mcp_namespaced_tool_name, endpoint - ORDER BY date DESC + GROUP BY GROUPING SETS ( + (date), + (date, api_key), + (date, model), + (date, model, api_key), + (date, model_group), + (date, model_group, api_key), + (date, custom_llm_provider), + (date, custom_llm_provider, api_key), + (date, mcp_namespaced_tool_name), + (date, mcp_namespaced_tool_name, api_key), + (date, endpoint), + (date, endpoint, api_key), + () + ) """ return sql_query, sql_params -async def _aggregate_spend_records( +def _aggregate_spend_records_sync( *, - prisma_client: PrismaClient, records: List[Any], + api_key_metadata: Dict[str, Dict[str, Any]], entity_id_field: Optional[str], entity_metadata_field: Optional[Dict[str, dict]], ) -> Dict[str, Any]: - """Aggregate rows into DailySpendData list and total metrics.""" - api_keys: Set[str] = set() - for record in records: - if record.api_key: - api_keys.add(record.api_key) - - api_key_metadata: Dict[str, Dict[str, Any]] = {} model_metadata: Dict[str, Dict[str, Any]] = {} provider_metadata: Dict[str, Dict[str, Any]] = {} - if api_keys: - api_key_metadata = await get_api_key_metadata(prisma_client, api_keys) results: List[DailySpendData] = [] total_metrics = SpendMetrics() @@ -631,6 +645,228 @@ async def _aggregate_spend_records( return {"results": results, "totals": total_metrics} +async def _aggregate_spend_records( + *, + prisma_client: PrismaClient, + records: List[Any], + entity_id_field: Optional[str], + entity_metadata_field: Optional[Dict[str, dict]], +) -> Dict[str, Any]: + """Aggregate rows into DailySpendData list and total metrics. + + The per-row loop is offloaded to a worker thread via asyncio.to_thread so + a large result set doesn't peg the event loop. + """ + api_keys: Set[str] = {record.api_key for record in records if record.api_key} + + api_key_metadata: Dict[str, Dict[str, Any]] = {} + if api_keys: + api_key_metadata = await get_api_key_metadata(prisma_client, api_keys) + + return await asyncio.to_thread( + _aggregate_spend_records_sync, + records=records, + api_key_metadata=api_key_metadata, + entity_id_field=entity_id_field, + entity_metadata_field=entity_metadata_field, + ) + + +# GROUPING() bitmask values for each grouping set emitted by +# _build_aggregated_sql_query. Per Postgres semantics, the rightmost argument +# is the least-significant bit. Argument order: +# date, api_key, model, model_group, custom_llm_provider, +# mcp_namespaced_tool_name, endpoint +# A bit is 1 when the corresponding column is rolled up (i.e. NOT in the +# current grouping set's key), 0 when the column is part of the key. +_GROUP_GRAND_TOTAL = 127 # 0b1111111 — all rolled up +_GROUP_DATE = 63 # 0b0111111 — only date kept +_GROUP_DATE_API_KEY = 31 # 0b0011111 +_GROUP_DATE_MODEL = 47 # 0b0101111 +_GROUP_DATE_MODEL_API_KEY = 15 # 0b0001111 +_GROUP_DATE_MODEL_GROUP = 55 # 0b0110111 +_GROUP_DATE_MODEL_GROUP_API_KEY = 23 # 0b0010111 +_GROUP_DATE_PROVIDER = 59 # 0b0111011 +_GROUP_DATE_PROVIDER_API_KEY = 27 # 0b0011011 +_GROUP_DATE_MCP = 61 # 0b0111101 +_GROUP_DATE_MCP_API_KEY = 29 # 0b0011101 +_GROUP_DATE_ENDPOINT = 62 # 0b0111110 +_GROUP_DATE_ENDPOINT_API_KEY = 30 # 0b0011110 + + +def _record_to_spend_metrics(record: Any) -> SpendMetrics: + """Build a SpendMetrics directly from one already-aggregated rollup row.""" + return SpendMetrics( + spend=record.spend, + prompt_tokens=record.prompt_tokens, + completion_tokens=record.completion_tokens, + total_tokens=record.prompt_tokens + record.completion_tokens, + cache_read_input_tokens=record.cache_read_input_tokens, + cache_creation_input_tokens=record.cache_creation_input_tokens, + api_requests=record.api_requests, + successful_requests=record.successful_requests, + failed_requests=record.failed_requests, + ) + + +def _key_metadata( + api_key_metadata: Dict[str, Dict[str, Any]], api_key: str +) -> KeyMetadata: + meta = api_key_metadata.get(api_key, {}) + return KeyMetadata(key_alias=meta.get("key_alias"), team_id=meta.get("team_id")) + + +def _aggregate_grouping_sets_records_sync( # noqa: PLR0915 + *, + records: List[Any], + api_key_metadata: Dict[str, Dict[str, Any]], +) -> Dict[str, Any]: + """Build the response from rollup rows produced by the GROUPING SETS query. + + Each row carries a `group_level` bitmask (from Postgres GROUPING()) that + identifies which rollup level it belongs to. We dispatch the row's + pre-aggregated metrics straight into the matching bucket — no per-row + summing in Python and no nested update_metrics calls. + """ + total_metrics = SpendMetrics() + grouped_data: Dict[str, Dict[str, Any]] = {} + + def ensure_date(date_str: str) -> Dict[str, Any]: + bucket = grouped_data.get(date_str) + if bucket is None: + bucket = {"metrics": SpendMetrics(), "breakdown": BreakdownMetrics()} + grouped_data[date_str] = bucket + return bucket + + def assign_metric_with_metadata( + target: Dict[str, MetricWithMetadata], key: str, metrics: SpendMetrics + ) -> None: + existing = target.get(key) + if existing is None: + target[key] = MetricWithMetadata(metrics=metrics, metadata={}) + else: + existing.metrics = metrics + + def assign_api_key_breakdown( + target: Dict[str, MetricWithMetadata], + parent_key: str, + api_key: str, + metrics: SpendMetrics, + ) -> None: + parent = target.get(parent_key) + if parent is None: + parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={}) + target[parent_key] = parent + parent.api_key_breakdown[api_key] = KeyMetricWithMetadata( + metrics=metrics, metadata=_key_metadata(api_key_metadata, api_key) + ) + + for record in records: + level = record.group_level + metrics = _record_to_spend_metrics(record) + + if level == _GROUP_GRAND_TOTAL: + total_metrics = metrics + continue + + if level == _GROUP_DATE: + ensure_date(record.date)["metrics"] = metrics + continue + + breakdown = ensure_date(record.date)["breakdown"] + + if level == _GROUP_DATE_API_KEY: + if record.api_key: + breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( + metrics=metrics, + metadata=_key_metadata(api_key_metadata, record.api_key), + ) + elif level == _GROUP_DATE_MODEL: + if record.model: + assign_metric_with_metadata(breakdown.models, record.model, metrics) + elif level == _GROUP_DATE_MODEL_API_KEY: + if record.model and record.api_key: + assign_api_key_breakdown( + breakdown.models, record.model, record.api_key, metrics + ) + elif level == _GROUP_DATE_MODEL_GROUP: + if record.model_group: + assign_metric_with_metadata( + breakdown.model_groups, record.model_group, metrics + ) + elif level == _GROUP_DATE_MODEL_GROUP_API_KEY: + if record.model_group and record.api_key: + assign_api_key_breakdown( + breakdown.model_groups, + record.model_group, + record.api_key, + metrics, + ) + elif level == _GROUP_DATE_PROVIDER: + provider = record.custom_llm_provider or "unknown" + assign_metric_with_metadata(breakdown.providers, provider, metrics) + elif level == _GROUP_DATE_PROVIDER_API_KEY: + if record.api_key: + provider = record.custom_llm_provider or "unknown" + assign_api_key_breakdown( + breakdown.providers, provider, record.api_key, metrics + ) + elif level == _GROUP_DATE_MCP: + if record.mcp_namespaced_tool_name: + assign_metric_with_metadata( + breakdown.mcp_servers, record.mcp_namespaced_tool_name, metrics + ) + elif level == _GROUP_DATE_MCP_API_KEY: + if record.mcp_namespaced_tool_name and record.api_key: + assign_api_key_breakdown( + breakdown.mcp_servers, + record.mcp_namespaced_tool_name, + record.api_key, + metrics, + ) + elif level == _GROUP_DATE_ENDPOINT: + if record.endpoint: + assign_metric_with_metadata( + breakdown.endpoints, record.endpoint, metrics + ) + elif level == _GROUP_DATE_ENDPOINT_API_KEY: + if record.endpoint and record.api_key: + assign_api_key_breakdown( + breakdown.endpoints, record.endpoint, record.api_key, metrics + ) + + results = [ + DailySpendData( + date=datetime.strptime(date_str, "%Y-%m-%d").date(), + metrics=data["metrics"], + breakdown=data["breakdown"], + ) + for date_str, data in grouped_data.items() + ] + results.sort(key=lambda x: x.date, reverse=True) + + return {"results": results, "totals": total_metrics} + + +async def _aggregate_grouping_sets_records( + *, + prisma_client: PrismaClient, + records: List[Any], +) -> Dict[str, Any]: + """Async wrapper: fetch api_key_metadata, then dispatch on a worker thread.""" + api_keys: Set[str] = {r.api_key for r in records if r.api_key} + + api_key_metadata: Dict[str, Dict[str, Any]] = {} + if api_keys: + api_key_metadata = await get_api_key_metadata(prisma_client, api_keys) + + return await asyncio.to_thread( + _aggregate_grouping_sets_records_sync, + records=records, + api_key_metadata=api_key_metadata, + ) + + async def get_daily_activity( prisma_client: Optional[PrismaClient], table_name: str, @@ -771,21 +1007,18 @@ async def get_daily_activity_aggregated( timezone_offset_minutes=timezone_offset_minutes, ) - # Execute GROUP BY query — returns pre-aggregated dicts + # Execute GROUPING SETS query — returns one row per rollup level. rows = await prisma_client.db.query_raw(sql_query, *sql_params) if rows is None: rows = [] - # Convert dicts to objects for compatibility with _aggregate_spend_records records = [SimpleNamespace(**row) for row in rows] - # entity_id_field=None skips entity breakdown (entity dimension was - # collapsed by the GROUP BY, so per-entity data is not available) - aggregated = await _aggregate_spend_records( + # The grouping-sets dispatcher places each row directly in its bucket + # using the row's GROUPING() bitmask. No Python-side summing needed. + aggregated = await _aggregate_grouping_sets_records( prisma_client=prisma_client, records=records, - entity_id_field=None, - entity_metadata_field=None, ) return SpendAnalyticsPaginatedResponse( diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index c62e40e90d4..7bda0f87ccd 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -152,8 +152,14 @@ if MCP_AVAILABLE: UserAPIKeyAuth, UserMCPManagementMode, ) - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - from litellm.proxy.common_utils.http_parsing_utils import _read_request_body + from litellm.proxy.auth.user_api_key_auth import ( + _user_api_key_auth_builder, + user_api_key_auth, + ) + from litellm.proxy.common_utils.http_parsing_utils import ( + _read_request_body, + populate_request_with_path_params, + ) from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.types.mcp import MCPCredentials @@ -1492,6 +1498,55 @@ if MCP_AVAILABLE: return _redact_mcp_credentials(temp_record) + async def _mcp_oauth_user_api_key_auth(request: Request) -> UserAPIKeyAuth: + """ + Auth dependency for MCP OAuth browser-navigation endpoints (/authorize, /token). + + Tries the Authorization header first. Falls back to decoding the UI + 'token' session cookie (set by SSO login) to extract the API key, which + allows browser-based OAuth redirects to work without an explicit + Authorization header. + """ + import jwt as _jwt + + from litellm.proxy.proxy_server import master_key + + auth_header = request.headers.get("Authorization", "") + api_key = auth_header # _get_bearer_token will strip "Bearer " prefix + + if not api_key: + token_cookie = request.cookies.get("token") + if token_cookie and master_key: + try: + decoded = _jwt.decode( + token_cookie, + master_key, + algorithms=["HS256"], + # UI session cookies may omit exp; don't require it. + options={"verify_exp": False}, + ) + if decoded.get("login_method") in ("sso", "username_password"): + cookie_key = decoded.get("key", "") + if cookie_key: + api_key = f"Bearer {cookie_key}" + except _jwt.InvalidTokenError: + pass + + request_data = await _read_request_body(request=request) + request_data = populate_request_with_path_params( + request_data=request_data, request=request + ) + + return await _user_api_key_auth_builder( + request=request, + api_key=api_key, + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data=request_data, + ) + async def _get_cached_temporary_mcp_server_or_404( server_id: str, user_api_key_dict: UserAPIKeyAuth, @@ -1542,12 +1597,12 @@ if MCP_AVAILABLE: @router.get( "/server/oauth/{server_id}/authorize", include_in_schema=False, - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(_mcp_oauth_user_api_key_auth)], ) async def mcp_authorize( request: Request, server_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + user_api_key_dict: UserAPIKeyAuth = Depends(_mcp_oauth_user_api_key_auth), client_id: Optional[str] = None, redirect_uri: str = Query(...), state: str = "", @@ -1587,12 +1642,12 @@ if MCP_AVAILABLE: @router.post( "/server/oauth/{server_id}/token", include_in_schema=False, - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(_mcp_oauth_user_api_key_auth)], ) async def mcp_token( request: Request, server_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + user_api_key_dict: UserAPIKeyAuth = Depends(_mcp_oauth_user_api_key_auth), grant_type: str = Form(...), code: Optional[str] = Form(None), redirect_uri: Optional[str] = Form(None), diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index 04699f19ffb..1f20764f837 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -4,6 +4,7 @@ This is an enterprise feature and requires a premium license. """ +import re from typing import Any, Dict, List, Optional, Set, Tuple from fastapi import ( @@ -843,6 +844,18 @@ async def get_service_provider_config(request: Request): return SCIMServiceProviderConfig(meta=meta) +def _parse_scim_eq_filter(scim_filter: str) -> Optional[Tuple[str, str]]: + """Parse the SCIM equality filters Okta uses before user lifecycle changes.""" + match = re.match( + r"""\s*([\w.]+)\s+eq\s+(['"]?)(.*?)\2\s*$""", + scim_filter, + flags=re.IGNORECASE, + ) + if not match: + return None + return match.group(1).lower(), match.group(3) + + # User Endpoints @scim_router.get( "/Users", @@ -867,15 +880,21 @@ async def get_users( try: prisma_client = await _get_prisma_client_or_raise_exception() # Parse filter if provided (basic support) - where_conditions = {} + where_conditions: Dict[str, Any] = {} if filter: - # Very basic filter support - only handling userName eq and emails.value eq - if "userName eq" in filter: - user_id = filter.split("userName eq ")[1].strip("\"'") - where_conditions["user_id"] = user_id - elif "emails.value eq" in filter: - email = filter.split("emails.value eq ")[1].strip("\"'") - where_conditions["user_email"] = email + # Okta locates users by userName before deprovisioning. LiteLLM + # exposes SCIM userName from user_email, while older SCIM-created + # users may still have user_id == userName, so support both. + parsed_filter = _parse_scim_eq_filter(filter) + if parsed_filter: + filter_attribute, filter_value = parsed_filter + if filter_attribute == "username": + where_conditions["OR"] = [ + {"user_email": filter_value}, + {"user_id": filter_value}, + ] + elif filter_attribute == "emails.value": + where_conditions["user_email"] = filter_value # Get users from database users: List[LiteLLM_UserTable] = ( diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index d3b225e6e4d..439c3b2118d 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -46,25 +46,46 @@ def get_audit_log_changed_by( def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]: - """Resolve a string callback name to a CustomLogger instance, with caching.""" + """Resolve a string callback name to a CustomLogger instance, with caching. + + For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a + dedicated `S3Logger` so audit logs can target a different bucket than the + normal-log singleton served by `_init_custom_logger_compatible_class`. + """ if name in _audit_log_callback_cache: return _audit_log_callback_cache[name] - from litellm.litellm_core_utils.litellm_logging import ( - _init_custom_logger_compatible_class, - ) + instance: Optional[CustomLogger] + if ( + name == "s3_v2" + and getattr(litellm, "s3_audit_callback_params", None) is not None + ): + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger - instance = _init_custom_logger_compatible_class( - logging_integration=name, # type: ignore - internal_usage_cache=None, - llm_router=None, - ) + instance = S3V2Logger( + s3_callback_params_override=litellm.s3_audit_callback_params + ) + else: + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + + instance = _init_custom_logger_compatible_class( + logging_integration=name, # type: ignore + internal_usage_cache=None, + llm_router=None, + ) if instance is not None: _audit_log_callback_cache[name] = instance return instance +def reset_audit_log_callback_cache() -> None: + """Clear cached audit-log callback instances. Call on config reload.""" + _audit_log_callback_cache.clear() + + def _build_audit_log_payload( request_data: LiteLLM_AuditLogs, ) -> StandardAuditLogPayload: diff --git a/litellm/proxy/middleware/request_size_limit_middleware.py b/litellm/proxy/middleware/request_size_limit_middleware.py new file mode 100644 index 00000000000..78a38e3572e --- /dev/null +++ b/litellm/proxy/middleware/request_size_limit_middleware.py @@ -0,0 +1,121 @@ +import json +from typing import Callable, Optional, Union + +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +MaxRequestSizeGetter = Callable[[], Optional[Union[int, float]]] +RequestSizeLimitEnabledGetter = Callable[[], bool] + + +class RequestEntityTooLarge(Exception): + pass + + +class RequestSizeLimitMiddleware: + """ + Reject oversized requests before downstream auth/routes parse the body. + + Content-Length can be rejected without reading any body bytes. Requests + without Content-Length are counted as the ASGI stream is consumed, limiting + memory exposure to the configured threshold plus the current chunk. + """ + + def __init__( + self, + app: ASGIApp, + get_max_request_size_mb: MaxRequestSizeGetter, + is_request_size_limit_enabled: RequestSizeLimitEnabledGetter, + ) -> None: + self.app = app + self.get_max_request_size_mb = get_max_request_size_mb + self.is_request_size_limit_enabled = is_request_size_limit_enabled + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + max_request_size_mb = self.get_max_request_size_mb() + max_request_size_bytes = _mb_to_bytes(max_request_size_mb) + if max_request_size_bytes is None or not self.is_request_size_limit_enabled(): + await self.app(scope, receive, send) + return + + content_length = _get_content_length(scope=scope) + if content_length is not None and content_length > max_request_size_bytes: + await _send_request_too_large( + send=send, max_request_size_mb=max_request_size_mb + ) + return + + received_body_bytes = 0 + response_started = False + + async def limited_receive() -> Message: + nonlocal received_body_bytes + + message = await receive() + if message["type"] != "http.request": + return message + + received_body_bytes += len(message.get("body", b"")) + if received_body_bytes > max_request_size_bytes: + raise RequestEntityTooLarge + return message + + async def tracking_send(message: Message) -> None: + nonlocal response_started + + if message["type"] == "http.response.start": + response_started = True + await send(message) + + try: + await self.app(scope, limited_receive, tracking_send) + except RequestEntityTooLarge: + if response_started: + raise + await _send_request_too_large( + send=send, max_request_size_mb=max_request_size_mb + ) + + +def _mb_to_bytes(max_request_size_mb: Optional[Union[int, float]]) -> Optional[int]: + if max_request_size_mb is None: + return None + if max_request_size_mb <= 0: + return None + return int(max_request_size_mb * 1024 * 1024) + + +def _get_content_length(scope: Scope) -> Optional[int]: + headers = dict(scope.get("headers") or []) + raw_content_length = headers.get(b"content-length") + if raw_content_length is None: + return None + + try: + return int(raw_content_length) + except ValueError: + return None + + +async def _send_request_too_large( + send: Send, + max_request_size_mb: Optional[Union[int, float]], +) -> None: + body = json.dumps( + {"error": f"Request size is too large. Max size is {max_request_size_mb} MB"}, + separators=(",", ":"), + ).encode("utf-8") + await send( + { + "type": "http.response.start", + "status": 413, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode("latin-1")), + ], + } + ) + await send({"type": "http.response.body", "body": body, "more_body": False}) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 71aeea67884..06fc0819a76 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -169,6 +169,66 @@ class ProxyInitializationHelpers: ) return uvicorn_args + @staticmethod + def _get_reload_options(config_path: Optional[str]) -> dict: + """Build uvicorn reload kwargs so --reload also reacts to YAML edits.""" + options: dict = {"reload": True} + if not config_path: + return options + config_abs = os.path.abspath(config_path) + config_dir = os.path.dirname(config_abs) + cwd = os.path.abspath(os.getcwd()) + reload_dirs = [cwd] + if config_dir and config_dir != cwd: + reload_dirs.append(config_dir) + options["reload_dirs"] = reload_dirs + # Must be a basename, not an absolute path: uvicorn's + # resolve_reload_patterns() calls pathlib.Path.glob(), which raises + # NotImplementedError on absolute patterns (uvicorn discussion #2156). + options["reload_includes"] = ["*.py", os.path.basename(config_abs)] + return options + + @staticmethod + def _patch_statreload_for_config(config_path: str) -> bool: + """Make uvicorn's StatReload reloader notice YAML config changes. + + Uvicorn uses WatchFilesReload when the optional `watchfiles` package + is installed, otherwise StatReload. StatReload hard-codes `*.py` in + `iter_py_files()` and silently ignores `reload_includes`, so the + kwargs from `_get_reload_options` alone don't trigger reloads on YAML + edits. We monkey-patch `iter_py_files` to also yield the config path. + + Idempotent across calls and a no-op for the WatchFilesReload path. + """ + try: + from uvicorn.supervisors.statreload import StatReload + except ImportError: # pragma: no cover - uvicorn is a hard dep + return False + + if not config_path: + return False + + from pathlib import Path + + config_abs = Path(config_path).resolve() + + patched_paths = getattr(StatReload, "_litellm_patched_config_paths", None) + if patched_paths is None: + original_iter = StatReload.iter_py_files + patched_paths = set() + + def _iter_with_config(self): # type: ignore[no-untyped-def] + yield from original_iter(self) + for path in StatReload._litellm_patched_config_paths: + if path.exists(): + yield path + + StatReload.iter_py_files = _iter_with_config # type: ignore[assignment] + StatReload._litellm_patched_config_paths = patched_paths # type: ignore[attr-defined] + + patched_paths.add(config_abs) + return True + @staticmethod def _init_hypercorn_server( app: FastAPI, @@ -619,7 +679,7 @@ class ProxyInitializationHelpers: "--reload", is_flag=True, default=False, - help="Enable uvicorn hot reload (dev only). Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.", + help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.", ) def run_server( # noqa: PLR0915 host, @@ -990,11 +1050,6 @@ def run_server( # noqa: PLR0915 litellm_settings=litellm_settings if config else None, # type: ignore[possibly-unbound] ) - # --- SEPARATE HEALTH APP LOGIC --- - # To run the health app separately, use: - # uvicorn litellm.proxy.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=4001 - # This is compatible with the SEPARATE_HEALTH_APP Docker/supervisord pattern. - # --- END SEPARATE HEALTH APP LOGIC --- # Skip server startup if requested (after all setup is done) if skip_server_startup: print( # noqa @@ -1028,7 +1083,11 @@ def run_server( # noqa: PLR0915 uvicorn_args["loop"] = loop_type if reload: - uvicorn_args["reload"] = True + uvicorn_args.update( + ProxyInitializationHelpers._get_reload_options(config) + ) + if config: + ProxyInitializationHelpers._patch_statreload_for_config(config) uvicorn.run( **uvicorn_args, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a5905765c6e..c96d0acb008 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -403,6 +403,9 @@ from litellm.proxy.middleware.in_flight_requests_middleware import ( InFlightRequestsMiddleware, ) from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware +from litellm.proxy.middleware.request_size_limit_middleware import ( + RequestSizeLimitMiddleware, +) from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, @@ -3820,6 +3823,11 @@ class ProxyConfig: f"{blue_color_code} Initialized Failure Callbacks - {litellm.failure_callback} {reset_color_code}" ) # noqa elif key == "audit_log_callbacks": + from litellm.proxy.management_helpers.audit_logs import ( + reset_audit_log_callback_cache, + ) + + reset_audit_log_callback_cache() litellm.audit_log_callbacks = [] for callback in value: @@ -3901,6 +3909,21 @@ class ProxyConfig: f"{blue_color_code} setting litellm.{key}={value}{reset_color_code}" ) setattr(litellm, key, value) + if key in {"s3_audit_callback_params", "s3_callback_params"}: + from litellm.proxy.management_helpers.audit_logs import ( + reset_audit_log_callback_cache, + ) + from litellm.litellm_core_utils.litellm_logging import ( + _in_memory_loggers, + ) + from litellm.integrations.s3_v2 import S3Logger as S3V2Logger + + reset_audit_log_callback_cache() + _in_memory_loggers[:] = [ + cb + for cb in _in_memory_loggers + if not isinstance(cb, S3V2Logger) + ] ## GENERAL SERVER SETTINGS (e.g. master key,..) # do this after initializing litellm, to ensure sentry logging works for proxylogging general_settings = config.get("general_settings", {}) @@ -14881,6 +14904,11 @@ app.include_router(ui_discovery_endpoints_router) app.include_router(google_router) attach_lazy_features(app) +app.add_middleware( + RequestSizeLimitMiddleware, + get_max_request_size_mb=lambda: general_settings.get("max_request_size_mb"), + is_request_size_limit_enabled=lambda: premium_user is True, +) async def _stream_mcp_asgi_response( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e7f5f4ee396..a52dc8e55fb 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2274,6 +2274,13 @@ class ProxyLogging: Covers: 1. /chat/completions """ + from litellm.proxy.proxy_server import llm_router + + # Merge model-level guardrails before checking which guardrails to run + request_data = _check_and_merge_model_level_guardrails( + data=request_data, llm_router=llm_router + ) + current_response = response for callback in litellm.callbacks: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 986ec39f3bb..abe58199dfd 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1878,14 +1878,24 @@ class OpenAIRealtimeStreamSessionEvents(TypedDict): class OpenAIRealtimeStreamResponseOutputItemContent(TypedDict, total=False): audio: str - """Base64-encoded audio bytes, used for 'input_audio' content types""" + """Base64-encoded audio bytes, used for 'input_audio' / 'audio' / 'output_audio' content types""" id: str """The ID of the previous conversation item for reference""" text: str - """The text content, used for 'input_text' and 'text' content types""" + """The text content, used for 'input_text' / 'text' / 'output_text' content types""" transcript: str - """The transcript content, used for 'input_audio' content types""" - type: Literal["input_audio", "input_text", "text", "item_reference", "audio"] + """The transcript content, used for 'input_audio' / 'audio' content types""" + type: Literal[ + "input_audio", + "input_text", + # Beta assistant content types + "text", + "audio", + "item_reference", + # GA assistant content types (aligns with Responses API) + "output_text", + "output_audio", + ] """The type of content""" @@ -1945,23 +1955,46 @@ class OpenAIRealtimeConversationCreated(TypedDict, total=False): class OpenAIRealtimeConversationItemCreated(TypedDict, total=False): + """Beta: single event emitted when a conversation item is created.""" + type: Required[Literal["conversation.item.created"]] item: OpenAIRealtimeStreamResponseOutputItem event_id: str - previous_item_id: str + previous_item_id: Optional[str] # None when this is the first item + + +class OpenAIRealtimeConversationItemAdded(TypedDict, total=False): + """GA: emitted immediately when a conversation item is added (replaces .created).""" + + type: Required[Literal["conversation.item.added"]] + item: OpenAIRealtimeStreamResponseOutputItem + event_id: str + previous_item_id: Optional[str] # None when this is the first item + + +class OpenAIRealtimeConversationItemDone(TypedDict, total=False): + """GA: emitted when a conversation item is fully complete (e.g. transcription done).""" + + type: Required[Literal["conversation.item.done"]] + item: OpenAIRealtimeStreamResponseOutputItem + event_id: str + previous_item_id: Optional[str] # None when this is the first item class OpenAIRealtimeResponseContentPart(TypedDict, total=False): audio: str - """Base64-encoded audio bytes, if type is 'audio'""" + """Base64-encoded audio bytes, if type is 'audio' or 'output_audio'""" text: str - """The text content, if type is 'text'""" + """The text content, if type is 'text' or 'output_text'""" transcript: str - """The transcript content, if type is 'audio'""" + """The transcript content, if type is 'audio' or 'output_audio'""" - type: Literal["audio", "text"] + type: Union[ + Literal["audio", "text"], # beta + Literal["output_audio", "output_text"], # GA + ] """The type of content""" @@ -1982,7 +2015,14 @@ class OpenAIRealtimeResponseDelta(TypedDict): item_id: str output_index: int response_id: str - type: Union[Literal["response.text.delta"], Literal["response.audio.delta"]] + type: Union[ + Literal["response.text.delta"], + Literal["response.audio.delta"], + # GA renamed events + Literal["response.output_text.delta"], + Literal["response.output_audio.delta"], + Literal["response.output_audio_transcript.delta"], + ] class OpenAIRealtimeResponseTextDone(TypedDict): @@ -1992,7 +2032,10 @@ class OpenAIRealtimeResponseTextDone(TypedDict): output_index: int response_id: str text: str - type: Literal["response.text.done"] + type: Union[ + Literal["response.text.done"], + Literal["response.output_text.done"], # GA rename + ] class OpenAIRealtimeResponseAudioDone(TypedDict): @@ -2001,7 +2044,11 @@ class OpenAIRealtimeResponseAudioDone(TypedDict): item_id: str output_index: int response_id: str - type: Literal["response.audio.done"] + type: Union[ + Literal["response.audio.done"], + Literal["response.output_audio.done"], # GA rename + Literal["response.output_audio_transcript.done"], # GA rename + ] class OpenAIRealtimeContentPartDone(TypedDict): @@ -2046,10 +2093,18 @@ class OpenAIRealtimeDoneEvent(TypedDict): class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" + # Beta delta event names RESPONSE_TEXT_DELTA = "response.text.delta" RESPONSE_AUDIO_DELTA = "response.audio.delta" RESPONSE_TEXT_DONE = "response.text.done" RESPONSE_AUDIO_DONE = "response.audio.done" + # GA renamed delta event names + RESPONSE_OUTPUT_TEXT_DELTA = "response.output_text.delta" + RESPONSE_OUTPUT_AUDIO_DELTA = "response.output_audio.delta" + RESPONSE_OUTPUT_AUDIO_TRANSCRIPT_DELTA = "response.output_audio_transcript.delta" + RESPONSE_OUTPUT_TEXT_DONE = "response.output_text.done" + RESPONSE_OUTPUT_AUDIO_DONE = "response.output_audio.done" + RESPONSE_OUTPUT_AUDIO_TRANSCRIPT_DONE = "response.output_audio_transcript.done" RESPONSE_DONE = "response.done" RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added" RESPONSE_CONTENT_PART_ADDED = "response.content_part.added" @@ -2060,7 +2115,11 @@ OpenAIRealtimeEvents = Union[ OpenAIRealtimeStreamSessionEvents, OpenAIRealtimeStreamResponseOutputItemAdded, OpenAIRealtimeResponseContentPartAdded, + # Beta conversation item event OpenAIRealtimeConversationItemCreated, + # GA conversation item events + OpenAIRealtimeConversationItemAdded, + OpenAIRealtimeConversationItemDone, OpenAIRealtimeConversationCreated, OpenAIRealtimeResponseDelta, OpenAIRealtimeResponseTextDone, diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index ebabf3fb6f8..21d4da82041 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -37,6 +37,7 @@ class MCPAuth(str, enum.Enum): oauth2 = "oauth2" aws_sigv4 = "aws_sigv4" token = "token" + oauth2_token_exchange = "oauth2_token_exchange" # MCP Literals @@ -54,6 +55,7 @@ MCPAuthType = Optional[ MCPAuth.oauth2, MCPAuth.aws_sigv4, MCPAuth.token, + MCPAuth.oauth2_token_exchange, ] ] @@ -117,6 +119,22 @@ class MCPCredentials(TypedDict, total=False): aws_session_name: Optional[str] """Session name for STS AssumeRole (used in CloudTrail). Not a secret — stored unencrypted.""" + audience: Optional[str] + """ + Target audience for OAuth 2.0 Token Exchange (RFC 8693) + """ + + token_exchange_endpoint: Optional[str] + """ + IDP token endpoint for OAuth 2.0 Token Exchange (RFC 8693) + """ + + subject_token_type: Optional[str] + """ + Subject token type for OAuth 2.0 Token Exchange (RFC 8693). + Default: urn:ietf:params:oauth:token-type:access_token + """ + class MCPServerCostInfo(TypedDict, total=False): default_cost_per_query: Optional[float] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 8f8673b0a7d..268d064eacc 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -57,6 +57,10 @@ class MCPServer(BaseModel): aws_service_name: Optional[str] = None # defaults to "bedrock-agentcore" aws_role_name: Optional[str] = None # IAM role ARN for STS AssumeRole aws_session_name: Optional[str] = None # session name for CloudTrail auditing + # Token Exchange (OBO) fields — RFC 8693 + token_exchange_endpoint: Optional[str] = None + audience: Optional[str] = None + subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token" # Stdio-specific fields command: Optional[str] = None args: Optional[List[str]] = None @@ -127,3 +131,12 @@ class MCPServer(BaseModel): return any(h.lower() in auth_header_names for h in self.extra_headers) return False + + @property + def has_token_exchange_config(self) -> bool: + """True if this server is configured for OAuth2 token exchange (OBO / RFC 8693).""" + return ( + self.auth_type == MCPAuth.oauth2_token_exchange + and bool(self.client_id and self.client_secret) + and bool(self.token_exchange_endpoint or self.token_url) + ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c05c46e0d45..00a7748309b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3247,6 +3247,7 @@ class LlmProviders(str, Enum): A2A = "a2a" GIGACHAT = "gigachat" NVIDIA_NIM = "nvidia_nim" + NVIDIA_RIVA = "nvidia_riva" CEREBRAS = "cerebras" AI21_CHAT = "ai21_chat" VOLCENGINE = "volcengine" diff --git a/litellm/utils.py b/litellm/utils.py index 0c5b694bd77..019fbc2add8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8545,6 +8545,12 @@ class ProviderConfigManager: ) return MistralAudioTranscriptionConfig() + elif litellm.LlmProviders.NVIDIA_RIVA == provider: + from litellm.llms.nvidia_riva.audio_transcription.transformation import ( + NvidiaRivaAudioTranscriptionConfig, + ) + + return NvidiaRivaAudioTranscriptionConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 61e7c1de843..92e87c00ef6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -28879,6 +28879,19 @@ "mode": "chat", "output_cost_per_token": 0.0 }, + "sambanova/MiniMax-M2.7": { + "input_cost_per_token": 3e-07, + "litellm_provider": "sambanova", + "max_input_tokens": 204800, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://cloud.sambanova.ai/plans/pricing", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "sambanova/DeepSeek-R1": { "input_cost_per_token": 5e-06, "litellm_provider": "sambanova", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 3fc7cd43187..1d577213a1b 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1610,6 +1610,22 @@ "interactions": true } }, + "nvidia_riva": { + "display_name": "Nvidia Riva (`nvidia_riva`)", + "url": "https://docs.litellm.ai/docs/providers/nvidia_riva", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, "oci": { "display_name": "OCI (`oci`)", "url": "https://docs.litellm.ai/docs/providers/oci", diff --git a/pyproject.toml b/pyproject.toml index 8445a5a60f8..d194d467913 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.84.0" +version = "1.85.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -10,18 +10,22 @@ authors = [ { name = "BerriAI" }, ] dependencies = [ - "fastuuid==0.14.0", - "httpx==0.28.1", - "openai==2.33.0", - "python-dotenv==1.2.2", - "tiktoken==0.12.0", - "importlib-metadata==8.5.0", - "tokenizers==0.23.1", - "click==8.1.8", - "jinja2==3.1.6", - "aiohttp==3.13.4", - "pydantic==2.12.5", - "jsonschema==4.23.0", + # Ranges (not exact pins) so SDK consumers can coexist with their other + # deps. Reproducibility for our Docker/CI comes from `uv.lock`. + # When changing a floor, verify it installs + imports on every supported + # Python with: `uv pip install --resolution=lowest-direct .` + "fastuuid>=0.14.0,<1.0", + "httpx>=0.28.0,<1.0", + "openai>=2.20.0,<3.0.0", + "python-dotenv>=1.0.0,<2.0", + "tiktoken>=0.8.0,<1.0", + "importlib-metadata>=8.0.0,<9.0", + "tokenizers>=0.21.0,<1.0", + "click>=8.0.0,<9.0", + "jinja2>=3.1.0,<4.0", + "aiohttp>=3.10,<4.0", + "pydantic>=2.10.0,<3.0.0", + "jsonschema>=4.0.0,<5.0", ] [project.urls] @@ -86,6 +90,14 @@ grpc = [ # Newest non-yanked release older than the 30-day cutoff. "grpcio==1.78.0", ] +stt-nvidia-riva = [ + # NVIDIA Riva STT provider (gRPC). These are imported lazily inside the + # provider handler so litellm core remains usable without them. + "nvidia-riva-client>=2.15.0", + "soundfile>=0.12.1", + "audioread>=3.0.1", + "numpy>=1.26.0", +] google = ["google-cloud-aiplatform==1.133.0"] proxy-runtime = [ # Historically bundled in the proxy Docker images via requirements.txt. @@ -238,7 +250,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.84.0" +version = "1.85.0" version_files = [ "pyproject.toml:^version", ] diff --git a/scripts/mock_bedrock_passthrough_target.py b/scripts/mock_bedrock_passthrough_target.py new file mode 100644 index 00000000000..e993cd99bde --- /dev/null +++ b/scripts/mock_bedrock_passthrough_target.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +""" +Minimal HTTP target for testing LiteLLM **Bedrock pass-through** (`/bedrock/...` on the proxy). + +What it does + - Serves a tiny Converse-shaped JSON (and optional invoke-shaped) response so the proxy can + complete a round trip without calling AWS. + - Does **not** verify SigV4 (Bedrock does); any Authorization header is accepted. + +How to run + uv run python scripts/mock_bedrock_passthrough_target.py --host 127.0.0.1 --port 9999 + +Wire LiteLLM to this host (use **one** of these patterns): + + 1) model_list (recommended) — set the Bedrock runtime base to the mock: + + model_list: + - model_name: mock-bedrock-claude + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + custom_llm_provider: bedrock + aws_region_name: us-west-2 + api_base: "http://127.0.0.1:9999" + + 2) Environment (see litellm BaseAWSLLM.get_runtime_endpoint):: + + export AWS_BEDROCK_RUNTIME_ENDPOINT="http://127.0.0.1:9999" + +Then call the proxy, e.g. (model_name must match config):: + + curl -sS -X POST "http://127.0.0.1:4000/bedrock/model/mock-bedrock-claude/converse" \ + -H "Authorization: Bearer $LITELLM_KEY" -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":[{"text":"hi"}]}]}' + +The proxy will forward to: {api_base}/model//converse (SigV4-signed). +This mock implements POST .../converse and returns a minimal valid Converse response. + +Notes + - `invoke-with-response-stream` returns a real **binary** AWS event stream + (`application/vnd.amazon.eventstream`) with Anthropic-style JSON payloads inside each + `PayloadPart`, matching Bedrock's InvokeModelWithResponseStream wire format. See + https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_InvokeModelWithResponseStream.html + and https://docs.aws.amazon.com/awstreams/latest/devguide/message-formats.html + - `converse-stream` is still JSON-only placeholder (different inner event shapes). + - Use real (or any non-empty) AWS creds in the environment of the **proxy**; signing still runs. +""" +from __future__ import annotations + +import argparse +import base64 +import json +from binascii import crc32 +from struct import pack +from typing import Any, Dict, Iterator, List + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from starlette.responses import StreamingResponse + +app = FastAPI(title="Mock Bedrock runtime (pass-through test target)") + + +# Minimal structure compatible with Converse: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_Converse.html +def _converse_response_body() -> Dict[str, Any]: + return { + "output": { + "message": { + "role": "assistant", + "content": [ + {"text": "mock: ok from mock_bedrock_passthrough_target.py"} + ], + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1, + "outputTokens": 2, + "totalTokens": 3, + }, + } + + +# Minimal invoke (Anthropic messages on bedrock) style — adjust if you test /invoke +def _invoke_response_body() -> Dict[str, Any]: + return { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "mock invoke response"}], + "model": "mock", + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 2}, + } + + +def _encode_event_stream_message(headers: Dict[str, str], payload: bytes) -> bytes: + """Single AWS binary event-stream frame (same layout botocore's ``EventStreamBuffer`` parses).""" + header_blob = b"" + for name, value in headers.items(): + nb = name.encode("utf-8") + vb = value.encode("utf-8") + header_blob += bytes([len(nb)]) + nb + bytes([7]) + pack("!H", len(vb)) + vb + headers_length = len(header_blob) + payload_length = len(payload) + total_length = 12 + headers_length + payload_length + 4 + prelude_wo_crc = pack("!II", total_length, headers_length) + prelude_crc_val = crc32(prelude_wo_crc) & 0xFFFFFFFF + prelude = prelude_wo_crc + pack("!I", prelude_crc_val) + wo_msg_crc = prelude + header_blob + payload + msg_crc_val = crc32(wo_msg_crc[8:], prelude_crc_val) & 0xFFFFFFFF + return wo_msg_crc + pack("!I", msg_crc_val) + + +def _bedrock_payload_part(inner_event: Dict[str, Any]) -> bytes: + """Outer JSON expected by bedrock-runtime ``ResponseStream`` / ``PayloadPart``.""" + inner_bytes = json.dumps(inner_event, separators=(",", ":")).encode("utf-8") + outer = { + "chunk": { + "bytes": base64.b64encode(inner_bytes).decode("ascii"), + } + } + return json.dumps(outer, separators=(",", ":")).encode("utf-8") + + +def _anthropic_invoke_stream_events( + model_id: str, assistant_text: str +) -> List[Dict[str, Any]]: + """ + Minimal Anthropic Messages stream events as returned inside Bedrock stream chunks. + Mirrors the sequence Amazon emits for Claude on ``invoke-with-response-stream``. + """ + msg_id = "msg_mock_bedrock_stream" + input_tokens = 3 + output_tokens = max(1, len(assistant_text) // 4) + events: List[Dict[str, Any]] = [ + { + "type": "message_start", + "message": { + "model": model_id, + "id": msg_id, + "type": "message", + "role": "assistant", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": input_tokens, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + }, + }, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ] + # Split text into small deltas so downstream streaming behavior is visible. + step = 24 + for i in range(0, len(assistant_text), step): + events.append( + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "text_delta", + "text": assistant_text[i : i + step], + }, + } + ) + events.append({"type": "content_block_stop", "index": 0}) + events.append( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + } + ) + events.append( + { + "type": "message_stop", + "amazon-bedrock-invocationMetrics": { + "inputTokenCount": input_tokens, + "outputTokenCount": output_tokens, + "invocationLatency": 42, + "firstByteLatency": 10, + }, + } + ) + return events + + +def _iter_invoke_with_response_stream(model_id: str) -> Iterator[bytes]: + text = ( + "mock streaming: ok from scripts/mock_bedrock_passthrough_target.py " + "(invoke-with-response-stream)." + ) + headers = { + ":event-type": "chunk", + ":content-type": "application/json", + ":message-type": "event", + } + for ev in _anthropic_invoke_stream_events(model_id, text): + yield _encode_event_stream_message(headers, _bedrock_payload_part(ev)) + + +@app.get("/health") +def health() -> Dict[str, str]: + return {"status": "ok"} + + +@app.post("/model/{model_path:path}/converse") +async def converse(model_path: str, request: Request) -> JSONResponse: + # Optional: log body for debugging + _ = await request.body() + return JSONResponse(content=_converse_response_body()) + + +@app.post("/model/{model_path:path}/converse-stream") +async def converse_stream(model_path: str, request: Request) -> JSONResponse: + """ + Not a real AWS event stream — returns JSON for quick smoke tests only. + """ + _ = await request.body() + return JSONResponse( + content={ + "note": "This mock does not implement application/vnd.amazon.eventstream; use /converse for basic tests." + } + ) + + +@app.post("/model/{model_path:path}/invoke") +async def invoke(model_path: str, request: Request) -> JSONResponse: + _ = await request.body() + return JSONResponse(content=_invoke_response_body()) + + +@app.post("/model/{model_path:path}/invoke-with-response-stream") +async def invoke_with_response_stream( + model_path: str, request: Request +) -> StreamingResponse: + """ + Binary ``application/vnd.amazon.eventstream`` body compatible with boto3/botocore + ``InvokeModelWithResponseStream`` / LiteLLM's Bedrock invoke streaming path. + """ + _ = await request.body() + return StreamingResponse( + _iter_invoke_with_response_stream(model_id=model_path), + media_type="application/vnd.amazon.eventstream", + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=9999) + args = parser.parse_args() + + import uvicorn + + uvicorn.run(app, host=args.host, port=args.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/scripts/tpm_headline_test.sh b/scripts/tpm_headline_test.sh new file mode 100755 index 00000000000..a2f4063d311 --- /dev/null +++ b/scripts/tpm_headline_test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Concurrent TPM bypass test — mints a virtual key with tpm_limit=100 +# (api_key scope in the v3 rate-limiter), races 10 concurrent calls, +# prints a verdict, then deletes the key. +# +# Note: the `tpm: 100` on a model_list deployment is the *router's* +# load-balancing TPM, not a v3 rate-limit descriptor. The v3 limiter +# enforces against limits set on the key/team/user — so we set +# tpm_limit=100 on the key itself. +# +# Pre-PR: ~all 10 return 200 (race lets concurrent requests bypass the limit). +# Post-PR: only ~1–2 fit under tpm_limit=100, rest return 429. +# +# Setup (separate terminal): +# kubectl port-forward -n litellm svc/yassin-veks-litellm-helm 4000:4000 +# +# Run: +# bash scripts/tpm_headline_test.sh +set -u +PROXY="${PROXY:-http://localhost:4000}" +MASTER_KEY="${MASTER_KEY:-sk-perf-test-fixed-do-not-rotate}" +MODEL="${MODEL:-opus-4.6}" + +echo "=== Concurrent TPM bypass test ===" +echo "proxy=$PROXY model=$MODEL key tpm_limit=100 concurrency=10 max_tokens=50" +echo + +gen_resp=$(curl -s -X POST "$PROXY/key/generate" \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"models\":[\"$MODEL\"],\"tpm_limit\":100,\"duration\":\"10m\",\"key_alias\":\"tpm-headline-$$-$(date +%s)\"}") + +KEY=$(printf '%s' "$gen_resp" | sed -n 's/.*"key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') +if [ -z "$KEY" ]; then + echo "FAIL — could not mint virtual key. Response: $gen_resp" + exit 1 +fi +echo "Minted virtual key: ${KEY:0:12}…" +echo + +cleanup() { + curl -s -X POST "$PROXY/key/delete" \ + -H "Authorization: Bearer $MASTER_KEY" \ + -H "Content-Type: application/json" \ + -d "{\"keys\":[\"$KEY\"]}" > /dev/null 2>&1 || true + [ -n "${tmp:-}" ] && rm -rf "$tmp" +} +trap cleanup EXIT + +tmp=$(mktemp -d) +for i in $(seq 1 10); do + ( curl -s -o "$tmp/body.$i" -w "%{http_code}" \ + "$PROXY/v1/chat/completions" \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"concurrent tpm test $i\"}],\"max_tokens\":50}" \ + > "$tmp/code.$i" ) & +done +wait + +ok=0; limited=0; other=0 +for i in $(seq 1 10); do + code=$(cat "$tmp/code.$i") + case "$code" in + 200) ok=$((ok+1)) ;; + 429) limited=$((limited+1)) ;; + *) other=$((other+1)); echo "req $i -> $code: $(cat "$tmp/body.$i" | head -c 200)" ;; + esac +done + +echo +echo "Results: 200=$ok 429=$limited other=$other" +if [ "$limited" -ge 1 ] && [ "$ok" -ge 1 ]; then + echo "PASS — reservation enforced under concurrency." + exit 0 +elif [ "$ok" -eq 10 ]; then + echo "FAIL — all 10 succeeded; concurrent bypass still possible." + exit 1 +else + echo "INCONCLUSIVE — investigate non-200/429 above." + exit 2 +fi diff --git a/tests/_vcr_conftest_common.py b/tests/_vcr_conftest_common.py index 73a5635ab67..b2c7eeb78db 100644 --- a/tests/_vcr_conftest_common.py +++ b/tests/_vcr_conftest_common.py @@ -7,7 +7,9 @@ from __future__ import annotations import atexit import hashlib +import json import os +import re import sys from typing import Iterable @@ -74,6 +76,17 @@ FILTERED_RESPONSE_HEADERS = ( "date", ) +# Tiny placeholder used to replace base64 image payloads in cassettes. +# Decodes to b"test" — short, valid base64 so test code that decodes +# the field still succeeds. +VCR_IMAGE_B64_PLACEHOLDER = "dGVzdA==" + +# Fixed boundary substituted into multipart request bodies so the +# ``safe_body`` matcher sees the same bytes across record and replay. +# httpx generates a fresh random boundary per request via os.urandom, +# which otherwise turns every multipart cassette into a permanent miss. +VCR_FIXED_MULTIPART_BOUNDARY = "vcr-static-boundary" + def _scrub_response(response): if not isinstance(response, dict): @@ -86,8 +99,88 @@ def _scrub_response(response): return response +def _replace_b64_json_in_place(obj) -> bool: + """Recursively replace ``b64_json`` string values in a JSON tree. + + Returns ``True`` if any value was rewritten. The check on the + existing value's length keeps the function idempotent — once a + value has been swapped to the placeholder, subsequent invocations + are no-ops. + """ + changed = False + if isinstance(obj, dict): + for key, value in obj.items(): + if ( + key == "b64_json" + and isinstance(value, str) + and len(value) > len(VCR_IMAGE_B64_PLACEHOLDER) + ): + obj[key] = VCR_IMAGE_B64_PLACEHOLDER + changed = True + elif _replace_b64_json_in_place(value): + changed = True + elif isinstance(obj, list): + for item in obj: + if _replace_b64_json_in_place(item): + changed = True + return changed + + +def _strip_image_b64_payloads(response): + """Replace ``b64_json`` payloads in image-gen responses before save. + + Image-edit and image-generation responses carry the full base64 + PNG/JPEG (1-10+ MB) in ``data[*].b64_json``. The image_gen tests + only assert response shape — the field decodes, schema validates — + they never inspect pixel content. Swapping to a 4-byte placeholder + preserves all those checks while shrinking cassettes by ~99%. + """ + if not isinstance(response, dict): + return response + body = response.get("body") + if not isinstance(body, dict): + return response + raw = body.get("string") + if raw is None: + return response + + if isinstance(raw, (bytes, bytearray)): + try: + text = bytes(raw).decode("utf-8") + except UnicodeDecodeError: + return response + was_bytes = True + elif isinstance(raw, str): + text = raw + was_bytes = False + else: + return response + + try: + payload = json.loads(text) + except (ValueError, TypeError): + return response + + if not _replace_b64_json_in_place(payload): + return response + + new_text = json.dumps(payload, separators=(",", ":")) + body["string"] = new_text.encode("utf-8") if was_bytes else new_text + + headers = response.get("headers") + if isinstance(headers, dict): + new_len_value = str(len(new_text.encode("utf-8"))) + for key in list(headers): + if str(key).lower() == "content-length": + value = headers[key] + headers[key] = ( + [new_len_value] if isinstance(value, list) else new_len_value + ) + return response + + def _before_record_response(response): - return filter_non_2xx_response(_scrub_response(response)) + return filter_non_2xx_response(_scrub_response(_strip_image_b64_payloads(response))) def _safe_body_matcher(r1, r2) -> None: @@ -172,8 +265,84 @@ def _strip_headers(headers, names: Iterable[str]) -> None: pass +def _normalize_multipart_boundary(request) -> None: + """Rewrite random multipart boundaries to a fixed string in-place. + + httpx generates a fresh ``boundary=`` for every + multipart request via ``os.urandom``. Without normalization, the + request body bytes differ across runs even when everything else is + identical, the ``safe_body`` matcher misses, and the persister + keeps appending new episodes until ``MAX_EPISODES_PER_CASSETTE`` + refuses the save — leaving audio-transcription tests effectively + unmocked. Replacing the boundary in both the Content-Type header + and the body bytes makes the request deterministic. + + Idempotent — vcrpy invokes this hook multiple times per request, + so the second invocation sees ``boundary=vcr-static-boundary`` + already and short-circuits. + """ + headers = getattr(request, "headers", None) + if headers is None: + return + + content_type_key = None + content_type_value = None + try: + for key in list(headers.keys()): + if str(key).lower() == "content-type": + content_type_key = key + value = headers[key] + content_type_value = value if isinstance(value, str) else str(value) + break + except AttributeError: + return + + if not content_type_value or "multipart/" not in content_type_value.lower(): + return + + fixed_param = f"boundary={VCR_FIXED_MULTIPART_BOUNDARY}" + if fixed_param in content_type_value: + return + + match = re.search(r"boundary=([^\s;]+)", content_type_value) + if not match: + return + current_boundary = match.group(1).strip('"') + if current_boundary == VCR_FIXED_MULTIPART_BOUNDARY: + return + + try: + headers[content_type_key] = content_type_value.replace( + match.group(0), fixed_param + ) + except (TypeError, AttributeError): + return + + body = getattr(request, "body", None) + if body is None: + return + + if isinstance(body, (bytes, bytearray)): + try: + new_body = bytes(body).replace( + current_boundary.encode("utf-8"), + VCR_FIXED_MULTIPART_BOUNDARY.encode("utf-8"), + ) + except (TypeError, ValueError): + return + elif isinstance(body, str): + new_body = body.replace(current_boundary, VCR_FIXED_MULTIPART_BOUNDARY) + else: + return + + try: + request.body = new_body + except (AttributeError, TypeError): + pass + + def _before_record_request(request): - """Fingerprint API keys, then scrub them. + """Fingerprint API keys, scrub them, and normalize multipart boundaries. Order matters in two ways: @@ -187,7 +356,8 @@ def _before_record_request(request): auth headers we already stripped, so re-hashing would yield ``"no-key"`` and the stored vs. incoming fingerprints would diverge. Skip the recompute when the header is already set so - this hook is idempotent. + this hook is idempotent. The boundary normalizer is also + idempotent for the same reason. """ headers = getattr(request, "headers", None) if headers is None: @@ -199,6 +369,7 @@ def _before_record_request(request): except (TypeError, AttributeError): pass _strip_headers(headers, FILTERED_REQUEST_HEADERS) + _normalize_multipart_boundary(request) return request diff --git a/tests/audio_tests/test_audio_speech.py b/tests/audio_tests/test_audio_speech.py index 46d45158910..52a2316a16f 100644 --- a/tests/audio_tests/test_audio_speech.py +++ b/tests/audio_tests/test_audio_speech.py @@ -26,24 +26,7 @@ import pytest import litellm -@pytest.mark.parametrize( - "sync_mode", - [True, False], -) -@pytest.mark.parametrize( - "model, api_key, api_base", - [ - ( - "azure/tts", - os.getenv("AZURE_TTS_API_KEY"), - os.getenv("AZURE_TTS_API_BASE"), - ), - ("openai/tts-1", os.getenv("OPENAI_API_KEY"), None), - ], -) # , -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_audio_speech_litellm(sync_mode, model, api_base, api_key): +async def _run_audio_speech_litellm(sync_mode, model, api_base, api_key): litellm._turn_on_debug() speech_file_path = Path(__file__).parent / "speech.mp3" @@ -85,6 +68,30 @@ async def test_audio_speech_litellm(sync_mode, model, api_base, api_key): assert isinstance(response, HttpxBinaryResponseContent) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) +async def test_audio_speech_litellm_azure(sync_mode): + await _run_audio_speech_litellm( + sync_mode=sync_mode, + model="azure/tts", + api_base=os.getenv("AZURE_TTS_API_BASE"), + api_key=os.getenv("AZURE_TTS_API_KEY"), + ) + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) +async def test_audio_speech_litellm_openai(sync_mode): + await _run_audio_speech_litellm( + sync_mode=sync_mode, + model="openai/tts-1", + api_base=None, + api_key=os.getenv("OPENAI_API_KEY"), + ) + + @pytest.mark.parametrize( "sync_mode", [False, True], diff --git a/tests/audio_tests/test_whisper.py b/tests/audio_tests/test_whisper.py index 199b0e6a4f9..cdf079f8cb4 100644 --- a/tests/audio_tests/test_whisper.py +++ b/tests/audio_tests/test_whisper.py @@ -39,24 +39,7 @@ import litellm from litellm import Router -@pytest.mark.parametrize( - "model, api_key, api_base", - [ - ("whisper-1", None, None), - ( - "azure/whisper", - os.getenv("AZURE_WHISPER_API_KEY"), - os.getenv("AZURE_WHISPER_API_BASE"), - ), - ], -) -@pytest.mark.parametrize( - "response_format, timestamp_granularities", - [("json", None), ("vtt", None), ("verbose_json", ["word"])], -) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_transcription( +async def _run_transcription( model, api_key, api_base, response_format, timestamp_granularities ): transcript = await litellm.atranscription( @@ -74,6 +57,38 @@ async def test_transcription( assert transcript.text is not None +@pytest.mark.parametrize( + "response_format, timestamp_granularities", + [("json", None), ("vtt", None), ("verbose_json", ["word"])], +) +@pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) +async def test_transcription_openai_whisper(response_format, timestamp_granularities): + await _run_transcription( + model="whisper-1", + api_key=None, + api_base=None, + response_format=response_format, + timestamp_granularities=timestamp_granularities, + ) + + +@pytest.mark.parametrize( + "response_format, timestamp_granularities", + [("json", None), ("vtt", None), ("verbose_json", ["word"])], +) +@pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) +async def test_transcription_azure_whisper(response_format, timestamp_granularities): + await _run_transcription( + model="azure/whisper", + api_key=os.getenv("AZURE_WHISPER_API_KEY"), + api_base=os.getenv("AZURE_WHISPER_API_BASE"), + response_format=response_format, + timestamp_granularities=timestamp_granularities, + ) + + @pytest.mark.asyncio() async def test_transcription_caching(): import litellm diff --git a/tests/batches_tests/conftest.py b/tests/batches_tests/conftest.py index e6e31546a82..ecb606b2cf3 100644 --- a/tests/batches_tests/conftest.py +++ b/tests/batches_tests/conftest.py @@ -1,7 +1,6 @@ # conftest.py import asyncio -import importlib import os import sys @@ -12,16 +11,6 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm # noqa: E402,F401 -from tests._vcr_conftest_common import ( # noqa: E402 - VerboseReporterState, - apply_vcr_auto_marker_to_items, - record_vcr_outcome, - register_persister_if_enabled, - vcr_config_dict, -) - -_verbose_state = VerboseReporterState() - @pytest.fixture(scope="session") def event_loop(): @@ -31,37 +20,3 @@ def event_loop(): loop = asyncio.new_event_loop() yield loop loop.close() - - -@pytest.fixture(scope="module") -def vcr_config(): - return vcr_config_dict() - - -def pytest_recording_configure(config, vcr): - register_persister_if_enabled(vcr) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - rep = outcome.get_result() - setattr(item, f"rep_{rep.when}", rep) - - -@pytest.fixture(autouse=True) -def _vcr_outcome_gate(request, vcr): - yield - record_vcr_outcome(request, vcr) - - -def pytest_configure(config): - _verbose_state.remember_pluginmanager(config) - - -def pytest_runtest_logreport(report): - _verbose_state.maybe_emit_verdict(report) - - -def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items(items) diff --git a/tests/batches_tests/test_fine_tuning_api.py b/tests/batches_tests/test_fine_tuning_api.py index 1c1af308df0..c489685eaff 100644 --- a/tests/batches_tests/test_fine_tuning_api.py +++ b/tests/batches_tests/test_fine_tuning_api.py @@ -47,6 +47,37 @@ class TestCustomLogger(CustomLogger): self.standard_logging_object = kwargs["standard_logging_object"] +async def _acreate_fine_tuning_job_with_propagation_retry( + *, max_attempts: int = 12, initial_delay: float = 1.0, **kwargs +): + """ + Wrap litellm.acreate_fine_tuning_job and retry on the eventual-consistency + 400 OpenAI returns when a freshly-uploaded training file isn't yet visible + to the fine-tuning endpoint (`'file-... does not exist'`). + + Polling the files-retrieve endpoint or `FileObject.status` doesn't help — + OpenAI's `status` field is deprecated, and the retrieve and fine-tuning + endpoints don't share a consistency model. Retrying the operation itself + is the only reliable signal that propagation has finished. + + Total budget with defaults: ~70s across 12 attempts (exp backoff capped at + 8s). + """ + delay = initial_delay + last_error: Optional[openai.BadRequestError] = None + for _ in range(max_attempts): + try: + return await litellm.acreate_fine_tuning_job(**kwargs) + except openai.BadRequestError as e: + if "does not exist" not in str(e): + raise + last_error = e + await asyncio.sleep(delay) + delay = min(delay * 1.5, 8.0) + assert last_error is not None + raise last_error + + @pytest.mark.asyncio async def test_create_fine_tune_jobs_async(): try: @@ -64,9 +95,11 @@ async def test_create_fine_tune_jobs_async(): ) print("Response from creating file=", file_obj) - create_fine_tuning_response = await litellm.acreate_fine_tuning_job( - model="gpt-3.5-turbo-0125", - training_file=file_obj.id, + create_fine_tuning_response = ( + await _acreate_fine_tuning_job_with_propagation_retry( + model="gpt-4o-mini-2024-07-18", + training_file=file_obj.id, + ) ) print( @@ -74,7 +107,7 @@ async def test_create_fine_tune_jobs_async(): ) assert create_fine_tuning_response.id is not None - assert create_fine_tuning_response.model == "gpt-3.5-turbo-0125" + assert create_fine_tuning_response.model == "gpt-4o-mini-2024-07-18" await asyncio.sleep(2) _logged_standard_logging_object = custom_logger.standard_logging_object @@ -83,7 +116,7 @@ async def test_create_fine_tune_jobs_async(): "custom_logger.standard_logging_object=", json.dumps(_logged_standard_logging_object, indent=4), ) - assert _logged_standard_logging_object["model"] == "gpt-3.5-turbo-0125" + assert _logged_standard_logging_object["model"] == "gpt-4o-mini-2024-07-18" assert _logged_standard_logging_object["id"] == create_fine_tuning_response.id # list fine tuning jobs @@ -427,10 +460,10 @@ async def test_mock_openai_create_fine_tune_job(): with patch.object(client.fine_tuning.jobs, "create") as mock_create: mock_create.return_value = FineTuningJob( id="ft-123", - model="gpt-3.5-turbo-0125", + model="gpt-4o-mini-2024-07-18", created_at=1677610602, status="validating_files", - fine_tuned_model="ft:gpt-3.5-turbo-0125:org:custom_suffix:id", + fine_tuned_model="ft:gpt-4o-mini-2024-07-18:org:custom_suffix:id", object="fine_tuning.job", hyperparameters=Hyperparameters( n_epochs=3, @@ -442,7 +475,7 @@ async def test_mock_openai_create_fine_tune_job(): ) response = await litellm.acreate_fine_tuning_job( - model="gpt-3.5-turbo-0125", + model="gpt-4o-mini-2024-07-18", training_file="file-123", hyperparameters={"n_epochs": 3}, suffix="custom_suffix", @@ -453,16 +486,19 @@ async def test_mock_openai_create_fine_tune_job(): mock_create.assert_called_once() request_params = mock_create.call_args.kwargs - assert request_params["model"] == "gpt-3.5-turbo-0125" + assert request_params["model"] == "gpt-4o-mini-2024-07-18" assert request_params["training_file"] == "file-123" assert request_params["hyperparameters"] == {"n_epochs": 3} assert request_params["suffix"] == "custom_suffix" # Verify the response assert response.id == "ft-123" - assert response.model == "gpt-3.5-turbo-0125" + assert response.model == "gpt-4o-mini-2024-07-18" assert response.status == "validating_files" - assert response.fine_tuned_model == "ft:gpt-3.5-turbo-0125:org:custom_suffix:id" + assert ( + response.fine_tuned_model + == "ft:gpt-4o-mini-2024-07-18:org:custom_suffix:id" + ) @pytest.mark.asyncio diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 2100aa1377f..5a09403c570 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -126,6 +126,7 @@ sentry_sdk: >=2.21.0 # Unknown license cryptography: >=43.0.1 # Unknown license tzdata: >=2025.1 # Unknown license urllib3: >=2.0.0 # MIT license - https://github.com/urllib3/urllib3 +audioread: >=3.0.1 # MIT license manually verified - https://github.com/beetbox/audioread python-dotenv: >=1.0.0 # Unknown license tiktoken: >=0.8.0 # Unknown license click: >=8.1.7 # Unknown license diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 9f879b9d501..e6c76fa7cd6 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -661,14 +661,14 @@ async def test_async_log_failure_event(prometheus_logger): # litellm_llm_api_failed_requests_metric incremented # Labels: end_user, hashed_api_key, api_key_alias, model, team, team_alias, user, model_id prometheus_logger.litellm_llm_api_failed_requests_metric.labels.assert_called_once_with( - None, # end_user_id - "test_hash", - "test_alias", - "gpt-3.5-turbo", - "test_team", - "test_team_alias", - "test_user", - "model-123", # model_id from standard_logging_payload + end_user=None, + hashed_api_key="test_hash", + api_key_alias="test_alias", + model="gpt-3.5-turbo", + team="test_team", + team_alias="test_team_alias", + user="test_user", + model_id="model-123", ) prometheus_logger.litellm_llm_api_failed_requests_metric.labels().inc.assert_called_once() diff --git a/tests/litellm/test_sambanova_model_metadata.py b/tests/litellm/test_sambanova_model_metadata.py new file mode 100644 index 00000000000..bc31bfb0af2 --- /dev/null +++ b/tests/litellm/test_sambanova_model_metadata.py @@ -0,0 +1,29 @@ +import json +from pathlib import Path + +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + +def test_sambanova_minimax_m27_model_info(): + model = "sambanova/MiniMax-M2.7" + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + info = model_cost.get(model) + assert ( + info is not None + ), f"{model} not found in model_prices_and_context_window.json" + assert info["litellm_provider"] == "sambanova" + assert info["mode"] == "chat" + assert info["input_cost_per_token"] > 0 + assert info["output_cost_per_token"] > 0 + assert info["max_input_tokens"] == 204800 + assert info["max_output_tokens"] == 131072 + assert info["supports_function_calling"] is True + assert info["supports_reasoning"] is True + assert info["supports_tool_choice"] is True + + routed_model, provider, _, _ = get_llm_provider(model=model) + assert routed_model == "MiniMax-M2.7" + assert provider == "sambanova" diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index aedc4f810cd..77850dac457 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -853,7 +853,11 @@ class BaseLLMChatTest(ABC): @pytest.mark.parametrize( "image_url", [ - "http://img1.etsystatic.com/260/0/7813604/il_fullxfull.4226713999_q86e.jpg", + # In-repo logo served via jsdelivr (sha-pinned, immutable). + # Bedrock fetches the URL and base64-embeds it in the + # Converse request body; using a multi-MB hosted product + # photo here previously bloated cassettes to ~60 MB each. + "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/ui/litellm-dashboard/public/assets/logos/litellm_logo.jpg", "https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png", ], ) diff --git a/tests/llm_translation/realtime/test_openai_realtime.py b/tests/llm_translation/realtime/test_openai_realtime.py index e08b2de7fe5..c5f77de6beb 100644 --- a/tests/llm_translation/realtime/test_openai_realtime.py +++ b/tests/llm_translation/realtime/test_openai_realtime.py @@ -101,7 +101,7 @@ async def test_openai_realtime_direct_call_no_intent(): try: await litellm._arealtime( - model="openai/gpt-4o-realtime-preview-2024-10-01", + model="openai/gpt-4o-realtime-preview", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), timeout=60, @@ -250,13 +250,13 @@ async def test_openai_realtime_direct_call_with_intent(): caught_exception = None query_params: RealtimeQueryParams = { - "model": "openai/gpt-4o-realtime-preview-2024-10-01", + "model": "openai/gpt-4o-realtime-preview", "intent": "chat", } try: await litellm._arealtime( - model="openai/gpt-4o-realtime-preview-2024-10-01", + model="openai/gpt-4o-realtime-preview", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), query_params=query_params, @@ -331,7 +331,7 @@ def test_realtime_query_params_construction(): from litellm.types.realtime import RealtimeQueryParams # Test case 1: intent is None (should not be included) - model = "gpt-4o-realtime-preview-2024-10-01" + model = "gpt-4o-realtime-preview" intent = None query_params: RealtimeQueryParams = {"model": model} @@ -369,17 +369,17 @@ async def test_realtime_query_params_use_normalized_model_name(monkeypatch): ) def fake_get_llm_provider(model, api_base=None, api_key=None): - return ("gpt-4o-realtime-preview-2024-10-01", "openai", None, None) + return ("gpt-4o-realtime-preview", "openai", None, None) monkeypatch.setattr(realtime_main, "get_llm_provider", fake_get_llm_provider) query_params: RealtimeQueryParams = { - "model": "openai/gpt-4o-realtime-preview-2024-10-01", + "model": "openai/gpt-4o-realtime-preview", "intent": "chat", } await realtime_main._arealtime( - model="openai/gpt-4o-realtime-preview-2024-10-01", + model="openai/gpt-4o-realtime-preview", websocket=MagicMock(), api_key="sk-test", query_params=query_params, @@ -387,7 +387,5 @@ async def test_realtime_query_params_use_normalized_model_name(monkeypatch): ) called_kwargs = mock_async_realtime.call_args.kwargs - assert ( - called_kwargs["query_params"]["model"] == "gpt-4o-realtime-preview-2024-10-01" - ) + assert called_kwargs["query_params"]["model"] == "gpt-4o-realtime-preview" assert called_kwargs["query_params"]["intent"] == "chat" diff --git a/tests/llm_translation/test_evals_api.py b/tests/llm_translation/test_evals_api.py index 89263000200..4a55663e669 100644 --- a/tests/llm_translation/test_evals_api.py +++ b/tests/llm_translation/test_evals_api.py @@ -2,6 +2,7 @@ Tests for Evals API operations across providers """ +import hashlib import os import sys from abc import ABC, abstractmethod @@ -20,6 +21,46 @@ from litellm.types.llms.openai_evals import ( ) +def _stable_eval_name(test_node_name: str, suffix: str = "") -> str: + """Deterministic eval name keyed off the test's node name. + + The previous ``f"Test Eval {int(time.time())}"`` pattern embedded a + fresh value into the request body every run, defeating VCR's + ``safe_body`` matcher and forcing a real OpenAI ``create`` call on + every CI run. With a stable per-test name the cassette matches on + replay, and provider-side resources stay bounded because each test + deletes the eval it owns on teardown. + """ + nonce = hashlib.sha1(test_node_name.encode()).hexdigest()[:12] + return f"vcr-managed-{nonce}{suffix}" + + +_TESTING_CRITERIA = [ + { + "type": "label_model", + "model": "gpt-4o", + "input": [ + { + "role": "developer", + "content": "Classify the sentiment as 'positive' or 'negative'", + }, + {"role": "user", "content": "Statement: {{item.input}}"}, + ], + "passing_labels": ["positive"], + "labels": ["positive", "negative"], + "name": "Sentiment grader", + } +] + + +_PROVIDER_FLAKINESS = ( + litellm.InternalServerError, + litellm.APIConnectionError, + litellm.Timeout, + litellm.ServiceUnavailableError, +) + + class BaseEvalsAPITest(ABC): """ Base test class for Evals API operations. @@ -41,13 +82,64 @@ class BaseEvalsAPITest(ABC): """Return the API base URL for the provider""" pass + @pytest.fixture + def managed_eval(self, request): + """Create a stable-named eval for this test; delete on teardown. + + Function-scoped so each cassette captures the full + create→test→delete cycle. A class-scoped fixture would push + the create into whichever test ran first and the delete into + whichever ran last, which is fragile under reordering. + + Replaces the prior ``list_evals().data[0].id`` pattern, which + made the URL of ``get_eval`` / ``update_eval`` vary across + runs (the "first" eval depends on what other runs left + behind). + """ + custom_llm_provider = self.get_custom_llm_provider() + api_key = self.get_api_key() + api_base = self.get_api_base() + + if not api_key: + pytest.skip(f"No API key provided for {custom_llm_provider}") + + try: + created = litellm.create_eval( + name=_stable_eval_name(request.node.name), + data_source_config={ + "type": "stored_completions", + "metadata": {"usecase": "chatbot", "vcr": "managed"}, + }, + testing_criteria=_TESTING_CRITERIA, + custom_llm_provider=custom_llm_provider, + api_key=api_key, + api_base=api_base, + ) + except _PROVIDER_FLAKINESS: + pytest.skip("Provider service unavailable") + except litellm.RateLimitError: + pytest.skip("Rate limit exceeded") + + yield created + + # Best-effort cleanup. OpenAI eval names are not unique-keyed + # (only IDs are), so a failed delete doesn't block the next + # run's create. + try: + litellm.delete_eval( + eval_id=created.id, + custom_llm_provider=custom_llm_provider, + api_key=api_key, + api_base=api_base, + ) + except Exception: + pass + @pytest.mark.flaky(retries=3, delay=2) - def test_create_eval(self): + def test_create_eval(self, request): """ Test creating an evaluation. """ - import time - custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() @@ -56,53 +148,45 @@ class BaseEvalsAPITest(ABC): pytest.skip(f"No API key provided for {custom_llm_provider}") litellm.set_verbose = True + unique_name = _stable_eval_name(request.node.name) - # Create eval with stored_completions data source - unique_name = f"Test Eval {int(time.time())}" - + created_id = None try: - response = litellm.create_eval( - name=unique_name, - data_source_config={ - "type": "stored_completions", - "metadata": {"usecase": "chatbot"}, - }, - testing_criteria=[ - { - "type": "label_model", - "model": "gpt-4o", - "input": [ - { - "role": "developer", - "content": "Classify the sentiment as 'positive' or 'negative'", - }, - {"role": "user", "content": "Statement: {{item.input}}"}, - ], - "passing_labels": ["positive"], - "labels": ["positive", "negative"], - "name": "Sentiment grader", - } - ], - custom_llm_provider=custom_llm_provider, - api_key=api_key, - api_base=api_base, - ) - except ( - litellm.InternalServerError, - litellm.APIConnectionError, - litellm.Timeout, - litellm.ServiceUnavailableError, - ): - pytest.skip("Provider service unavailable") - except litellm.RateLimitError: - pytest.skip("Rate limit exceeded") + try: + response = litellm.create_eval( + name=unique_name, + data_source_config={ + "type": "stored_completions", + "metadata": {"usecase": "chatbot"}, + }, + testing_criteria=_TESTING_CRITERIA, + custom_llm_provider=custom_llm_provider, + api_key=api_key, + api_base=api_base, + ) + except _PROVIDER_FLAKINESS: + pytest.skip("Provider service unavailable") + except litellm.RateLimitError: + pytest.skip("Rate limit exceeded") - assert response is not None - assert isinstance(response, Eval) - assert response.id is not None - assert response.name == unique_name - print(f"Created eval: {response}") - print(f"Eval ID: {response.id}") + assert response is not None + assert isinstance(response, Eval) + assert response.id is not None + assert response.name == unique_name + created_id = response.id + print(f"Created eval: {response}") + print(f"Eval ID: {response.id}") + finally: + if created_id is not None: + try: + litellm.delete_eval( + eval_id=created_id, + custom_llm_provider=custom_llm_provider, + api_key=api_key, + api_base=api_base, + ) + except Exception: + pass def test_list_evals(self): """ @@ -130,7 +214,7 @@ class BaseEvalsAPITest(ABC): assert hasattr(response, "has_more") print(f"Listed evals: {len(response.data)} evaluations") - def test_get_eval(self): + def test_get_eval(self, managed_eval): """ Test getting a specific evaluation by ID. """ @@ -138,89 +222,54 @@ class BaseEvalsAPITest(ABC): api_key = self.get_api_key() api_base = self.get_api_base() - if not api_key: - pytest.skip(f"No API key provided for {custom_llm_provider}") - litellm.set_verbose = True - # First list existing evals to get an ID - list_response = litellm.list_evals( - limit=1, + response = litellm.get_eval( + eval_id=managed_eval.id, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) - assert isinstance(list_response, ListEvalsResponse) + assert response is not None + assert isinstance(response, Eval) + assert response.id == managed_eval.id + print(f"Retrieved eval: {response}") - if list_response.data and len(list_response.data) > 0: - eval_id = list_response.data[0].id - print(f"Testing with eval ID: {eval_id}") - - # Get the eval - response = litellm.get_eval( - eval_id=eval_id, - custom_llm_provider=custom_llm_provider, - api_key=api_key, - api_base=api_base, - ) - - assert response is not None - assert isinstance(response, Eval) - assert response.id == eval_id - print(f"Retrieved eval: {response}") - else: - pytest.skip("No existing evals to test with") - - def test_update_eval(self): + @pytest.mark.flaky(retries=3, delay=2) + def test_update_eval(self, request, managed_eval): """ Test updating an evaluation. """ - import time - custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() - if not api_key: - pytest.skip(f"No API key provided for {custom_llm_provider}") - litellm.set_verbose = True + updated_name = _stable_eval_name(request.node.name, suffix="-updated") - # First list existing evals - list_response = litellm.list_evals( - limit=1, + response = litellm.update_eval( + eval_id=managed_eval.id, + name=updated_name, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) - assert isinstance(list_response, ListEvalsResponse) - - if list_response.data and len(list_response.data) > 0: - eval_id = list_response.data[0].id - updated_name = f"Updated Eval {int(time.time())}" - - # Update the eval - response = litellm.update_eval( - eval_id=eval_id, - name=updated_name, - custom_llm_provider=custom_llm_provider, - api_key=api_key, - api_base=api_base, - ) - - assert response is not None - assert isinstance(response, Eval) - assert response.id == eval_id - assert response.name == updated_name - print(f"Updated eval: {response}") - else: - pytest.skip("No existing evals to test with") + assert response is not None + assert isinstance(response, Eval) + assert response.id == managed_eval.id + assert response.name == updated_name + print(f"Updated eval: {response}") def test_delete_eval(self): """ Test deleting an evaluation. + + Real delete coverage now lives in the ``managed_eval`` fixture + teardown and in ``test_create_eval``'s ``finally`` block, so + this stays a no-op skip rather than creating a fresh resource + just to delete it. """ custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() @@ -229,8 +278,7 @@ class BaseEvalsAPITest(ABC): if not api_key: pytest.skip(f"No API key provided for {custom_llm_provider}") - # Skip this test to avoid deleting production evals - pytest.skip("Skipping delete test to preserve existing evals") + pytest.skip("Delete is exercised via managed_eval fixture teardown.") class TestOpenAIEvalsAPI(BaseEvalsAPITest): diff --git a/tests/llm_translation/test_vcr_filters.py b/tests/llm_translation/test_vcr_filters.py new file mode 100644 index 00000000000..03891682781 --- /dev/null +++ b/tests/llm_translation/test_vcr_filters.py @@ -0,0 +1,220 @@ +"""Unit tests for the VCR record-time filters that keep cassettes small. + +Covers: +- ``_strip_image_b64_payloads`` — replaces base64 image bodies in + image-gen responses so cassettes don't carry MB-class PNG payloads. +- ``_normalize_multipart_boundary`` — rewrites random multipart + boundaries to a fixed string so audio-transcription request bodies + match across record and replay. +""" + +from __future__ import annotations + +import json +import os +import sys + +from vcr.request import Request + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) + +from tests._vcr_conftest_common import ( # noqa: E402 + VCR_FIXED_MULTIPART_BOUNDARY, + VCR_IMAGE_B64_PLACEHOLDER, + _normalize_multipart_boundary, + _strip_image_b64_payloads, +) + + +# --------------------------------------------------------------------------- +# Image b64 stripper +# --------------------------------------------------------------------------- + + +def _image_response(b64_payload: str, body_type: str = "bytes") -> dict: + body_text = json.dumps({"data": [{"b64_json": b64_payload}]}) + body_string = body_text.encode("utf-8") if body_type == "bytes" else body_text + return { + "status": {"code": 200, "message": "OK"}, + "headers": { + "content-type": ["application/json"], + "content-length": [str(len(body_text.encode("utf-8")))], + }, + "body": {"string": body_string}, + } + + +def test_strip_image_b64_replaces_payload_when_body_is_bytes(): + response = _image_response("A" * 5000, body_type="bytes") + out = _strip_image_b64_payloads(response) + payload = json.loads(out["body"]["string"].decode("utf-8")) + assert payload["data"][0]["b64_json"] == VCR_IMAGE_B64_PLACEHOLDER + + +def test_strip_image_b64_replaces_payload_when_body_is_str(): + response = _image_response("A" * 5000, body_type="str") + out = _strip_image_b64_payloads(response) + payload = json.loads(out["body"]["string"]) + assert payload["data"][0]["b64_json"] == VCR_IMAGE_B64_PLACEHOLDER + + +def test_strip_image_b64_updates_content_length(): + response = _image_response("A" * 5000) + out = _strip_image_b64_payloads(response) + expected_len = len(out["body"]["string"]) + assert out["headers"]["content-length"] == [str(expected_len)] + + +def test_strip_image_b64_is_idempotent(): + response = _image_response("A" * 5000) + once = _strip_image_b64_payloads(response) + twice = _strip_image_b64_payloads(once) + assert once["body"]["string"] == twice["body"]["string"] + + +def test_strip_image_b64_handles_nested_data(): + body_text = json.dumps( + { + "outer": { + "data": [ + {"b64_json": "X" * 4000, "label": "first"}, + {"b64_json": "Y" * 4000, "label": "second"}, + ] + } + } + ) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/json"]}, + "body": {"string": body_text.encode("utf-8")}, + } + out = _strip_image_b64_payloads(response) + payload = json.loads(out["body"]["string"].decode("utf-8")) + assert payload["outer"]["data"][0]["b64_json"] == VCR_IMAGE_B64_PLACEHOLDER + assert payload["outer"]["data"][1]["b64_json"] == VCR_IMAGE_B64_PLACEHOLDER + assert payload["outer"]["data"][0]["label"] == "first" + + +def test_strip_image_b64_leaves_non_image_response_unchanged(): + body_text = json.dumps({"choices": [{"message": {"content": "hello"}}]}) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/json"]}, + "body": {"string": body_text.encode("utf-8")}, + } + out = _strip_image_b64_payloads(response) + assert json.loads(out["body"]["string"].decode("utf-8")) == json.loads(body_text) + + +def test_strip_image_b64_leaves_invalid_json_unchanged(): + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/octet-stream"]}, + "body": {"string": b"\x89PNG\r\n\x1a\n binary stuff not json"}, + } + out = _strip_image_b64_payloads(response) + assert out["body"]["string"] == b"\x89PNG\r\n\x1a\n binary stuff not json" + + +def test_strip_image_b64_skips_short_values(): + """Already-placeholder values aren't re-replaced (idempotency guard).""" + body_text = json.dumps({"data": [{"b64_json": VCR_IMAGE_B64_PLACEHOLDER}]}) + response = { + "status": {"code": 200, "message": "OK"}, + "headers": {"content-type": ["application/json"]}, + "body": {"string": body_text.encode("utf-8")}, + } + out = _strip_image_b64_payloads(response) + payload = json.loads(out["body"]["string"].decode("utf-8")) + assert payload["data"][0]["b64_json"] == VCR_IMAGE_B64_PLACEHOLDER + + +# --------------------------------------------------------------------------- +# Multipart boundary normalizer +# --------------------------------------------------------------------------- + + +def _multipart_request(boundary: str): + body_text = ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="file"; filename="audio.wav"\r\n' + "Content-Type: audio/wav\r\n" + "\r\n" + "fake-audio-bytes\r\n" + f"--{boundary}--\r\n" + ) + return Request( + method="POST", + uri="https://api.openai.com/v1/audio/transcriptions", + body=body_text.encode("utf-8"), + headers={ + "content-type": f"multipart/form-data; boundary={boundary}", + }, + ) + + +def test_normalize_multipart_rewrites_header_and_body(): + req = _multipart_request("abc123random") + _normalize_multipart_boundary(req) + assert ( + req.headers["content-type"] + == f"multipart/form-data; boundary={VCR_FIXED_MULTIPART_BOUNDARY}" + ) + assert b"abc123random" not in req.body + assert VCR_FIXED_MULTIPART_BOUNDARY.encode("utf-8") in req.body + + +def test_normalize_multipart_is_idempotent(): + req = _multipart_request("abc123random") + _normalize_multipart_boundary(req) + body_first = req.body + header_first = req.headers["content-type"] + _normalize_multipart_boundary(req) + assert req.body == body_first + assert req.headers["content-type"] == header_first + + +def test_normalize_multipart_two_distinct_boundaries_match_after_normalize(): + """Whisper-style: two requests with different random boundaries should + end up with byte-identical bodies after normalization.""" + req1 = _multipart_request("boundaryAAA") + req2 = _multipart_request("boundaryBBB") + _normalize_multipart_boundary(req1) + _normalize_multipart_boundary(req2) + assert req1.body == req2.body + assert req1.headers["content-type"] == req2.headers["content-type"] + + +def test_normalize_multipart_skips_non_multipart_requests(): + req = Request( + method="POST", + uri="https://api.openai.com/v1/chat/completions", + body=b'{"model":"gpt-4o"}', + headers={"content-type": "application/json"}, + ) + _normalize_multipart_boundary(req) + assert req.headers["content-type"] == "application/json" + assert req.body == b'{"model":"gpt-4o"}' + + +def test_normalize_multipart_skips_request_without_content_type(): + req = Request( + method="POST", + uri="https://api.openai.com/v1/chat/completions", + body=b"unknown body", + headers={}, + ) + _normalize_multipart_boundary(req) + assert req.body == b"unknown body" + + +def test_normalize_multipart_handles_quoted_boundary(): + req = Request( + method="POST", + uri="https://api.openai.com/v1/audio/transcriptions", + body=b"--quoted-boundary--body content--quoted-boundary--", + headers={"content-type": 'multipart/form-data; boundary="quoted-boundary"'}, + ) + _normalize_multipart_boundary(req) + assert b"quoted-boundary" not in req.body + assert VCR_FIXED_MULTIPART_BOUNDARY.encode("utf-8") in req.body diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index 82510b6f4fd..14626aa8e45 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -193,19 +193,12 @@ def _azure_ai_image_mock_response(*args, **kwargs): return new_response -@pytest.mark.parametrize( - "model, api_base, api_key", - [ - ( - "azure_ai/Cohere-embed-v3-multilingual-2", - os.getenv("AZURE_AI_API_BASE"), - os.getenv("AZURE_AI_API_KEY"), - ) - ], -) @pytest.mark.parametrize("sync_mode", [True]) # , False @pytest.mark.asyncio -async def test_azure_ai_embedding_image(model, api_base, api_key, sync_mode): +async def test_azure_ai_embedding_image(sync_mode): + model = "azure_ai/Cohere-embed-v3-multilingual-2" + api_base = os.getenv("AZURE_AI_API_BASE") + api_key = os.getenv("AZURE_AI_API_KEY") try: os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/tests/logging_callback_tests/conftest.py b/tests/logging_callback_tests/conftest.py index 4f847d93a52..7042d6094d9 100644 --- a/tests/logging_callback_tests/conftest.py +++ b/tests/logging_callback_tests/conftest.py @@ -101,6 +101,7 @@ _SCALAR_ATTRS = ( "redact_messages_in_exceptions", "redact_user_api_key_info", "s3_callback_params", + "s3_audit_callback_params", "datadog_params", "vector_store_registry", ) @@ -128,6 +129,7 @@ def isolate_litellm_state(): leaking across tests within the same xdist worker. """ from litellm.litellm_core_utils import litellm_logging as ll_logging + from litellm.proxy.management_helpers import audit_logs as ll_audit_logs # Flush cache and clear internal logger instances before test if hasattr(litellm, "in_memory_llm_clients_cache"): @@ -135,6 +137,7 @@ def isolate_litellm_state(): # Clear cached logger instances (LangsmithLogger, SlackAlerting, etc.) ll_logging._in_memory_loggers.clear() + ll_audit_logs._audit_log_callback_cache.clear() # Reset ALL attrs to their true defaults before the test runs. # This undoes any module-level mutations from test file imports. @@ -156,6 +159,7 @@ def isolate_litellm_state(): litellm.in_memory_llm_clients_cache.flush_cache() ll_logging._in_memory_loggers.clear() + ll_audit_logs._audit_log_callback_cache.clear() for attr in _LIST_ATTRS: if attr in _DEFAULTS: diff --git a/tests/ocr_tests/base_ocr_unit_tests.py b/tests/ocr_tests/base_ocr_unit_tests.py index 2120abc8a00..ae65efd952d 100644 --- a/tests/ocr_tests/base_ocr_unit_tests.py +++ b/tests/ocr_tests/base_ocr_unit_tests.py @@ -12,7 +12,15 @@ from abc import ABC, abstractmethod # Test resources TEST_IMAGE_PATH = "test_image_edit.png" -TEST_PDF_URL = "https://arxiv.org/pdf/2201.04234" +# Tiny in-repo PDF served via jsdelivr (sha-pinned, immutable). The arxiv +# PDF previously used here was several MB — once base64-encoded into the +# Vertex OCR request it ballooned cassettes past 100 MB per test. Keep +# the URL stable across runs so cassettes don't churn. +TEST_PDF_URL = ( + "https://cdn.jsdelivr.net/gh/BerriAI/litellm" + "@d769e81c90d453240c61fc572cdb27fae06a89d0" + "/tests/llm_translation/fixtures/dummy.pdf" +) class BaseOCRTest(ABC): diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 3332e77f2dd..70232f25c37 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -501,21 +501,30 @@ def test_is_request_body_safe_model_enabled( assert expect_error == error_raised -@pytest.mark.parametrize( - "api_key_value, expect_complete", - [ - ("sk-real-key", True), - ("", False), - (None, False), - (" ", False), - ], -) -def test_check_complete_credentials_api_key_values(api_key_value, expect_complete): +def _assert_check_complete_credentials(api_key_value, expect_complete): request_body = {"model": "gpt-3.5-turbo", "api_key": api_key_value} result = check_complete_credentials(request_body=request_body) assert result == expect_complete +def test_check_complete_credentials_with_real_key(): + _assert_check_complete_credentials( + api_key_value="sk-" + "x" * 8, expect_complete=True + ) + + +def test_check_complete_credentials_with_empty_string(): + _assert_check_complete_credentials(api_key_value="", expect_complete=False) + + +def test_check_complete_credentials_with_none(): + _assert_check_complete_credentials(api_key_value=None, expect_complete=False) + + +def test_check_complete_credentials_with_whitespace(): + _assert_check_complete_credentials(api_key_value=" ", expect_complete=False) + + def test_reading_openai_org_id_from_headers(): from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup diff --git a/tests/proxy_unit_tests/test_request_size_limit_middleware.py b/tests/proxy_unit_tests/test_request_size_limit_middleware.py new file mode 100644 index 00000000000..3e8792e0179 --- /dev/null +++ b/tests/proxy_unit_tests/test_request_size_limit_middleware.py @@ -0,0 +1,135 @@ +import pytest +from starlette.responses import JSONResponse +from starlette.testclient import TestClient +from starlette.types import Message + +from litellm.proxy.middleware.request_size_limit_middleware import ( + RequestSizeLimitMiddleware, +) + + +def test_request_size_limit_middleware_rejects_content_length_before_body_read(): + downstream_called = False + + async def app(scope, receive, send): + nonlocal downstream_called + downstream_called = True + response = JSONResponse({"ok": True}) + await response(scope, receive, send) + + client = TestClient( + RequestSizeLimitMiddleware( + app, + get_max_request_size_mb=lambda: 1, + is_request_size_limit_enabled=lambda: True, + ) + ) + + response = client.post( + "/chat/completions", + content=b"x" * (1024 * 1024 + 1), + headers={"content-type": "application/json"}, + ) + + assert response.status_code == 413 + assert response.json() == {"error": "Request size is too large. Max size is 1 MB"} + assert response.headers["content-length"] == str(len(response.content)) + assert downstream_called is False + + +def test_request_size_limit_middleware_zero_limit_disables_guard(): + downstream_called = False + + async def app(scope, receive, send): + nonlocal downstream_called + downstream_called = True + response = JSONResponse({"ok": True}) + await response(scope, receive, send) + + client = TestClient( + RequestSizeLimitMiddleware( + app, + get_max_request_size_mb=lambda: 0, + is_request_size_limit_enabled=lambda: True, + ) + ) + + response = client.post( + "/chat/completions", + content=b"x", + headers={"content-type": "application/json"}, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True} + assert downstream_called is True + + +@pytest.mark.asyncio +async def test_request_size_limit_middleware_rejects_streamed_body_without_content_length(): + received_body_bytes = 0 + + async def app(scope, receive, send): + nonlocal received_body_bytes + while True: + message = await receive() + if message["type"] == "http.disconnect": + break + received_body_bytes += len(message.get("body", b"")) + if not message.get("more_body", False): + break + + response = JSONResponse({"ok": True}) + await response(scope, receive, send) + + middleware = RequestSizeLimitMiddleware( + app, + get_max_request_size_mb=lambda: 1, + is_request_size_limit_enabled=lambda: True, + ) + sent_messages: list[Message] = [] + receive_messages: list[Message] = [ + { + "type": "http.request", + "body": b"x" * (1024 * 1024), + "more_body": True, + }, + { + "type": "http.request", + "body": b"y", + "more_body": False, + }, + ] + + async def receive(): + return receive_messages.pop(0) + + async def send(message): + sent_messages.append(message) + + await middleware( + { + "type": "http", + "method": "POST", + "path": "/chat/completions", + "headers": [(b"content-type", b"application/json")], + }, + receive, + send, + ) + + expected_body = b'{"error":"Request size is too large. Max size is 1 MB"}' + assert sent_messages[0] == { + "type": "http.response.start", + "status": 413, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(expected_body)).encode("latin-1")), + ], + } + assert sent_messages[1] == { + "type": "http.response.body", + "body": expected_body, + "more_body": False, + } + assert received_body_bytes == 1024 * 1024 diff --git a/tests/proxy_unit_tests/test_user_api_key_auth.py b/tests/proxy_unit_tests/test_user_api_key_auth.py index 543cabb6b4c..210347aaf94 100644 --- a/tests/proxy_unit_tests/test_user_api_key_auth.py +++ b/tests/proxy_unit_tests/test_user_api_key_auth.py @@ -269,9 +269,7 @@ async def test_aaauser_personal_budgets(key_ownership): test_user_cache = getattr(litellm.proxy.proxy_server, "user_api_key_cache") assert ( - test_user_cache.get_cache( - key=hash_token(user_key), model_type=UserAPIKeyAuth - ) + test_user_cache.get_cache(key=hash_token(user_key), model_type=UserAPIKeyAuth) == valid_token ) @@ -514,36 +512,59 @@ async def test_auth_not_connected_to_db(): assert valid_token.token == "failed-to-connect-to-db" -@pytest.mark.parametrize( - "headers, custom_header_name, expected_api_key", - [ - # Test with valid Bearer token - ({"x-custom-api-key": "Bearer sk-12345678"}, "x-custom-api-key", "sk-12345678"), - # Test with raw token (no Bearer prefix) - ({"x-custom-api-key": "Bearer sk-12345678"}, "x-custom-api-key", "sk-12345678"), - # Test with empty header value - ({"x-custom-api-key": ""}, "x-custom-api-key", ""), - # Test with missing header - ({}, "X-Custom-API-Key", ""), - # Test with different header casing - ({"X-CUSTOM-API-KEY": "Bearer sk-12345678"}, "X-Custom-API-Key", "sk-12345678"), - ], -) -def test_get_api_key_from_custom_header(headers, custom_header_name, expected_api_key): +def _assert_api_key_from_custom_header(headers, custom_header_name, expected_api_key): verbose_proxy_logger.setLevel(logging.DEBUG) - - # Mock the Request object request = MagicMock(spec=Request) request.headers = headers - - # Call the function and verify it doesn't raise an exception - api_key = get_api_key_from_custom_header( request=request, custom_litellm_key_header_name=custom_header_name ) assert api_key == expected_api_key +def test_get_api_key_from_custom_header_bearer_token(): + token = "sk-" + "1" * 8 + _assert_api_key_from_custom_header( + headers={"x-custom-api-key": f"Bearer {token}"}, + custom_header_name="x-custom-api-key", + expected_api_key=token, + ) + + +def test_get_api_key_from_custom_header_raw_token(): + token = "sk-" + "1" * 8 + _assert_api_key_from_custom_header( + headers={"x-custom-api-key": f"Bearer {token}"}, + custom_header_name="x-custom-api-key", + expected_api_key=token, + ) + + +def test_get_api_key_from_custom_header_empty_value(): + _assert_api_key_from_custom_header( + headers={"x-custom-api-key": ""}, + custom_header_name="x-custom-api-key", + expected_api_key="", + ) + + +def test_get_api_key_from_custom_header_missing_header(): + _assert_api_key_from_custom_header( + headers={}, + custom_header_name="X-Custom-API-Key", + expected_api_key="", + ) + + +def test_get_api_key_from_custom_header_different_casing(): + token = "sk-" + "1" * 8 + _assert_api_key_from_custom_header( + headers={"X-CUSTOM-API-KEY": f"Bearer {token}"}, + custom_header_name="X-Custom-API-Key", + expected_api_key=token, + ) + + from litellm.proxy._types import LitellmUserRoles diff --git a/tests/test_litellm/integrations/test_azure_sentinel.py b/tests/test_litellm/integrations/test_azure_sentinel.py index 031b85211f7..30b246202fc 100644 --- a/tests/test_litellm/integrations/test_azure_sentinel.py +++ b/tests/test_litellm/integrations/test_azure_sentinel.py @@ -2,13 +2,18 @@ Test Azure Sentinel logging integration """ -import datetime -from unittest.mock import AsyncMock, patch +import json +from unittest.mock import AsyncMock, MagicMock, patch import pytest from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger -from litellm.types.utils import StandardLoggingPayload +from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload + + +def _close_periodic_flush_task(coro): + coro.close() + return None @pytest.mark.asyncio @@ -20,7 +25,7 @@ async def test_azure_sentinel_oauth_and_send_batch(): test_client_id = "test-client-id" test_client_secret = "test-client-secret" - with patch("asyncio.create_task"): + with patch("asyncio.create_task", side_effect=_close_periodic_flush_task): logger = AzureSentinelLogger( dcr_immutable_id=test_dcr_id, endpoint=test_endpoint, @@ -42,9 +47,6 @@ async def test_azure_sentinel_oauth_and_send_batch(): # Add to queue logger.log_queue.append(standard_payload) - # Mock OAuth token response - from unittest.mock import MagicMock - mock_token_response = MagicMock() mock_token_response.status_code = 200 mock_token_response.json = MagicMock( @@ -91,3 +93,173 @@ async def test_azure_sentinel_oauth_and_send_batch(): # Verify queue is cleared assert len(logger.log_queue) == 0 + + +@pytest.mark.asyncio +async def test_azure_sentinel_queues_audit_log_event(): + """Test that Azure Sentinel supports direct audit log callbacks""" + with patch("asyncio.create_task", side_effect=_close_periodic_flush_task): + logger = AzureSentinelLogger( + dcr_immutable_id="dcr-test123456789", + endpoint="https://test-dce.eastus-1.ingest.monitor.azure.com", + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + ) + + logger.batch_size = 2 + logger.async_send_audit_batch = AsyncMock() + + audit_log = StandardAuditLogPayload( + id="audit-123", + updated_at="2026-05-06T04:39:00+00:00", + changed_by="user-1", + changed_by_api_key="sk-test", + action="created", + table_name="LiteLLM_TeamTable", + object_id="team-1", + before_value=None, + updated_values='{"team_alias": "sentinel-demo"}', + ) + + await logger.async_log_audit_log_event(audit_log) + + assert logger.audit_log_queue == [audit_log] + logger.async_send_audit_batch.assert_not_called() + + await logger.async_log_audit_log_event(audit_log) + + assert logger.audit_log_queue == [audit_log, audit_log] + logger.async_send_audit_batch.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_azure_sentinel_sends_audit_log_payload_to_ingestion_api(): + """Test that queued audit logs are sent to Azure Monitor Logs Ingestion""" + with patch("asyncio.create_task", side_effect=_close_periodic_flush_task): + logger = AzureSentinelLogger( + dcr_immutable_id="dcr-test123456789", + endpoint="https://test-dce.eastus-1.ingest.monitor.azure.com", + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + ) + + audit_log = StandardAuditLogPayload( + id="audit-123", + updated_at="2026-05-06T04:39:00+00:00", + changed_by="user-1", + changed_by_api_key="sk-test", + action="created", + table_name="LiteLLM_TeamTable", + object_id="team-1", + before_value=None, + updated_values='{"team_alias": "sentinel-demo"}', + ) + await logger.async_log_audit_log_event(audit_log) + + mock_token_response = MagicMock() + mock_token_response.status_code = 200 + mock_token_response.json = MagicMock( + return_value={ + "access_token": "test-bearer-token", + "expires_in": 3600, + } + ) + mock_token_response.text = "Success" + + mock_api_response = MagicMock() + mock_api_response.status_code = 204 + mock_api_response.text = "Success" + + async def mock_post(*args, **kwargs): + if "oauth2/v2.0/token" in kwargs.get("url", ""): + return mock_token_response + return mock_api_response + + logger.async_httpx_client.post = AsyncMock(side_effect=mock_post) + + await logger.flush_queue() + + api_call_args = logger.async_httpx_client.post.call_args_list[-1] + body = json.loads(api_call_args.kwargs["data"].decode("utf-8")) + assert body == [audit_log] + assert "dcr-test123456789" in api_call_args.kwargs["url"] + assert "Custom-LiteLLM" in api_call_args.kwargs["url"] + assert len(logger.audit_log_queue) == 0 + + +@pytest.mark.asyncio +async def test_azure_sentinel_flushes_standard_and_audit_logs_separately(): + """Test mixed callback roles do not send schema-mismatched batches.""" + with patch("asyncio.create_task", side_effect=_close_periodic_flush_task): + logger = AzureSentinelLogger( + dcr_immutable_id="dcr-test123456789", + stream_name="Custom-LiteLLM-Standard", + audit_stream_name="Custom-LiteLLM-Audit", + endpoint="https://test-dce.eastus-1.ingest.monitor.azure.com", + tenant_id="test-tenant-id", + client_id="test-client-id", + client_secret="test-client-secret", + ) + + standard_payload = StandardLoggingPayload( + id="standard-123", + call_type="completion", + model="gpt-3.5-turbo", + status="success", + messages=[{"role": "user", "content": "Hello"}], + response={"choices": [{"message": {"content": "Hi"}}]}, + ) + audit_log = StandardAuditLogPayload( + id="audit-123", + updated_at="2026-05-06T04:39:00+00:00", + changed_by="user-1", + changed_by_api_key="sk-test", + action="created", + table_name="LiteLLM_TeamTable", + object_id="team-1", + before_value=None, + updated_values='{"team_alias": "sentinel-demo"}', + ) + + logger.log_queue.append(standard_payload) + await logger.async_log_audit_log_event(audit_log) + + mock_token_response = MagicMock() + mock_token_response.status_code = 200 + mock_token_response.json = MagicMock( + return_value={ + "access_token": "test-bearer-token", + "expires_in": 3600, + } + ) + mock_token_response.text = "Success" + + mock_api_response = MagicMock() + mock_api_response.status_code = 204 + mock_api_response.text = "Success" + + async def mock_post(*args, **kwargs): + if "oauth2/v2.0/token" in kwargs.get("url", ""): + return mock_token_response + return mock_api_response + + logger.async_httpx_client.post = AsyncMock(side_effect=mock_post) + + await logger.flush_queue() + + ingestion_calls = [ + call + for call in logger.async_httpx_client.post.call_args_list + if "dataCollectionRules" in call.kwargs["url"] + ] + assert len(ingestion_calls) == 2 + + standard_call, audit_call = ingestion_calls + assert "Custom-LiteLLM-Standard" in standard_call.kwargs["url"] + assert json.loads(standard_call.kwargs["data"].decode("utf-8")) == [ + standard_payload + ] + assert "Custom-LiteLLM-Audit" in audit_call.kwargs["url"] + assert json.loads(audit_call.kwargs["data"].decode("utf-8")) == [audit_log] diff --git a/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py b/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py new file mode 100644 index 00000000000..99eb5abb7b5 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_custom_metadata_label_counts.py @@ -0,0 +1,159 @@ +import logging +import sys + +import pytest +from prometheus_client import REGISTRY + +import litellm +from litellm.integrations.prometheus import PrometheusLogger + + +def _clear_prometheus_registry() -> None: + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + REGISTRY.unregister(collector) + + +def _create_prometheus_logger_with_custom_labels(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + litellm, + "custom_prometheus_metadata_labels", + ["metadata.department", "metadata.environment"], + ) + _clear_prometheus_registry() + return PrometheusLogger() + + +def _standard_logging_payload_with_requester_metadata() -> dict: + return { + "model_id": "model-123", + "model_group": "gpt-4o-mini", + "api_base": "https://api.openai.com", + "custom_llm_provider": "openai", + "metadata": { + "user_api_key_hash": "test-hash", + "user_api_key_alias": "test-alias", + "user_api_key_team_id": "test-team", + "user_api_key_team_alias": "test-team-alias", + "user_api_key_user_id": "test-user", + "user_api_key_user_email": "test@example.com", + "user_api_key_org_id": None, + "requester_metadata": { + "department": "engineering", + "environment": "production", + }, + "user_api_key_auth_metadata": None, + "spend_logs_metadata": None, + }, + "request_tags": [], + "completion_tokens": 0, + "total_tokens": 0, + "response_cost": 0, + } + + +def _metric_samples(metric_name: str): + return [ + sample + for metric in REGISTRY.collect() + for sample in metric.samples + if sample.name == metric_name + ] + + +@pytest.mark.asyncio +async def test_async_log_failure_event_accepts_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +): + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + kwargs = { + "model": "gpt-4o-mini", + "litellm_params": { + "metadata": { + "user_api_key_end_user_id": "test-end-user", + } + }, + "standard_logging_object": _standard_logging_payload_with_requester_metadata(), + } + + with caplog.at_level(logging.ERROR): + await prometheus_logger.async_log_failure_event( + kwargs=kwargs, + response_obj=None, + start_time=None, + end_time=None, + ) + + assert "Incorrect label count" not in caplog.text + samples = _metric_samples("litellm_llm_api_failed_requests_metric_total") + assert any( + sample.labels.get("metadata_department") == "engineering" + and sample.labels.get("metadata_environment") == "production" + for sample in samples + ) + + +def test_virtual_key_rate_limit_metrics_accept_custom_metadata_labels( + monkeypatch: pytest.MonkeyPatch, +): + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + metadata = { + "model_group": "gpt-4o-mini", + "litellm-key-remaining-requests-gpt-4o-mini": 3, + "litellm-key-remaining-tokens-gpt-4o-mini": 200, + } + kwargs = { + "litellm_params": { + "metadata": metadata, + }, + "standard_logging_object": _standard_logging_payload_with_requester_metadata(), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + samples = _metric_samples("litellm_remaining_api_key_requests_for_model") + assert any( + sample.labels.get("metadata_department") == "engineering" + and sample.labels.get("metadata_environment") == "production" + and sample.value == 3 + for sample in samples + ) + + +def test_virtual_key_rate_limit_metrics_preserve_zero_remaining_values( + monkeypatch: pytest.MonkeyPatch, +): + prometheus_logger = _create_prometheus_logger_with_custom_labels(monkeypatch) + metadata = { + "model_group": "gpt-4o-mini", + "litellm-key-remaining-requests-gpt-4o-mini": 0, + "litellm-key-remaining-tokens-gpt-4o-mini": 0, + } + kwargs = { + "litellm_params": { + "metadata": metadata, + }, + "standard_logging_object": _standard_logging_payload_with_requester_metadata(), + } + + prometheus_logger._set_virtual_key_rate_limit_metrics( + user_api_key="test-hash", + user_api_key_alias="test-alias", + kwargs=kwargs, + metadata=metadata, + model_id="model-123", + ) + + request_samples = _metric_samples("litellm_remaining_api_key_requests_for_model") + token_samples = _metric_samples("litellm_remaining_api_key_tokens_for_model") + + assert any(sample.value == 0 for sample in request_samples) + assert any(sample.value == 0 for sample in token_samples) + assert not any(sample.value == sys.maxsize for sample in request_samples) + assert not any(sample.value == sys.maxsize for sample in token_samples) diff --git a/tests/test_litellm/integrations/test_prometheus_end_user_cardinality.py b/tests/test_litellm/integrations/test_prometheus_end_user_cardinality.py new file mode 100644 index 00000000000..868d86a6c24 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_end_user_cardinality.py @@ -0,0 +1,181 @@ +from time import monotonic + +import pytest +from prometheus_client import REGISTRY + +import litellm +from litellm.integrations.prometheus import PrometheusLogger +from litellm.integrations.prometheus_helpers import bounded_prometheus_series_tracker +from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import ( + BoundedPrometheusSeriesTracker, +) +from litellm.types.integrations.prometheus import UserAPIKeyLabelValues + + +@pytest.fixture(autouse=True) +def cleanup_prometheus_registry(): + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + try: + REGISTRY.unregister(collector) + except Exception: + pass + + old_enable_end_user = litellm.enable_end_user_cost_tracking_prometheus_only + old_metrics_config = litellm.prometheus_metrics_config + old_max_series = litellm.prometheus_end_user_metrics_max_series_per_metric + old_ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds + old_cleanup_interval_seconds = ( + litellm.prometheus_end_user_metrics_cleanup_interval_seconds + ) + + yield + + litellm.enable_end_user_cost_tracking_prometheus_only = old_enable_end_user + litellm.prometheus_metrics_config = old_metrics_config + litellm.prometheus_end_user_metrics_max_series_per_metric = old_max_series + litellm.prometheus_end_user_metrics_ttl_seconds = old_ttl_seconds + litellm.prometheus_end_user_metrics_cleanup_interval_seconds = ( + old_cleanup_interval_seconds + ) + + collectors = list(REGISTRY._collector_to_names.keys()) + for collector in collectors: + try: + REGISTRY.unregister(collector) + except Exception: + pass + + +def test_prometheus_end_user_series_are_capped_per_metric(): + litellm.enable_end_user_cost_tracking_prometheus_only = True + litellm.prometheus_metrics_config = [ + { + "group": "end-user-spend", + "metrics": ["litellm_spend_metric"], + "include_labels": ["end_user"], + } + ] + litellm.prometheus_end_user_metrics_max_series_per_metric = 3 + litellm.prometheus_end_user_metrics_ttl_seconds = None + logger = PrometheusLogger() + + for index in range(6): + PrometheusLogger._inc_labeled_counter( + logger, + logger.litellm_spend_metric, + "litellm_spend_metric", + UserAPIKeyLabelValues(end_user=f"end-user-{index}"), + amount=0.01, + ) + + assert len(logger.litellm_spend_metric._metrics) == 3 + assert set(logger.litellm_spend_metric._metrics) == { + ("end-user-3",), + ("end-user-4",), + ("end-user-5",), + } + + +def test_bounded_prometheus_series_tracker_is_label_agnostic(): + class FakeMetric: + def __init__(self): + self.removed_label_values = [] + + def remove(self, *label_values): + self.removed_label_values.append(label_values) + + metric = FakeMetric() + tracker = BoundedPrometheusSeriesTracker() + + for index in range(4): + tracker.track_series( + metric=metric, + metric_name="generic_metric", + label_values=(f"route-{index}", "200"), + max_series=2, + ttl_seconds=None, + cleanup_interval_seconds=60.0, + ) + + assert metric.removed_label_values == [ + ("route-0", "200"), + ("route-1", "200"), + ] + + +def test_bounded_prometheus_series_tracker_treats_zero_max_as_unlimited(): + # A misconfigured ``max_series=0`` must not silently evict every emission. + class FakeMetric: + def __init__(self): + self.removed_label_values = [] + + def remove(self, *label_values): + self.removed_label_values.append(label_values) + + metric = FakeMetric() + tracker = BoundedPrometheusSeriesTracker() + + for index in range(3): + tracker.track_series( + metric=metric, + metric_name="generic_metric", + label_values=(f"end-user-{index}",), + max_series=0, + ttl_seconds=None, + cleanup_interval_seconds=60.0, + ) + + assert metric.removed_label_values == [] + + +def test_prometheus_end_user_series_expire_by_ttl(monkeypatch): + litellm.enable_end_user_cost_tracking_prometheus_only = True + litellm.prometheus_metrics_config = [ + { + "group": "end-user-spend", + "metrics": ["litellm_spend_metric"], + "include_labels": ["end_user"], + } + ] + litellm.prometheus_end_user_metrics_max_series_per_metric = None + litellm.prometheus_end_user_metrics_ttl_seconds = 10.0 + litellm.prometheus_end_user_metrics_cleanup_interval_seconds = 0.0 + logger = PrometheusLogger() + + current_time = [monotonic()] + monkeypatch.setattr( + bounded_prometheus_series_tracker.time, + "monotonic", + lambda: current_time[0], + ) + PrometheusLogger._inc_labeled_counter( + logger, + logger.litellm_spend_metric, + "litellm_spend_metric", + UserAPIKeyLabelValues(end_user="stale-end-user"), + amount=0.01, + ) + + current_time[0] += 11.0 + PrometheusLogger._inc_labeled_counter( + logger, + logger.litellm_spend_metric, + "litellm_spend_metric", + UserAPIKeyLabelValues(end_user="fresh-end-user"), + amount=0.01, + ) + + assert set(logger.litellm_spend_metric._metrics) == {("fresh-end-user",)} + + +def test_prometheus_end_user_not_tracked_by_default(): + litellm.enable_end_user_cost_tracking_prometheus_only = None + labels = PrometheusLogger().get_labels_for_metric("litellm_spend_metric") + assert "end_user" in labels + + label_values = UserAPIKeyLabelValues(end_user="not-exported") + from litellm.integrations.prometheus import prometheus_label_factory + + prometheus_labels = prometheus_label_factory(labels, label_values) + assert prometheus_labels["end_user"] is None diff --git a/tests/test_litellm/integrations/test_s3_v2.py b/tests/test_litellm/integrations/test_s3_v2.py index 771002db92a..3f21de41c53 100644 --- a/tests/test_litellm/integrations/test_s3_v2.py +++ b/tests/test_litellm/integrations/test_s3_v2.py @@ -1123,3 +1123,74 @@ async def test_combined_prefix_reflects_in_s3_object_key(): result = logger.create_s3_batch_logging_element(datetime.utcnow(), payload) key = result.s3_object_key assert "myteam/apikey/" in key, f"Expected both prefixes in key: {key}" + + +# -------------------------------------------------------------- +# params_source / s3_callback_params_override (audit-log decoupling) +# -------------------------------------------------------------- +def test_s3_callback_params_override_uses_alternate_dict(): + """`s3_callback_params_override` makes the logger read its config from + the override dict instead of `litellm.s3_callback_params`.""" + import litellm + + original = litellm.s3_callback_params + litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} + try: + logger = S3Logger( + s3_callback_params_override={ + "s3_bucket_name": "audit-bucket", + "s3_path": "audit-prefix", + "s3_region_name": "us-west-2", + } + ) + assert logger.s3_bucket_name == "audit-bucket" + assert logger.s3_path == "audit-prefix" + assert logger.s3_region_name == "us-west-2" + finally: + litellm.s3_callback_params = original + + +def test_s3_callback_params_override_does_not_mutate_inputs(monkeypatch): + """Resolving `os.environ/X` markers must not mutate the override dict + or `litellm.s3_callback_params`.""" + import litellm + + monkeypatch.setenv("MY_AUDIT_BUCKET", "resolved-bucket") + override = {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"} + original_global = litellm.s3_callback_params + litellm.s3_callback_params = {"s3_bucket_name": "os.environ/MY_AUDIT_BUCKET"} + try: + logger = S3Logger(s3_callback_params_override=override) + assert logger.s3_bucket_name == "resolved-bucket" + assert override["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + assert ( + litellm.s3_callback_params["s3_bucket_name"] == "os.environ/MY_AUDIT_BUCKET" + ) + finally: + litellm.s3_callback_params = original_global + + +def test_s3_callback_params_override_none_falls_back_to_global(): + """No override → behaves exactly as today (reads `litellm.s3_callback_params`).""" + import litellm + + original = litellm.s3_callback_params + litellm.s3_callback_params = {"s3_bucket_name": "from-global"} + try: + logger = S3Logger() + assert logger.s3_bucket_name == "from-global" + finally: + litellm.s3_callback_params = original + + +def test_s3_callback_params_override_empty_dict_is_opt_in(): + """An empty override dict skips the global entirely (env/IAM-only config).""" + import litellm + + original = litellm.s3_callback_params + litellm.s3_callback_params = {"s3_bucket_name": "from-global"} + try: + logger = S3Logger(s3_callback_params_override={}) + assert logger.s3_bucket_name is None + finally: + litellm.s3_callback_params = original diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index cfcc426aa24..11d61d4c82e 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -157,14 +157,15 @@ class TestResponseCompliance: # Check CreateModelInteractionParams which includes output fields schema = spec_dict["components"]["schemas"]["CreateModelInteractionParams"] - # Output fields (readOnly) + # Output fields (readOnly). Google renamed `outputs` → `steps` in the + # upstream spec; keep this list aligned with the live schema. output_fields = [ "id", "status", "created", "updated", "role", - "outputs", + "steps", "usage", ] diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 22d2610eecb..99dbfd19f33 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -367,3 +367,128 @@ def test_update_messages_with_model_file_ids_skips_non_openai_file_blocks(): # Messages pass through unchanged when there is no `file` sub-dict to remap. assert updated == messages + + +# Reusable fixture (decodes to: litellm_proxy:application/pdf;unified_id,...; +# target_model_names,gpt-4o;llm_output_file_id,file-ECBPW7ML9g7XHdwGgUPZaM; +# llm_output_file_model_id,...) +UNIFIED_FILE_ID_B64 = ( + "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9wZGY7dW5pZmllZF9pZCw2YzBiNTg5MC04OTE0" + "LTQ4ZTAtYjhmNC0wYWU1ZWQzYzE0YTU7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00bztsbG1f" + "b3V0cHV0X2ZpbGVfaWQsZmlsZS1FQ0JQVzdNTDlnN1hIZHdHZ1VQWmFNO2xsbV9vdXRwdXRf" + "ZmlsZV9tb2RlbF9pZCxlMjY0NTNmOWU3NmU3OTkzNjgwZDAwNjhkOThjMWY0Y2MyMDViYmFk" + "MDk2N2EzM2M2NjQ4OTM1NjhjYTc0M2My" +) + + +def test_update_messages_with_model_file_ids_decodes_unified_id_when_mapping_empty(): + """When the mapping is empty (e.g. multi-replica cache miss), the function + must decode the base64-encoded unified file id and substitute the embedded + llm_output_file_id — mirroring the Responses-API sibling.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this recording?"}, + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "audio/wav", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "any-model-id", {}) + + assert updated[0]["content"][1]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + # Customer-supplied format is preserved (this is the field whose absence + # the misleading error message used to complain about). + assert updated[0]["content"][1]["file"]["format"] == "audio/wav" + + +def test_update_messages_with_model_file_ids_mapping_takes_precedence_over_decode(): + """When both mapping and decode would resolve, the mapping must win + (preserves per-deployment routing precision).""" + mapping = {UNIFIED_FILE_ID_B64: {"model-A": "mapped-provider-file-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": { + "file_id": UNIFIED_FILE_ID_B64, + "format": "application/pdf", + }, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "mapped-provider-file-id" + + +def test_update_messages_with_model_file_ids_non_unified_passes_through(): + """A raw provider id (e.g. gs:// URI or a random string) must be left + untouched when the mapping doesn't resolve it. The decode fallback must + not corrupt non-unified ids.""" + raw_id = "gs://my-bucket/uploads/abc-123.wav" + messages = [ + { + "role": "user", + "content": [ + {"type": "file", "file": {"file_id": raw_id, "format": "audio/wav"}}, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", {}) + + assert updated[0]["content"][0]["file"]["file_id"] == raw_id + + +def test_update_messages_with_model_file_ids_mapping_miss_falls_back_to_decode(): + """A mapping that exists but doesn't contain this file_id should still + trigger the decode fallback — covers the case where the hook resolved + *some* ids but not this one.""" + other_id = "some-other-file-id" + mapping = {other_id: {"model-A": "other-provider-id"}} + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": {"file_id": UNIFIED_FILE_ID_B64, "format": "audio/wav"}, + }, + ], + } + ] + + updated = update_messages_with_model_file_ids(messages, "model-A", mapping) + + assert updated[0]["content"][0]["file"]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" + + +def test_update_messages_with_model_file_ids_tolerates_non_dict_content_items(): + """Content list items aren't always dicts. text_completion forwards + token-ids (list of ints, or list of list of ints for batch) through + this path. The function must skip non-dict items instead of indexing + into them.""" + messages_token_ids = [{"role": "user", "content": [15496, 995]}] + messages_token_ids_batch = [{"role": "user", "content": [[15496, 995], [9906, 0]]}] + + # Both should pass through unchanged without raising. + assert ( + update_messages_with_model_file_ids(messages_token_ids, "model-A", {}) + == messages_token_ids + ) + assert ( + update_messages_with_model_file_ids(messages_token_ids_batch, "model-A", {}) + == messages_token_ids_batch + ) diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 8d842fefb7b..2b238b0cdf7 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -13,7 +13,10 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.litellm_core_utils.realtime_streaming import ( + RealTimeStreaming, + client_sent_openai_beta_realtime_header, +) from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ( OpenAIRealtimeStreamResponseBaseObject, @@ -80,6 +83,175 @@ def test_realtime_streaming_store_message(): assert len(streaming.messages) == 2 # Should not store the new message +def test_remap_beta_session_to_ga_normalizes_modalities_and_audio(): + out = RealTimeStreaming._remap_beta_session_to_ga( + {"modalities": ["audio", "text"], "voice": "alloy"} + ) + assert out["type"] == "realtime" + assert out["output_modalities"] == ["audio"] + assert out["audio"]["output"]["voice"] == "alloy" + + +def test_remap_beta_session_to_ga_preserves_ga_audio_format_dicts(): + input_format = {"type": "audio/pcm", "rate": 24000} + output_format = {"type": "audio/G711-ulaw", "rate": 8000} + + out = RealTimeStreaming._remap_beta_session_to_ga( + { + "input_audio_format": input_format, + "output_audio_format": output_format, + } + ) + + assert out["audio"]["input"]["format"] == input_format + assert out["audio"]["output"]["format"] == output_format + + +def test_make_disable_auto_response_message_produces_ga_shape(): + """_make_disable_auto_response_message must produce a GA-shaped session.update. + + The GA Realtime API requires: + - session.type = "realtime" + - turn_detection nested at session.audio.input.turn_detection + The old beta-style flat ``session.turn_detection`` is rejected by GA upstreams. + """ + websocket = MagicMock() + backend_ws = MagicMock() + logging_obj = MagicMock() + streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) + + raw = streaming._make_disable_auto_response_message() + msg = json.loads(raw) + + assert msg["type"] == "session.update" + session = msg["session"] + assert ( + session.get("type") == "realtime" + ), "GA session.update must include session.type='realtime'" + # turn_detection must NOT be at the flat beta location + assert ( + "turn_detection" not in session + ), "turn_detection must not be at the top-level session (beta shape); use audio.input" + # turn_detection must be nested under audio.input + assert session["audio"]["input"]["turn_detection"]["create_response"] is False + + +def test_make_disable_auto_response_message_produces_beta_shape_for_beta_clients(): + websocket = MagicMock() + websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + backend_ws = MagicMock() + logging_obj = MagicMock() + streaming = RealTimeStreaming(websocket, backend_ws, logging_obj) + + raw = streaming._make_disable_auto_response_message() + msg = json.loads(raw) + + assert msg["type"] == "session.update" + session = msg["session"] + assert session == {"turn_detection": {"create_response": False}} + + +@pytest.mark.asyncio +async def test_client_ack_messages_keeps_beta_session_shape_for_beta_clients(): + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + session_update = json.dumps( + { + "type": "session.update", + "session": { + "modalities": ["audio", "text"], + "voice": "alloy", + "turn_detection": {"create_response": False}, + }, + } + ) + client_ws.receive_text = AsyncMock( + side_effect=[ + session_update, + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + session = sent_to_backend["session"] + assert session["modalities"] == ["audio", "text"] + assert session["voice"] == "alloy" + assert session["turn_detection"] == {"create_response": False} + assert "type" not in session + assert "output_modalities" not in session + assert "audio" not in session + + +@pytest.mark.asyncio +async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): + client_ws = MagicMock() + session_update = json.dumps( + { + "type": "session.update", + "session": { + "modalities": ["audio", "text"], + "voice": "alloy", + "turn_detection": {"create_response": False}, + }, + } + ) + client_ws.receive_text = AsyncMock( + side_effect=[ + session_update, + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=True + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + session = sent_to_backend["session"] + assert session["modalities"] == ["audio", "text"] + assert session["voice"] == "alloy" + assert session["turn_detection"] == {"create_response": False} + assert "type" not in session + assert "output_modalities" not in session + assert "audio" not in session + + +def test_translate_event_to_beta_renames_delta_types(): + ev = RealTimeStreaming._translate_event_to_beta( + {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} + ) + assert ev is not None + assert ev["type"] == "response.audio.delta" + + +def test_translate_event_to_beta_drops_conversation_item_done(): + assert ( + RealTimeStreaming._translate_event_to_beta({"type": "conversation.item.done"}) + is None + ) + + +def test_client_sent_openai_beta_realtime_header_detects_header(): + ws = MagicMock() + ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + assert client_sent_openai_beta_realtime_header(ws) is True + empty = MagicMock() + empty.scope = {"headers": []} + assert client_sent_openai_beta_realtime_header(empty) is False + + def test_collect_user_input_from_text_conversation_item(): """ Test that conversation.item.create with input_text content is collected as user input. @@ -761,7 +933,14 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra assert ( len(session_updates) == 1 ), f"Expected one session.update injected to backend, got: {sent_to_backend}" - assert session_updates[0]["session"]["turn_detection"]["create_response"] is False + # GA shape: turn_detection must be nested under audio.input, not at top-level session + injected_session = session_updates[0]["session"] + assert ( + injected_session["type"] == "realtime" + ), "GA session.update must include session.type='realtime'" + assert ( + injected_session["audio"]["input"]["turn_detection"]["create_response"] is False + ), "GA session.update must nest turn_detection under audio.input" litellm.callbacks = [] # cleanup @@ -818,7 +997,14 @@ async def test_realtime_session_created_injects_session_update_for_pre_call_guar assert ( len(session_updates) == 1 ), f"pre_call guardrail should inject session.update to gate audio responses, got: {sent_to_backend}" - assert session_updates[0]["session"]["turn_detection"]["create_response"] is False + # GA shape: turn_detection must be nested under audio.input, not at top-level session + injected_session = session_updates[0]["session"] + assert ( + injected_session["type"] == "realtime" + ), "GA session.update must include session.type='realtime'" + assert ( + injected_session["audio"]["input"]["turn_detection"]["create_response"] is False + ), "GA session.update must nest turn_detection under audio.input" litellm.callbacks = [] # cleanup diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py index bf0461d89f1..2fdd639e74d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -1,7 +1,11 @@ +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import AsyncMock, MagicMock import pytest +import litellm from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.llms.anthropic.chat.handler import ModelResponseIterator, make_call from litellm.types.llms.openai import ( @@ -343,6 +347,289 @@ def test_text_only_streaming_has_index_zero(): ), f"Expected index=0, got {parsed.choices[0].index}" +def test_streaming_thinking_deltas_count_reasoning_tokens_in_usage(): + """Anthropic streaming usage should account for emitted thinking deltas.""" + chunks = [ + { + "type": "message_start", + "message": { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "thinking_delta", + "thinking": "First I need to count the favorable outcomes. ", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "thinking_delta", + "thinking": "Then I compare that count with all possible outcomes.", + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "sig_123"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": "The probability is 3/8."}, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 50}, + }, + ] + + iterator = ModelResponseIterator(None, sync_stream=True) + final_usage = None + reasoning_deltas = [] + + for chunk in chunks: + parsed = iterator.chunk_parser(chunk) + reasoning_content = getattr(parsed.choices[0].delta, "reasoning_content", None) + if reasoning_content: + reasoning_deltas.append(reasoning_content) + if parsed.usage is not None: + final_usage = parsed.usage + + assert reasoning_deltas == [ + "First I need to count the favorable outcomes. ", + "Then I compare that count with all possible outcomes.", + ] + assert final_usage is not None + completion_tokens_details = final_usage.completion_tokens_details + assert completion_tokens_details is not None + assert completion_tokens_details.reasoning_tokens > 0 + assert completion_tokens_details.text_tokens == ( + final_usage.completion_tokens - completion_tokens_details.reasoning_tokens + ) + + +def test_anthropic_completion_streaming_usage_matches_non_streaming_with_thinking(): + """The completion API should preserve Anthropic thinking usage in streaming mode.""" + thinking_parts = [ + "First I need to count the favorable outcomes. ", + "Then I compare that count with all possible outcomes.", + ] + thinking_text = "".join(thinking_parts) + answer_text = "The probability is 3/8." + requests_seen = [] + + class MockAnthropicHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, format, *args): # type: ignore[no-untyped-def] + return + + def do_POST(self): # type: ignore[no-untyped-def] + content_length = int(self.headers.get("content-length", "0")) + payload = json.loads(self.rfile.read(content_length).decode("utf-8")) + requests_seen.append( + { + "path": self.path, + "model": payload.get("model"), + "stream": payload.get("stream", False), + "thinking": payload.get("thinking"), + } + ) + + if payload.get("stream"): + events = [ + { + "type": "message_start", + "message": { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": payload.get("model"), + "content": [], + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "thinking_delta", + "thinking": thinking_parts[0], + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "thinking_delta", + "thinking": thinking_parts[1], + }, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": { + "type": "signature_delta", + "signature": "sig_mock", + }, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": ""}, + }, + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": answer_text}, + }, + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 50}, + }, + {"type": "message_stop"}, + ] + self._write_response( + content_type="text/event-stream", + body="".join( + f"data: {json.dumps(event)}\n\n" for event in events + ).encode("utf-8"), + ) + return + + self._write_response( + content_type="application/json", + body=json.dumps( + { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": payload.get("model"), + "content": [ + { + "type": "thinking", + "thinking": thinking_text, + "signature": "sig_mock", + }, + {"type": "text", "text": answer_text}, + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 50}, + } + ).encode("utf-8"), + ) + + def _write_response(self, content_type: str, body: bytes) -> None: + self.send_response(200) + self.send_header("content-type", content_type) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + server = ThreadingHTTPServer(("127.0.0.1", 0), MockAnthropicHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + request_kwargs = { + "model": "anthropic/claude-sonnet-4-6", + "api_base": f"http://127.0.0.1:{server.server_port}", + "api_key": "test", + "messages": [ + { + "role": "user", + "content": "Solve a probability problem and show thinking.", + } + ], + "thinking": {"type": "adaptive"}, + "max_tokens": 128, + } + + non_stream_response = litellm.completion(**request_kwargs, stream=False) + non_stream_details = non_stream_response.usage.completion_tokens_details + assert non_stream_details is not None + assert non_stream_details.reasoning_tokens > 0 + + reasoning_chunks = [] + content_chunks = [] + stream_usage = None + for chunk in litellm.completion( + **request_kwargs, + stream=True, + stream_options={"include_usage": True}, + ): + chunk_dict = chunk.model_dump(exclude_none=True) + choices = chunk_dict.get("choices") or [] + if choices: + delta = choices[0].get("delta") or {} + if delta.get("reasoning_content"): + reasoning_chunks.append(delta["reasoning_content"]) + if delta.get("content"): + content_chunks.append(delta["content"]) + if chunk_dict.get("usage"): + stream_usage = chunk_dict["usage"] + + assert reasoning_chunks == thinking_parts + assert content_chunks == [answer_text] + assert stream_usage is not None + stream_completion_details = stream_usage["completion_tokens_details"] + assert ( + stream_completion_details["reasoning_tokens"] + == non_stream_details.reasoning_tokens + ) + assert stream_completion_details["text_tokens"] == ( + stream_usage["completion_tokens"] + - stream_completion_details["reasoning_tokens"] + ) + assert requests_seen == [ + { + "path": "/v1/messages", + "model": "claude-sonnet-4-6", + "stream": False, + "thinking": {"type": "adaptive"}, + }, + { + "path": "/v1/messages", + "model": "claude-sonnet-4-6", + "stream": True, + "thinking": {"type": "adaptive"}, + }, + ] + finally: + server.shutdown() + + def test_text_and_tool_streaming_has_index_zero(): """Test that mixed text and tool streaming responses have choice index=0""" chunks = [ diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 26ed8d29c11..e38698c9100 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -97,6 +97,34 @@ def test_calculate_usage(): assert usage._cache_read_input_tokens == 0 +def test_calculate_usage_clamps_text_tokens_when_reasoning_estimate_exceeds_output(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 10, "output_tokens": 1}, + reasoning_content="This reasoning text intentionally tokenizes above one output token.", + ) + + assert usage.completion_tokens == 1 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == usage.completion_tokens + assert usage.completion_tokens_details.text_tokens == 0 + + +def test_calculate_usage_handles_mocked_output_tokens_with_reasoning_content(): + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 10, "output_tokens": MagicMock()}, + reasoning_content="mocked response reasoning", + ) + + assert usage.completion_tokens == 0 + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 0 + assert usage.completion_tokens_details.text_tokens == 0 + + @pytest.mark.parametrize( "usage_object,expected_usage", [ @@ -3914,3 +3942,764 @@ def test_strip_advisor_blocks_no_op_when_no_advisor_blocks(): original_content = [dict(b) for b in messages[1]["content"]] result = strip_advisor_blocks_from_messages(messages) assert result[1]["content"] == original_content + + +# --------------------------------------------------------------------------- +# Tool-name sanitization for Anthropic compatibility (^[a-zA-Z0-9_-]{1,128}$) +# Repro: Slack-bot agent sent an MCP tool named +# "github_openapi_mcp-actions/download-job-logs-for-workflow-run" which 400'd +# with `tools.N.custom.name: String should match pattern`. +# --------------------------------------------------------------------------- + + +def test_basic_sanitize_anthropic_tool_name_replaces_invalid_chars(): + from litellm.llms.anthropic.chat.transformation import ( + _basic_sanitize_anthropic_tool_name, + ) + + assert ( + _basic_sanitize_anthropic_tool_name( + "github_openapi_mcp-actions/download-job-logs-for-workflow-run" + ) + == "github_openapi_mcp-actions_download-job-logs-for-workflow-run" + ) + # other punctuation + assert _basic_sanitize_anthropic_tool_name("foo.bar:baz qux") == "foo_bar_baz_qux" + # already valid -> unchanged + assert _basic_sanitize_anthropic_tool_name("plain_tool-1") == "plain_tool-1" + # empty + assert _basic_sanitize_anthropic_tool_name("") == "" + # 128-char cap + long = "a/" * 200 + out = _basic_sanitize_anthropic_tool_name(long) + assert len(out) <= 128 + + +def test_build_anthropic_tool_name_maps_no_collisions(): + """Names that need rewriting go in the maps; valid names stay out.""" + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps( + [ + "fine_name", + "actions/download-job-logs-for-workflow-run", + "pulls/list-files", + ] + ) + assert forward == { + "actions/download-job-logs-for-workflow-run": ( + "actions_download-job-logs-for-workflow-run" + ), + "pulls/list-files": "pulls_list-files", + } + assert reverse == {v: k for k, v in forward.items()} + # untouched names absent + assert "fine_name" not in forward + assert "fine_name" not in reverse + + +def test_build_anthropic_tool_name_maps_disambiguates_collision_with_existing_valid(): + """If `foo/bar` would collapse to `foo_bar` but `foo_bar` already exists, + the rewritten one must get a unique suffix and only THAT one shows up in + the reverse map. The legitimately-named `foo_bar` round-trips identically.""" + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar"]) + # The original valid name keeps its slot. + assert "foo_bar" not in forward # untouched + # The rewritten one gets a disambiguating suffix. + assert forward["foo/bar"] == "foo_bar_2" + # Reverse map only has the rewritten entry. + assert reverse == {"foo_bar_2": "foo/bar"} + # CRITICAL: a legit `foo_bar` returned by the model must NOT round-trip + # to `foo/bar`. + assert "foo_bar" not in reverse + + +def test_build_anthropic_tool_name_maps_disambiguates_two_rewrites_to_same_target(): + """Two different invalid names that collapse to the same candidate must + both end up with unique sanitized forms.""" + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo.bar"]) + # First wins the canonical slot, second gets a suffix. + assert forward["foo/bar"] == "foo_bar" + assert forward["foo.bar"] == "foo_bar_2" + # Round-trip is unambiguous. + assert reverse["foo_bar"] == "foo/bar" + assert reverse["foo_bar_2"] == "foo.bar" + + +def test_build_anthropic_tool_name_maps_three_way_collision(): + """`foo/bar`, `foo.bar`, and an existing `foo_bar` must all coexist.""" + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps( + ["foo_bar", "foo/bar", "foo.bar"] + ) + assert "foo_bar" not in forward # untouched + assert forward["foo/bar"] == "foo_bar_2" + assert forward["foo.bar"] == "foo_bar_3" + # All three sanitized names are distinct. + sent_names = {"foo_bar", forward["foo/bar"], forward["foo.bar"]} + assert len(sent_names) == 3 + assert reverse == {"foo_bar_2": "foo/bar", "foo_bar_3": "foo.bar"} + + +def test_build_anthropic_tool_name_maps_reverse_order_collision(): + """REGRESSION: when the invalid name appears *before* the valid name that + its sanitized form collides with, both must still end up with distinct + names on the wire.""" + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo_bar"]) + # The valid name keeps its slot untouched. + assert "foo_bar" not in forward + # The rewritten one gets a disambiguating suffix. + assert forward["foo/bar"] == "foo_bar_2" + assert reverse == {"foo_bar_2": "foo/bar"} + assert "foo_bar" not in reverse + + +def test_build_anthropic_tool_name_maps_duplicate_originals(): + """REGRESSION: duplicate originals must not corrupt the forward map. + + Previously, the second occurrence of the same invalid name would + rewrite ``forward[original]`` to a suffixed name (``foo_bar_2``), + leaving ``foo_bar`` orphaned in ``used`` with no reverse mapping — + so when ``_sanitize_tool_names_in_request`` applied the forward + map, *both* tool entries got the suffixed name and Anthropic 400'd + on duplicates. + """ + from litellm.llms.anthropic.chat.transformation import ( + _build_anthropic_tool_name_maps, + ) + + forward, reverse = _build_anthropic_tool_name_maps(["foo/bar", "foo/bar"]) + # Same original sanitizes to the same target — no spurious suffix. + assert forward == {"foo/bar": "foo_bar"} + assert reverse == {"foo_bar": "foo/bar"} + + +def test_map_openai_params_does_not_pollute_optional_params_with_internal_keys(): + """REGRESSION: ``optional_params`` is what becomes the JSON body sent to + Anthropic (``data = {**optional_params}``). It MUST NOT carry LiteLLM- + internal coordination state like the per-request forward/reverse name + maps, or Anthropic 400s with ``Extra inputs are not permitted``. + Sanitization belongs in ``transform_request``, not here.""" + config = AnthropicConfig() + optional_params: dict = {} + config.map_openai_params( + non_default_params={ + "tools": [ + { + "type": "function", + "function": { + "name": "actions/download-job-logs-for-workflow-run", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + }, + optional_params=optional_params, + model="claude-sonnet-4", + drop_params=False, + ) + # No internal keys may appear in optional_params for ANY input. + for key in optional_params: + assert not key.startswith( + "_anthropic_tool_name" + ), f"optional_params leaked internal key {key!r}: {optional_params}" + # And no key starting with `_` either; optional_params should only + # contain documented Anthropic Messages API parameters. + for key in optional_params: + assert not key.startswith("_"), ( + f"optional_params leaked underscore-prefixed key {key!r}: " + f"{optional_params}" + ) + + +def test_map_openai_params_no_maps_when_all_names_already_valid(): + """Sanity check: an all-valid tool list adds nothing weird either.""" + config = AnthropicConfig() + optional_params: dict = {} + config.map_openai_params( + non_default_params={ + "tools": [ + { + "type": "function", + "function": { + "name": "plain_tool", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + }, + optional_params=optional_params, + model="claude-sonnet-4", + drop_params=False, + ) + for key in optional_params: + assert not key.startswith("_anthropic_tool_name") + + +def test_rewrite_tool_names_in_messages_uses_forward_map(): + config = AnthropicConfig() + forward_map = { + "actions/download-job-logs-for-workflow-run": ( + "actions_download-job-logs-for-workflow-run" + ) + } + messages = [ + {"role": "user", "content": "go"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "actions/download-job-logs-for-workflow-run", + "arguments": "{}", + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, + ] + + out = config._rewrite_tool_names_in_messages(messages, forward_map) + + # input list must not be mutated + assert ( + messages[1]["tool_calls"][0]["function"]["name"] + == "actions/download-job-logs-for-workflow-run" + ) + # output rewritten according to forward map + assert ( + out[1]["tool_calls"][0]["function"]["name"] + == "actions_download-job-logs-for-workflow-run" + ) + # non-tool-call messages pass through unchanged (same object) + assert out[0] is messages[0] + assert out[2] is messages[2] + + +def test_rewrite_tool_names_in_messages_leaves_unmapped_names_alone(): + """A tool_call name not in the forward map must NOT be rewritten, + even if it happens to look like a sanitized form of some other tool.""" + config = AnthropicConfig() + # `foo_bar` is NOT in the forward map (only `foo/bar` -> `foo_bar_2` is). + # If we naively re-sanitized, `foo_bar` would stay `foo_bar`, but more + # subtly, in a buggy implementation we might collide it with the codomain + # of some other rewrite. Either way: it must round-trip identically. + forward_map = {"foo/bar": "foo_bar_2"} + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "foo_bar", "arguments": "{}"}, + } + ], + }, + ] + out = config._rewrite_tool_names_in_messages(messages, forward_map) + assert out[0]["tool_calls"][0]["function"]["name"] == "foo_bar" + # input list must not be mutated either way + assert messages[0]["tool_calls"][0]["function"]["name"] == "foo_bar" + + +def test_rewrite_tool_names_in_messages_with_tool_calls_and_none_function_call(): + """When a message has tool_calls but function_call is explicitly None, + the rewrite must still apply to tool_calls and leave function_call as + None. Pins behavior at the boundary where ``new_msg = dict(msg)`` + copies the explicit-None key forward.""" + config = AnthropicConfig() + forward_map = {"foo/bar": "foo_bar"} + messages = [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "foo/bar", "arguments": "{}"}, + } + ], + "function_call": None, + }, + ] + out = config._rewrite_tool_names_in_messages(messages, forward_map) + assert out[0]["tool_calls"][0]["function"]["name"] == "foo_bar" + assert out[0]["function_call"] is None + # input list must not be mutated + assert messages[0]["tool_calls"][0]["function"]["name"] == "foo/bar" + + +def test_sanitize_tool_names_in_request_does_not_mutate_caller_tool_dicts(): + """REGRESSION: a caller reusing the same tool list/dicts across requests + must not see its inputs permanently rewritten. _sanitize_tool_names_in_request + builds a new list with copy-on-change entries.""" + config = AnthropicConfig() + original_name = "actions/download-job-logs-for-workflow-run" + caller_tool = { + "type": "custom", + "name": original_name, + "input_schema": {"type": "object", "properties": {}}, + } + caller_tools = [caller_tool] + optional_params: dict = {"tools": caller_tools} + + forward, reverse = config._sanitize_tool_names_in_request( + optional_params=optional_params + ) + + assert forward.get(original_name) + sanitized = forward[original_name] + assert optional_params["tools"][0]["name"] == sanitized + # caller's original dict + list must not be touched + assert caller_tool["name"] == original_name + assert caller_tools[0] is caller_tool + + +def test_transform_parsed_response_reverse_maps_tool_names(): + """End-to-end: rewritten tool name in Anthropic response -> original in OpenAI tool_calls.""" + import json as _json + + config = AnthropicConfig() + raw_response = MagicMock() + raw_response.headers = {} + raw_response.status_code = 200 + + completion_response = { + "id": "msg_x", + "model": "claude-sonnet-4", + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1}, + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "actions_download-job-logs-for-workflow-run", + "input": {"job_id": 123}, + } + ], + } + from litellm.types.utils import ModelResponse + + model_response = ModelResponse() + + out = config.transform_parsed_response( + completion_response=completion_response, + raw_response=raw_response, + model_response=model_response, + tool_name_reverse_map={ + "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run", + }, + ) + + tcs = out.choices[0].message.tool_calls + assert tcs is not None and len(tcs) == 1 + assert tcs[0].function.name == "actions/download-job-logs-for-workflow-run" + assert _json.loads(tcs[0].function.arguments) == {"job_id": 123} + + +def test_transform_parsed_response_does_not_rewrite_unmapped_names(): + """CRITICAL: a tool legitimately named `foo_bar` must NOT be rewritten + to `foo/bar` just because some other request had that pair. The reverse + map is per-request -- only entries we actually created go in it.""" + config = AnthropicConfig() + raw_response = MagicMock() + raw_response.headers = {} + raw_response.status_code = 200 + + # Caller registered `foo_bar` (valid) and `foo/bar` (rewrites to foo_bar_2). + # The reverse map only contains the rewrite. + reverse_map = {"foo_bar_2": "foo/bar"} + + completion_response = { + "id": "msg_x", + "model": "claude-sonnet-4", + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1}, + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "foo_bar", # the legit one, NOT in reverse map + "input": {}, + } + ], + } + from litellm.types.utils import ModelResponse + + model_response = ModelResponse() + out = config.transform_parsed_response( + completion_response=completion_response, + raw_response=raw_response, + model_response=model_response, + tool_name_reverse_map=reverse_map, + ) + # Must come back as-is, not rewritten to "foo/bar". + assert out.choices[0].message.tool_calls[0].function.name == "foo_bar" + + +def test_transform_parsed_response_no_reverse_map_is_noop(): + """When no map is provided, tool name is passed through unchanged.""" + config = AnthropicConfig() + raw_response = MagicMock() + raw_response.headers = {} + raw_response.status_code = 200 + + completion_response = { + "id": "msg_x", + "model": "claude-sonnet-4", + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1}, + "content": [ + { + "type": "tool_use", + "id": "toolu_1", + "name": "plain_tool", + "input": {}, + } + ], + } + from litellm.types.utils import ModelResponse + + model_response = ModelResponse() + out = config.transform_parsed_response( + completion_response=completion_response, + raw_response=raw_response, + model_response=model_response, + ) + assert out.choices[0].message.tool_calls[0].function.name == "plain_tool" + + +def test_streaming_iterator_reverse_maps_tool_use_name(): + """Streaming `content_block_start` for tool_use should reverse-map the name.""" + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + tool_name_reverse_map={ + "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run", + }, + ) + + chunk = { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_1", + "name": "actions_download-job-logs-for-workflow-run", + "input": {}, + }, + } + parsed = iterator.chunk_parser(chunk=chunk) + tool_calls = parsed.choices[0].delta.tool_calls + assert tool_calls is not None and len(tool_calls) == 1 + assert ( + tool_calls[0]["function"]["name"] + == "actions/download-job-logs-for-workflow-run" + ) + + +def test_streaming_iterator_passthrough_when_name_not_in_map(): + from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + iterator = ModelResponseIterator( + streaming_response=iter([]), + sync_stream=True, + tool_name_reverse_map=None, + ) + chunk = { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": "tool_use", + "id": "toolu_1", + "name": "plain_tool", + "input": {}, + }, + } + parsed = iterator.chunk_parser(chunk=chunk) + tool_calls = parsed.choices[0].delta.tool_calls + assert tool_calls is not None and len(tool_calls) == 1 + assert tool_calls[0]["function"]["name"] == "plain_tool" + + +# --------------------------------------------------------------------------- +# transform_request: end-to-end sanitization regression coverage +# --------------------------------------------------------------------------- + + +def _build_optional_params_for_tools(tools): + """Run a tools list through ``map_openai_params`` to get the same shape + ``transform_request`` will see from the router. Keeping this helper local + avoids duplicating the OpenAI->Anthropic param mapping in tests.""" + config = AnthropicConfig() + optional_params: dict = {} + config.map_openai_params( + non_default_params={"tools": tools}, + optional_params=optional_params, + model="claude-sonnet-4", + drop_params=False, + ) + return optional_params + + +def test_transform_request_does_not_leak_internal_keys_into_body(): + """REGRESSION for "_anthropic_tool_name_forward_map: Extra inputs are not + permitted". The dict returned by ``transform_request`` is what becomes + the JSON body POSTed to Anthropic. It must contain ONLY documented + Anthropic Messages fields -- no LiteLLM coordination state.""" + config = AnthropicConfig() + tools = [ + { + "type": "function", + "function": { + "name": "github_openapi_mcp-actions/download-job-logs-for-workflow-run", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + }, + { + "type": "function", + "function": { + "name": "plain_tool", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + }, + ] + optional_params = _build_optional_params_for_tools(tools) + litellm_params: dict = {} + + data = config.transform_request( + model="claude-sonnet-4", + messages=[{"role": "user", "content": "go"}], + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + # Body must not contain any LiteLLM-internal keys. + for key in data.keys(): + assert not key.startswith("_"), ( + f"transformed request body leaked underscore-prefixed key {key!r}; " + f"Anthropic will reject this with 'Extra inputs are not permitted'. " + f"body keys: {list(data.keys())}" + ) + + # Tool names in the body match Anthropic's pattern. + import re as _re + + for tool in data.get("tools", []): + name = tool.get("name") + assert isinstance(name, str) + assert _re.fullmatch( + r"[a-zA-Z0-9_-]{1,128}", name + ), f"sanitized tool name {name!r} still violates Anthropic regex" + + # Sent name for the bad tool is the disambiguated form, valid name passes through. + sent_names = {t["name"] for t in data["tools"]} + assert "github_openapi_mcp-actions_download-job-logs-for-workflow-run" in sent_names + assert "plain_tool" in sent_names + + # Reverse map landed on litellm_params (NOT optional_params, NOT body). + rmap = litellm_params["_anthropic_tool_name_map"] + assert ( + rmap["github_openapi_mcp-actions_download-job-logs-for-workflow-run"] + == "github_openapi_mcp-actions/download-job-logs-for-workflow-run" + ) + # The legitimately-named tool is not in the reverse map -- it round-trips + # untouched on the response side. + assert "plain_tool" not in rmap + + +def test_transform_request_no_reverse_map_when_all_names_valid(): + """If every name is already valid, ``litellm_params`` stays clean + (no reverse map key) -- minimizes blast radius for the common case.""" + config = AnthropicConfig() + tools = [ + { + "type": "function", + "function": { + "name": "plain_tool", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + }, + ] + optional_params = _build_optional_params_for_tools(tools) + litellm_params: dict = {} + + data = config.transform_request( + model="claude-sonnet-4", + messages=[{"role": "user", "content": "go"}], + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + assert data["tools"][0]["name"] == "plain_tool" + assert "_anthropic_tool_name_map" not in litellm_params + + +def test_transform_request_sanitizes_tool_choice_named_tool(): + """``tool_choice={"type": "function", "function": {"name": ""}}`` + must arrive at Anthropic as ``{"type": "tool", "name": ""}``, + matching the sanitized name in the tools array.""" + config = AnthropicConfig() + tools = [ + { + "type": "function", + "function": { + "name": "actions/download-job-logs-for-workflow-run", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + optional_params = AnthropicConfig().map_openai_params( + non_default_params={ + "tools": tools, + "tool_choice": { + "type": "function", + "function": {"name": "actions/download-job-logs-for-workflow-run"}, + }, + }, + optional_params={}, + model="claude-sonnet-4", + drop_params=False, + ) + litellm_params: dict = {} + data = config.transform_request( + model="claude-sonnet-4", + messages=[{"role": "user", "content": "go"}], + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + assert data["tool_choice"]["type"] == "tool" + assert data["tool_choice"]["name"] == "actions_download-job-logs-for-workflow-run" + assert data["tools"][0]["name"] == "actions_download-job-logs-for-workflow-run" + + +def test_transform_request_rewrites_tool_names_in_history(): + """Historical assistant messages with ``tool_calls`` referencing the bad + name must be rewritten to the sanitized form so Anthropic doesn't 400 on + ``tool_use.name`` mismatching the (sanitized) tools array.""" + config = AnthropicConfig() + tools = [ + { + "type": "function", + "function": { + "name": "actions/download-job-logs-for-workflow-run", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + optional_params = _build_optional_params_for_tools(tools) + messages = [ + {"role": "user", "content": "logs please"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_old", + "type": "function", + "function": { + "name": "actions/download-job-logs-for-workflow-run", + "arguments": "{}", + }, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_old", "content": "..."}, + {"role": "user", "content": "again"}, + ] + litellm_params: dict = {} + data = config.transform_request( + model="claude-sonnet-4", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + # Find the assistant tool_use block in the Anthropic-shaped messages. + tool_use_names = [] + for msg in data["messages"]: + content = msg.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + tool_use_names.append(block.get("name")) + assert ( + tool_use_names + ), "expected at least one tool_use block in transformed messages" + for name in tool_use_names: + assert name == "actions_download-job-logs-for-workflow-run", ( + f"history tool_use.name {name!r} not rewritten -- Anthropic will " + f"400 because it doesn't match the (sanitized) tools array" + ) + + +def test_sanitize_tool_names_in_request_skips_hosted_tools(): + """Hosted tools (web_search, computer_*, code_execution, ...) own + Anthropic-reserved names. The sanitizer must not enumerate them as + ``custom`` and must not rename them.""" + optional_params = { + "tools": [ + {"type": "web_search_20250305", "name": "web_search"}, + { + "type": "custom", + "name": "actions/download-job-logs-for-workflow-run", + "input_schema": {"type": "object", "properties": {}}, + }, + ], + } + forward, reverse = AnthropicConfig._sanitize_tool_names_in_request(optional_params) + # Only the custom tool was rewritten. + assert forward == { + "actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run" + } + assert reverse == { + "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run" + } + # Hosted tool's name unchanged. + assert optional_params["tools"][0]["name"] == "web_search" + # Custom tool's name updated in place. + assert ( + optional_params["tools"][1]["name"] + == "actions_download-job-logs-for-workflow-run" + ) + + +def test_sanitize_tool_names_in_request_no_tools_is_noop(): + """Empty / missing tools must not error or pollute return.""" + forward, reverse = AnthropicConfig._sanitize_tool_names_in_request({}) + assert forward == {} + assert reverse == {} + forward, reverse = AnthropicConfig._sanitize_tool_names_in_request({"tools": []}) + assert forward == {} + assert reverse == {} diff --git a/tests/test_litellm/llms/anthropic/test_message_sanitization.py b/tests/test_litellm/llms/anthropic/test_message_sanitization.py index a5f9c479d57..79ed321d0ee 100644 --- a/tests/test_litellm/llms/anthropic/test_message_sanitization.py +++ b/tests/test_litellm/llms/anthropic/test_message_sanitization.py @@ -339,6 +339,95 @@ class TestMessageSanitization: assert result[0]["role"] == "user" assert result[1]["role"] == "assistant" + def test_empty_string_content_sanitized_without_modify_params(self): + """ + Regression: An empty user message ({"role": "user", "content": ""}) must + be rewritten to a non-empty placeholder *before* it reaches Anthropic, + even when litellm.modify_params is False. Otherwise Anthropic returns: + "messages: text content blocks must be non-empty" + Reproduces a real failure from the pr-review agent (pydantic-ai). + """ + litellm.modify_params = False + + messages = [ + {"role": "user", "content": "First message"}, + {"role": "user", "content": "please review this"}, + {"role": "user", "content": ""}, + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + # All three user messages get merged into one user turn for Anthropic. + assert len(result) == 1 + assert result[0]["role"] == "user" + text_blocks = [ + b for b in result[0]["content"] if isinstance(b, dict) and b.get("type") == "text" + ] + assert len(text_blocks) == 3 + # No text block may be empty — that's the contract Anthropic enforces. + for block in text_blocks: + assert block["text"].strip() != "" + assert text_blocks[2]["text"] == ( + "[System: Empty message content sanitised to satisfy protocol]" + ) + + def test_empty_text_block_in_list_content_sanitized(self): + """ + Same regression for the list-of-blocks form: + {"role": "user", "content": [{"type": "text", "text": ""}]} + Empty text *blocks* must be rewritten too, regardless of modify_params. + """ + litellm.modify_params = False + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "real content"}, + {"type": "text", "text": ""}, + {"type": "text", "text": " \n "}, + ], + }, + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assert len(result) == 1 + text_blocks = [ + b for b in result[0]["content"] if isinstance(b, dict) and b.get("type") == "text" + ] + assert len(text_blocks) == 3 + assert text_blocks[0]["text"] == "real content" + for block in text_blocks[1:]: + assert block["text"].strip() != "" + + def test_non_empty_content_unchanged_without_modify_params(self): + """ + Sanity check: when nothing is empty, the messages flow through unchanged + even with modify_params disabled. + """ + litellm.modify_params = False + + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + {"role": "user", "content": "How are you?"}, + ] + + result = anthropic_messages_pt( + messages=messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + # Two user turns + one assistant turn (alternation preserved). + assert len(result) == 3 + assert result[0]["content"][0]["text"] == "Hello" + assert result[1]["content"][0]["text"] == "Hi there" + assert result[2]["content"][0]["text"] == "How are you?" + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py b/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py index 42108e46b59..41d301c5d5f 100644 --- a/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/test_litellm/llms/azure/realtime/test_azure_realtime_handler.py @@ -247,6 +247,10 @@ async def test_async_realtime_uses_ga_protocol_end_to_end(): assert "model=gpt-4o-realtime-preview" in called_url assert "api-version" not in called_url assert "deployment" not in called_url + assert ( + mock_realtime_streaming.call_args.kwargs["backend_uses_beta_protocol"] + is False + ) @pytest.mark.asyncio @@ -419,3 +423,7 @@ async def test_async_realtime_default_maintains_backwards_compatibility(): called_url = mock_ws_connect.call_args[0][0] assert "/openai/realtime?" in called_url assert "/openai/v1/realtime" not in called_url + assert ( + mock_realtime_streaming.call_args.kwargs["backend_uses_beta_protocol"] + is True + ) diff --git a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py index 55f810380b4..a4969e5dacc 100644 --- a/tests/test_litellm/llms/bedrock/test_base_aws_llm.py +++ b/tests/test_litellm/llms/bedrock/test_base_aws_llm.py @@ -18,7 +18,6 @@ from botocore.awsrequest import AWSPreparedRequest, AWSRequest from botocore.credentials import Credentials import litellm -from litellm.caching.caching import DualCache from litellm.llms.bedrock.base_aws_llm import ( AwsAuthError, BaseAWSLLM, @@ -32,6 +31,155 @@ BASE_AWS_LLM_PATH = os.path.join( ) +@pytest.fixture(autouse=True) +def flush_shared_bedrock_iam_cache(): + """Process-wide IAM cache must not leak static/env credential entries across tests.""" + BaseAWSLLM._shared_iam_cache.flush_cache() + yield + + +def test_base_aws_llm_instances_share_process_wide_iam_cache(): + """Regression LIT-2662: new instances must reuse iam_cache (Bedrock passthrough is per-request).""" + first = BaseAWSLLM() + second = BaseAWSLLM() + assert first.iam_cache is second.iam_cache + assert first.iam_cache is BaseAWSLLM._shared_iam_cache + + +def test_static_access_key_credentials_use_iam_cache_across_calls(): + """Static access-key path hits shared iam_cache; second identical call does not refetch.""" + base = BaseAWSLLM() + fake_creds = MagicMock() + + with patch.object( + base, + "_auth_with_access_key_and_secret_key", + return_value=(fake_creds, 3600), + ) as mock_static_auth: + base.get_credentials( + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="secret", + aws_region_name="us-east-1", + ) + base.get_credentials( + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="secret", + aws_region_name="us-east-1", + ) + mock_static_auth.assert_called_once() + + +def _os_environ_without_aws_keys() -> Dict[str, str]: + """Strip AWS_* so get_credentials hits the ambient-env branch when no explicit keys are passed.""" + return {k: v for k, v in os.environ.items() if not k.startswith("AWS_")} + + +def test_ambient_env_credentials_use_iam_cache_across_instances(): + """Else-branch env path uses shared iam_cache; second call on another instance does not refetch.""" + base_a = BaseAWSLLM() + base_b = BaseAWSLLM() + fake_creds = MagicMock() + with patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True): + with patch.object( + BaseAWSLLM, + "_auth_with_env_vars", + return_value=(fake_creds, None), + ) as mock_env: + base_a.get_credentials() + base_b.get_credentials() + mock_env.assert_called_once() + + +def test_static_access_key_path_boto3_session_constructed_once_when_cached(): + """With real _auth_with_access_key_and_secret_key, boto3.Session is only built once per cache key.""" + base_a = BaseAWSLLM() + base_b = BaseAWSLLM() + real_creds = Credentials("AKIAEXAMPLE", "secret-key-val", None) + mock_session_instance = MagicMock() + mock_session_instance.get_credentials.return_value = real_creds + with patch("boto3.Session", return_value=mock_session_instance) as mock_session_cls: + base_a.get_credentials( + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="secret-key-val", + aws_region_name="us-east-1", + ) + base_b.get_credentials( + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="secret-key-val", + aws_region_name="us-east-1", + ) + mock_session_cls.assert_called_once() + + +def test_ambient_env_path_boto3_session_constructed_once_when_cached(): + """Else branch: boto3.Session() inside _auth_with_env_vars runs once for two cache hits.""" + base_a = BaseAWSLLM() + base_b = BaseAWSLLM() + real_creds = Credentials("AKIAENV", "secret-env", None) + mock_session_instance = MagicMock() + mock_session_instance.get_credentials.return_value = real_creds + with patch.dict(os.environ, _os_environ_without_aws_keys(), clear=True): + with patch( + "boto3.Session", return_value=mock_session_instance + ) as mock_session_cls: + base_a.get_credentials() + base_b.get_credentials() + mock_session_cls.assert_called_once() + + +def test_explicit_session_token_tuple_not_cached_in_iam_cache(): + """Temporary key+secret+session paths must not use process-wide iam_cache between calls.""" + base = BaseAWSLLM() + with patch.object( + base, + "_auth_with_aws_session_token", + return_value=( + Credentials("ak", "sk", "token"), + None, + ), + ) as mock_sess: + base.get_credentials( + aws_access_key_id="AKIA", + aws_secret_access_key="sec", + aws_session_token="tok", + ) + base.get_credentials( + aws_access_key_id="AKIA", + aws_secret_access_key="sec", + aws_session_token="tok", + ) + assert mock_sess.call_count == 2 + + +def test_aws_profile_path_not_cached_in_iam_cache(): + base = BaseAWSLLM() + with patch.object( + base, + "_auth_with_aws_profile", + return_value=(Credentials("prof-ak", "prof-sk", None), None), + ) as mock_profile: + base.get_credentials(aws_profile_name="my-profile") + base.get_credentials(aws_profile_name="my-profile") + assert mock_profile.call_count == 2 + + +def test_web_identity_path_not_cached_in_iam_cache(): + base = BaseAWSLLM() + with patch.object( + base, + "_auth_with_web_identity_token", + return_value=(Credentials("wi-ak", "wi-sk", "wi-tok"), None), + ) as mock_wi: + kwargs = dict( + aws_web_identity_token="jwt-token", + aws_role_name="arn:aws:iam::123456789012:role/WebIdentity", + aws_session_name="web-id-session", + ) + base.get_credentials(**kwargs) + base.get_credentials(**kwargs) + assert mock_wi.call_count == 2 + + def test_boto3_init_tracer_wrapping(): """ Test that all boto3 initializations are wrapped in tracer.trace or @tracer.wrap @@ -48,6 +196,8 @@ def test_boto3_init_tracer_wrapping(): lines = content.split("\n") # Check each boto3 initialization is wrapped in tracer.trace for line_number, line in enumerate(lines, 1): + if line.lstrip().startswith("#"): + continue for pattern in boto3_init_patterns: if pattern in line: # Look back up to 5 lines for decorator or trace block @@ -544,7 +694,6 @@ def test_role_assumption_without_session_name(): # Mock the STS response with proper expiration handling mock_expiry = MagicMock() mock_expiry.tzinfo = timezone.utc - current_time = datetime.now(timezone.utc) # Create a timedelta object that returns 3600 when total_seconds() is called time_diff = MagicMock() time_diff.total_seconds.return_value = 3600 @@ -597,24 +746,61 @@ def test_role_assumption_without_session_name(): call_args = mock_sts_client.assume_role.call_args assert call_args[1]["RoleSessionName"] == "my-custom-session" - # Test case 3: Verify caching works with auto-generated session names - # Clear the cache first - base_aws_llm.iam_cache = DualCache() - + # Test case 3: AssumeRole is not stored in iam_cache; identical calls each invoke STS. + BaseAWSLLM._shared_iam_cache.flush_cache() mock_sts_client.reset_mock() with patch("boto3.client", return_value=mock_sts_client): - # First call credentials1 = base_aws_llm.get_credentials( aws_role_name="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole" ) - # Second call with same role should use cache (not call assume_role again) credentials2 = base_aws_llm.get_credentials( aws_role_name="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole" ) - # Should only be called once due to caching - assert mock_sts_client.assume_role.call_count == 1 + assert mock_sts_client.assume_role.call_count == 2 + assert credentials1.access_key == credentials2.access_key + + +def test_assume_role_path_does_not_use_process_iam_cache(): + """AssumeRole credentials are not cached; each get_credentials repeats STS AssumeRole.""" + base_aws_llm = BaseAWSLLM() + + mock_sts_client = MagicMock() + mock_expiry = MagicMock() + mock_expiry.tzinfo = timezone.utc + time_diff = MagicMock() + time_diff.total_seconds.return_value = 3600 + mock_expiry.__sub__ = MagicMock(return_value=time_diff) + + mock_sts_client.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "assumed-access-key", + "SecretAccessKey": "assumed-secret-key", + "SessionToken": "assumed-session-token", + "Expiration": mock_expiry, + } + } + mock_sts_client.get_caller_identity.return_value = { + "Arn": "arn:aws:sts::111111111111:assumed-role/SomeOtherRole/session-name", + } + + role_arn = "arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole" + env_without_irsa = { + k: v + for k, v in os.environ.items() + if k not in ("AWS_ROLE_ARN", "AWS_WEB_IDENTITY_TOKEN_FILE") + } + + with patch.dict(os.environ, env_without_irsa, clear=True): + with patch("boto3.client", return_value=mock_sts_client): + base_aws_llm.get_credentials(aws_role_name=role_arn) + mock_sts_client.get_caller_identity.reset_mock() + + base_aws_llm.get_credentials(aws_role_name=role_arn) + + mock_sts_client.get_caller_identity.assert_called() + assert mock_sts_client.assume_role.call_count == 2 def test_cache_keys_are_different_for_different_roles(): @@ -828,6 +1014,11 @@ def test_partial_credentials_still_use_ambient(): Test that if only one credential is provided, we still use ambient credentials. This handles edge cases where configuration might be incomplete. """ + env_without_aws_region = { + k: v + for k, v in os.environ.items() + if k not in ("AWS_REGION", "AWS_DEFAULT_REGION") + } base_aws_llm = BaseAWSLLM() # Mock the boto3 STS client @@ -850,37 +1041,43 @@ def test_partial_credentials_still_use_ambient(): } mock_sts_client.assume_role.return_value = mock_sts_response - with patch("boto3.client", return_value=mock_sts_client) as mock_boto3_client: + with patch.dict(os.environ, env_without_aws_region, clear=True): + with patch("boto3.client", return_value=mock_sts_client) as mock_boto3_client: - # Call with only access key (missing secret key) - credentials, ttl = base_aws_llm._auth_with_aws_role( - aws_access_key_id="AKIAEXAMPLE", - aws_secret_access_key=None, - aws_session_token=None, - aws_role_name="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole", - aws_session_name="test-session", - ) + # Call with only access key (missing secret key) + credentials, ttl = base_aws_llm._auth_with_aws_role( + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole", + aws_session_name="test-session", + ) - # Should still pass partial credentials to boto3.client - mock_boto3_client.assert_called_once_with( - "sts", - aws_access_key_id="AKIAEXAMPLE", - aws_secret_access_key=None, - aws_session_token=None, - verify=True, - ) + # Should still pass partial credentials to boto3.client + mock_boto3_client.assert_called_once_with( + "sts", + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key=None, + aws_session_token=None, + verify=True, + ) - # Should still call assume_role - mock_sts_client.assume_role.assert_called_once_with( - RoleArn="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole", - RoleSessionName="test-session", - ) + # Should still call assume_role + mock_sts_client.assume_role.assert_called_once_with( + RoleArn="arn:aws:iam::2222222222222:role/LitellmEvalBedrockRole", + RoleSessionName="test-session", + ) def test_cross_account_role_assumption(): """ Test assuming a role in a different AWS account (common in multi-account setups). """ + env_without_aws_region = { + k: v + for k, v in os.environ.items() + if k not in ("AWS_REGION", "AWS_DEFAULT_REGION") + } base_aws_llm = BaseAWSLLM() # Mock the boto3 STS client @@ -903,31 +1100,32 @@ def test_cross_account_role_assumption(): } mock_sts_client.assume_role.return_value = mock_sts_response - with patch("boto3.client", return_value=mock_sts_client) as mock_boto3_client: + with patch.dict(os.environ, env_without_aws_region, clear=True): + with patch("boto3.client", return_value=mock_sts_client) as mock_boto3_client: - # Assume role in different account (EKS/IRSA scenario) - credentials, ttl = base_aws_llm._auth_with_aws_role( - aws_access_key_id=None, - aws_secret_access_key=None, - aws_session_token=None, - aws_role_name="arn:aws:iam::999999999999:role/CrossAccountRole", - aws_session_name="cross-account-session", - ) + # Assume role in different account (EKS/IRSA scenario) + credentials, ttl = base_aws_llm._auth_with_aws_role( + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_role_name="arn:aws:iam::999999999999:role/CrossAccountRole", + aws_session_name="cross-account-session", + ) - # Should use ambient credentials - mock_boto3_client.assert_called_once_with("sts", verify=True) + # Should use ambient credentials + mock_boto3_client.assert_called_once_with("sts", verify=True) - # Should call assume_role with cross-account role - mock_sts_client.assume_role.assert_called_once_with( - RoleArn="arn:aws:iam::999999999999:role/CrossAccountRole", - RoleSessionName="cross-account-session", - ) + # Should call assume_role with cross-account role + mock_sts_client.assume_role.assert_called_once_with( + RoleArn="arn:aws:iam::999999999999:role/CrossAccountRole", + RoleSessionName="cross-account-session", + ) - # Verify cross-account credentials are returned - assert credentials.access_key == "cross-account-access-key" - assert credentials.secret_key == "cross-account-secret-key" - assert credentials.token == "cross-account-session-token" - assert ttl is not None + # Verify cross-account credentials are returned + assert credentials.access_key == "cross-account-access-key" + assert credentials.secret_key == "cross-account-secret-key" + assert credentials.token == "cross-account-session-token" + assert ttl is not None def test_role_assumption_with_custom_session_name(): @@ -1618,7 +1816,7 @@ def test_get_credentials_ecs_same_role_skips_assume_role(): base_aws_llm, "_is_already_running_as_role", return_value=True, - ): + ) as mock_already_running: with patch.object( base_aws_llm, "_auth_with_env_vars", @@ -1632,13 +1830,71 @@ def test_get_credentials_ecs_same_role_skips_assume_role(): aws_role_name="arn:aws:iam::123456789012:role/MyEcsTaskRole", aws_region_name="us-east-1", ) + base_aws_llm.get_credentials( + aws_role_name="arn:aws:iam::123456789012:role/MyEcsTaskRole", + aws_region_name="us-east-1", + ) - # Should use env vars, NOT role assumption + # Each get_credentials must check identity first; second call still checks before + # taking the iam_cache hit (no pre-peek that bypasses _is_already_running_as_role). + assert mock_already_running.call_count == 2 + # Cached env resolution: second call hits iam_cache, not _auth_with_env_vars again. mock_env_auth.assert_called_once() mock_role_auth.assert_not_called() assert credentials.access_key == "ecs-access-key" +def test_get_credentials_role_second_call_not_same_role_uses_assume_not_env_cache(): + """ + First request: already target role -> env path fills iam_cache. + Second request (e.g. identity changed): not same role -> AssumeRole path; must not reuse + cached env resolution from the first call. + """ + base_aws_llm = BaseAWSLLM() + + env_creds = MagicMock() + env_creds.access_key = "ambient-key" + env_creds.secret_key = "ambient-secret" + env_creds.token = "ambient-token" + + assumed_creds = MagicMock() + assumed_creds.access_key = "assumed-key" + assumed_creds.secret_key = "assumed-secret" + assumed_creds.token = "assumed-token" + + role_arn = "arn:aws:iam::123456789012:role/TargetRole" + + with patch.object( + base_aws_llm, + "_is_already_running_as_role", + side_effect=[True, False], + ) as mock_already: + with patch.object( + base_aws_llm, + "_auth_with_env_vars", + return_value=(env_creds, None), + ) as mock_env_auth: + with patch.object( + base_aws_llm, + "_auth_with_aws_role", + return_value=(assumed_creds, 3600), + ) as mock_role_auth: + first = base_aws_llm.get_credentials( + aws_role_name=role_arn, + aws_region_name="us-east-1", + ) + second = base_aws_llm.get_credentials( + aws_role_name=role_arn, + aws_region_name="us-east-1", + ) + + assert mock_already.call_count == 2 + mock_env_auth.assert_called_once() + mock_role_auth.assert_called_once() + assert first.access_key == "ambient-key" + assert second.access_key == "assumed-key" + + def test_parse_arn_account_and_role_name(): """Test the ARN parser helper for various ARN formats.""" parse = BaseAWSLLM._parse_arn_account_and_role_name diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index c356f866b07..8fa9290d3de 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -13,6 +13,110 @@ sys.path.insert( from litellm.llms.bedrock.common_utils import BedrockModelInfo +# --------------------------------------------------------------------------- # +# BEDROCK_RESPONSE_STREAM_SHAPE eager-load tests # +# --------------------------------------------------------------------------- # + + +def test_bedrock_response_stream_shape_loaded_at_import(): + """ + BEDROCK_RESPONSE_STREAM_SHAPE is resolved at module import time. + In a standard environment with botocore installed it must be non-None. + """ + from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE + + assert BEDROCK_RESPONSE_STREAM_SHAPE is not None + + +def test_bedrock_response_stream_shape_load_failure_returns_none(): + """ + If botocore's Loader raises (e.g. missing data files), _load_bedrock_response_stream_shape + should return None rather than propagating the exception, so the module + still imports cleanly. + """ + from unittest.mock import patch + + import litellm.llms.bedrock.common_utils as mod + + with patch( + "botocore.loaders.Loader.load_service_model", + side_effect=Exception("no data"), + ): + shape = mod._load_bedrock_response_stream_shape() + assert shape is None + + +def test_bedrock_response_stream_shape_is_structure_shape(): + """ + The loaded shape should be the botocore StructureShape for ResponseStream, + not a plain dict or any other type. + """ + from botocore.model import StructureShape + + from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE + + assert BEDROCK_RESPONSE_STREAM_SHAPE is not None, ( + "BEDROCK_RESPONSE_STREAM_SHAPE is None — botocore may not be installed" + ) + shape: StructureShape = BEDROCK_RESPONSE_STREAM_SHAPE # remove Optional + assert isinstance(shape, StructureShape) + assert shape.name == "ResponseStream" + + +def test_bedrock_response_stream_shape_same_object_across_imports(): + """ + Both bedrock modules that use the shape must reference the identical object — + confirming the constant is not re-loaded per import. + """ + from litellm.llms.bedrock.chat.invoke_handler import ( + BEDROCK_RESPONSE_STREAM_SHAPE as invoke_shape, + ) + from litellm.llms.bedrock.common_utils import ( + BEDROCK_RESPONSE_STREAM_SHAPE as common_shape, + ) + + assert common_shape is invoke_shape + + +def test_bedrock_event_stream_decoder_base_uses_module_shape(): + """ + BedrockEventStreamDecoderBase instances no longer carry their own + per-instance cache — _parse_message_from_event uses the module constant + directly, so there is no instance-level _response_stream_shape_cache attr. + """ + from litellm.llms.bedrock.common_utils import BedrockEventStreamDecoderBase + + decoder_a = BedrockEventStreamDecoderBase() + decoder_b = BedrockEventStreamDecoderBase() + + assert "_response_stream_shape_cache" not in decoder_a.__dict__ + assert "_response_stream_shape_cache" not in decoder_b.__dict__ + + +def test_bedrock_parse_message_from_event_raises_on_none_shape(): + """ + When BEDROCK_RESPONSE_STREAM_SHAPE is None (botocore unavailable), + _parse_message_from_event must raise BedrockError before touching the + botocore parser — not an opaque AttributeError from inside botocore. + """ + from unittest.mock import MagicMock, patch + + import litellm.llms.bedrock.common_utils as mod + from litellm.llms.bedrock.common_utils import BedrockError, BedrockEventStreamDecoderBase + + decoder = BedrockEventStreamDecoderBase() + mock_event = MagicMock() + + with patch.object(mod, "BEDROCK_RESPONSE_STREAM_SHAPE", None): + with pytest.raises(BedrockError) as exc_info: + decoder._parse_message_from_event(mock_event) + + assert exc_info.value.status_code == 500 + assert "botocore" in str(exc_info.value.message).lower() + # The botocore parser must never have been called + mock_event.to_response_dict.assert_not_called() + + def test_deepseek_cris(): """ Test that DeepSeek models with cross-region inference prefix use converse route diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index dd52304a703..26f50e8e492 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1,8 +1,10 @@ +import asyncio import io import os import pathlib import ssl import sys +import threading from unittest.mock import MagicMock, patch import certifi @@ -18,11 +20,111 @@ from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, + MaskedHTTPStatusError, _get_httpx_client, get_ssl_configuration, ) +@pytest.mark.asyncio +async def test_async_post_streaming_status_error_should_not_wait_forever_for_body( + monkeypatch, +): + """ + Vertex Anthropic streamRawPredict can return a pre-stream 4xx where the + streamed error body never terminates. The handler must still surface the + status promptly instead of blocking the downstream client. + """ + + class HangingErrorStream(httpx.AsyncByteStream): + async def __aiter__(self): + await asyncio.Event().wait() + if False: + yield b"" + + async def mock_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + request=request, + headers={"content-type": "application/json"}, + stream=HangingErrorStream(), + ) + + monkeypatch.setattr( + "litellm.llms.custom_httpx.http_handler._STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS", + 0.01, + ) + + litellm_handler = AsyncHTTPHandler() + await litellm_handler.client.aclose() + litellm_handler.client = httpx.AsyncClient( + transport=httpx.MockTransport(mock_handler) + ) + try: + with pytest.raises(MaskedHTTPStatusError) as exc_info: + await asyncio.wait_for( + litellm_handler.post( + "https://vertex.example/streamRawPredict", + stream=True, + ), + timeout=0.2, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.response.status_code == 400 + finally: + await litellm_handler.close() + + +def test_sync_post_streaming_status_error_should_not_wait_forever_for_body( + monkeypatch, +): + """ + Keep the sync streaming error path aligned with the async path so a + non-terminating streamed error body cannot block a worker thread forever. + """ + + class HangingSyncErrorStream(httpx.SyncByteStream): + def __init__(self): + self.closed_event = threading.Event() + + def __iter__(self): + self.closed_event.wait() + if False: + yield b"" + + def close(self): + self.closed_event.set() + + def mock_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + request=request, + headers={"content-type": "application/json"}, + stream=HangingSyncErrorStream(), + ) + + monkeypatch.setattr( + "litellm.llms.custom_httpx.http_handler._STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS", + 0.01, + ) + + litellm_handler = HTTPHandler() + litellm_handler.client.close() + litellm_handler.client = httpx.Client(transport=httpx.MockTransport(mock_handler)) + try: + with pytest.raises(MaskedHTTPStatusError) as exc_info: + litellm_handler.post( + "https://vertex.example/streamRawPredict", + stream=True, + ) + + assert exc_info.value.status_code == 400 + assert exc_info.value.response.status_code == 400 + finally: + litellm_handler.close() + + @pytest.mark.asyncio async def test_ssl_security_level(monkeypatch): # Ensure aiohttp transport is enabled for this test diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 4dc7c7a4054..29ab3790609 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -257,3 +257,65 @@ def test_hosted_vllm_thinking_blocks_with_list_content(): } assert assistant_msg["content"][2] == {"type": "text", "text": "Response text"} assert "thinking_blocks" not in assistant_msg + + +def test_hosted_vllm_custom_tools_are_converted_to_function_tools(): + config = HostedVLLMChatConfig() + optional_params = config.map_openai_params( + non_default_params={ + "tools": [ + { + "type": "custom", + "custom": { + "name": "apply_patch", + "description": "Apply text patch", + "format": { + "type": "grammar", + "grammar": {"syntax": "lark", "definition": "start: /.*/"}, + }, + }, + } + ] + }, + optional_params={}, + model="hosted_vllm/gpt-oss-120b", + drop_params=False, + ) + + tools = optional_params["tools"] + assert len(tools) == 1 + assert tools[0]["type"] == "function" + assert tools[0]["function"]["name"] == "apply_patch" + assert tools[0]["function"]["description"] == "Apply text patch" + assert tools[0]["function"]["parameters"]["type"] == "object" + assert "input" in tools[0]["function"]["parameters"]["properties"] + + +def test_hosted_vllm_custom_tools_use_top_level_input_schema(): + config = HostedVLLMChatConfig() + input_schema = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + } + optional_params = config.map_openai_params( + non_default_params={ + "tools": [ + { + "type": "custom", + "name": "search", + "description": "Search docs", + "input_schema": input_schema, + } + ] + }, + optional_params={}, + model="hosted_vllm/gpt-oss-120b", + drop_params=False, + ) + + tools = optional_params["tools"] + assert len(tools) == 1 + assert tools[0]["function"]["name"] == "search" + assert tools[0]["function"]["description"] == "Search docs" + assert tools[0]["function"]["parameters"] == input_schema diff --git a/tests/test_litellm/llms/nvidia_riva/__init__.py b/tests/test_litellm/llms/nvidia_riva/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/__init__.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py new file mode 100644 index 00000000000..0e355b91ca8 --- /dev/null +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_audio_utils.py @@ -0,0 +1,130 @@ +""" +Tests for the NVIDIA Riva audio resampling utility. + +The resampler turns arbitrary inbound audio (mp3/wav/m4a/...) into the wire +format Riva's gRPC ASR expects: 16 kHz mono LINEAR_PCM (int16 LE). +""" + +import io +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import soundfile as sf + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.nvidia_riva.audio_transcription.audio_utils import ( + resample_to_riva_pcm, +) +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException + + +def _wav_bytes(samples: np.ndarray, sample_rate: int) -> bytes: + buf = io.BytesIO() + sf.write(buf, samples, sample_rate, format="WAV", subtype="PCM_16") + return buf.getvalue() + + +def test_resample_24khz_stereo_to_16khz_mono_int16(): + sample_rate_in = 24000 + duration_seconds = 1.0 + n = int(sample_rate_in * duration_seconds) + t = np.linspace(0, duration_seconds, n, endpoint=False) + left = 0.5 * np.sin(2 * np.pi * 440.0 * t) + right = 0.5 * np.sin(2 * np.pi * 660.0 * t) + stereo = np.stack([left, right], axis=1).astype(np.float32) + + wav_in = _wav_bytes(stereo, sample_rate_in) + + resampled = resample_to_riva_pcm(wav_in) + + assert resampled.sample_rate_hz == 16000 + assert resampled.num_channels == 1 + # int16 = 2 bytes per sample + expected_samples = int(round(duration_seconds * 16000)) + assert len(resampled.pcm_bytes) == expected_samples * 2 + assert resampled.duration_seconds == pytest.approx(duration_seconds, abs=0.005) + + +def test_resample_16khz_mono_passes_through_int16_bytes_match_length(): + sample_rate = 16000 + n = sample_rate + samples = (0.1 * np.sin(np.linspace(0, 2 * np.pi * 200, n))).astype(np.float32) + wav_in = _wav_bytes(samples, sample_rate) + + resampled = resample_to_riva_pcm(wav_in) + + assert resampled.sample_rate_hz == 16000 + assert len(resampled.pcm_bytes) == n * 2 + assert resampled.duration_seconds == pytest.approx(1.0, abs=0.001) + + +def test_resample_preserves_int16_clip_range(): + sample_rate = 16000 + samples = np.array([2.0, -2.0, 0.0, 1.0], dtype=np.float32) + wav_in = _wav_bytes(samples, sample_rate) + + resampled = resample_to_riva_pcm(wav_in) + + decoded = np.frombuffer(resampled.pcm_bytes, dtype="= -32767 + + +def test_unknown_format_raises_clear_error(): + # 4 random bytes are not valid audio in any container we can decode. + with pytest.raises(NvidiaRivaException) as excinfo: + resample_to_riva_pcm(b"\x00\x01\x02\x03") + # Message must hint at what to do next. + assert "Riva STT" in excinfo.value.message + + +def test_audioread_fallback_writes_to_tempfile_path(monkeypatch): + """ + The audioread fallback handles compressed formats (mp3, m4a, ...). Most + audioread backends call into a subprocess (FFmpeg, GStreamer) and + require a real filesystem path — passing a BytesIO blows up with a + TypeError in subprocess.Popen. This test would have caught that bug: + we assert ``audio_open`` is called with a string path that points at a + file containing exactly the input bytes. + """ + payload = b"\xff\xfbfake-mp3-bytes-not-actually-decodable" + seen_paths = [] + + class FakeAudioSource: + samplerate = 22050 + channels = 1 + + def __iter__(self): + yield np.array([0, 0, 0, 0], dtype=np.int16).tobytes() + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def fake_audio_open(path): + assert isinstance(path, str), "audioread requires a filesystem path" + seen_paths.append(path) + with open(path, "rb") as fh: + assert fh.read() == payload + return FakeAudioSource() + + fake_audioread = SimpleNamespace(audio_open=fake_audio_open) + monkeypatch.setitem(sys.modules, "audioread", fake_audioread) + + fake_sf = MagicMock() + fake_sf.read.side_effect = RuntimeError("libsndfile cannot decode mp3") + monkeypatch.setitem(sys.modules, "soundfile", fake_sf) + + resampled = resample_to_riva_pcm(payload) + assert resampled.sample_rate_hz == 16000 + assert seen_paths and seen_paths[0].endswith(".audio") + # Tempfile must be cleaned up after decode. + assert not os.path.exists(seen_paths[0]) diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py new file mode 100644 index 00000000000..341a0e77ce0 --- /dev/null +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_handler.py @@ -0,0 +1,419 @@ +""" +End-to-end-ish tests for NvidiaRivaAudioTranscription. + +We mock ``riva.client`` so the test does not need the real gRPC SDK or a +running Riva server. The mock also lets us assert how Auth metadata is +constructed (NVCF vs self-hosted) and how the streaming generator output +is aggregated. +""" + +import asyncio +import io +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import soundfile as sf + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.nvidia_riva.audio_transcription import handler as handler_mod +from litellm.llms.nvidia_riva.audio_transcription.handler import ( + NvidiaRivaAudioTranscription, +) +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException +from litellm.types.utils import TranscriptionResponse + + +def _make_wav_bytes(seconds: float = 1.0, sample_rate: int = 16000) -> bytes: + n = int(sample_rate * seconds) + samples = (0.05 * np.sin(np.linspace(0, 2 * np.pi * 220 * seconds, n))).astype( + np.float32 + ) + buf = io.BytesIO() + sf.write(buf, samples, sample_rate, format="WAV", subtype="PCM_16") + return buf.getvalue() + + +def _fake_word(word: str, start_ms: int, end_ms: int): + return SimpleNamespace(word=word, start_time=start_ms, end_time=end_ms) + + +def _fake_alternative(transcript: str, words=None): + return SimpleNamespace(transcript=transcript, words=words or []) + + +def _fake_result(is_final: bool, alternatives): + return SimpleNamespace(is_final=is_final, alternatives=alternatives) + + +def _fake_response(results): + return SimpleNamespace(results=results) + + +@pytest.fixture +def mock_riva(monkeypatch): + """ + Stand-ins for the bits of ``riva.client`` the handler touches: + - ``Auth`` (constructor) + - ``ASRService`` with ``streaming_response_generator`` + - ``RecognitionConfig``, ``StreamingRecognitionConfig``, ``EndpointingConfig`` + - ``AudioEncoding`` namespace with ``LINEAR_PCM`` + """ + auth_calls = {} + + class FakeAuth: + def __init__(self, *args, **kwargs): + # Support both keyword and positional Auth constructors. + if kwargs: + auth_calls["uri"] = kwargs.get("uri") + auth_calls["use_ssl"] = kwargs.get("use_ssl") + auth_calls["metadata_args"] = kwargs.get("metadata_args") + else: + # positional: (None, use_ssl, uri, metadata) + auth_calls["use_ssl"] = args[1] if len(args) > 1 else None + auth_calls["uri"] = args[2] if len(args) > 2 else None + auth_calls["metadata_args"] = args[3] if len(args) > 3 else None + + class FakeRecognitionConfig: + def __init__(self, **kwargs): + self._kwargs = kwargs + self.endpointing_config = SimpleNamespace(CopyFrom=lambda _: None) + + class FakeStreamingRecognitionConfig: + def __init__(self, config, interim_results): + self.config = config + self.interim_results = interim_results + + class FakeEndpointingConfig: + def __init__(self, **kwargs): + self._kwargs = kwargs + + class FakeAudioEncoding: + LINEAR_PCM = "LINEAR_PCM" + + streaming_responses_holder = {"value": []} + + class FakeASRService: + def __init__(self, auth): + self.auth = auth + + def streaming_response_generator(self, audio_chunks, streaming_config): + # Drain audio_chunks generator so we exercise the chunking path. + list(audio_chunks) + yield from streaming_responses_holder["value"] + + fake_riva_client = SimpleNamespace( + Auth=FakeAuth, + ASRService=FakeASRService, + RecognitionConfig=FakeRecognitionConfig, + StreamingRecognitionConfig=FakeStreamingRecognitionConfig, + EndpointingConfig=FakeEndpointingConfig, + AudioEncoding=FakeAudioEncoding, + ) + + def fake_import_riva(): + return fake_riva_client, fake_riva_client + + monkeypatch.setattr(handler_mod, "_import_riva", fake_import_riva) + + return SimpleNamespace( + auth_calls=auth_calls, + responses=streaming_responses_holder, + client=fake_riva_client, + ) + + +@pytest.fixture +def logging_obj(): + return MagicMock() + + +def test_sync_path_aggregates_only_final_results(mock_riva, logging_obj): + mock_riva.responses["value"] = [ + # Empty heartbeat chunk: ignore. + _fake_response(results=[]), + # Interim chunk (not final): ignore. + _fake_response( + results=[ + _fake_result( + is_final=False, alternatives=[_fake_alternative("partial...")] + ) + ] + ), + # Two final chunks aggregated. + _fake_response( + results=[ + _fake_result( + is_final=True, + alternatives=[ + _fake_alternative( + "Hello,", + words=[_fake_word("Hello,", 0, 320)], + ) + ], + ) + ] + ), + _fake_response( + results=[ + _fake_result( + is_final=True, + alternatives=[ + _fake_alternative( + " world.", + words=[_fake_word("world.", 480, 870)], + ) + ], + ) + ] + ), + ] + + impl = NvidiaRivaAudioTranscription() + response: TranscriptionResponse = impl.audio_transcriptions( + model="nvidia/parakeet-ctc-1_1b-asr", + audio_file=_make_wav_bytes(), + optional_params={ + "language_code": "en-US", + "enable_word_time_offsets": True, + "response_format": "verbose_json", + "timestamp_granularities": ["word"], + }, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key="nvapi-xxx", + api_base="grpc.nvcf.nvidia.com:443", + ) + + assert response.text == "Hello, world." + # duration is propagated from the resampler. + assert response._hidden_params["audio_transcription_duration"] == pytest.approx( + 1.0, abs=0.05 + ) + # word timestamps converted from ms to seconds. + words = response["words"] + assert words[0]["start"] == pytest.approx(0.0) + assert words[1]["end"] == pytest.approx(0.87) + assert ( + logging_obj.pre_call.call_args.kwargs["additional_args"]["atranscription"] + is False + ) + + +def test_auth_nvcf_defaults_use_ssl_and_attaches_function_id(mock_riva, logging_obj): + mock_riva.responses["value"] = [ + _fake_response( + results=[ + _fake_result( + is_final=True, + alternatives=[_fake_alternative("ok")], + ) + ] + ) + ] + impl = NvidiaRivaAudioTranscription() + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={ + "nvcf_function_id": "abc-123", + "language_code": "en-US", + }, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key="nvapi-xxx", + api_base="grpc.nvcf.nvidia.com:443", + ) + + assert mock_riva.auth_calls["uri"] == "grpc.nvcf.nvidia.com:443" + assert mock_riva.auth_calls["use_ssl"] is True + metadata = dict(mock_riva.auth_calls["metadata_args"]) + assert metadata["function-id"] == "abc-123" + assert metadata["authorization"] == "Bearer nvapi-xxx" + + +def test_auth_self_hosted_defaults_no_ssl_and_no_function_id(mock_riva, logging_obj): + mock_riva.responses["value"] = [ + _fake_response( + results=[ + _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) + ] + ) + ] + impl = NvidiaRivaAudioTranscription() + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert mock_riva.auth_calls["uri"] == "localhost:50051" + assert mock_riva.auth_calls["use_ssl"] is False + metadata = dict(mock_riva.auth_calls["metadata_args"]) + # No function-id, no authorization metadata. + assert "function-id" not in metadata + assert "authorization" not in metadata + + +def test_explicit_use_ssl_override_wins(mock_riva, logging_obj): + """ + Self-hosted Riva behind an ingress with TLS termination is a real + deployment topology. ``use_ssl=True`` must be honored even without an + NVCF function id. + """ + mock_riva.responses["value"] = [ + _fake_response( + results=[ + _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) + ] + ) + ] + impl = NvidiaRivaAudioTranscription() + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"use_ssl": True, "language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="riva.internal.company.com:443", + ) + + assert mock_riva.auth_calls["use_ssl"] is True + + +def test_missing_api_base_raises_clear_error(mock_riva, logging_obj): + impl = NvidiaRivaAudioTranscription() + with pytest.raises(NvidiaRivaException) as excinfo: + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base=None, + ) + assert "api_base" in excinfo.value.message + + +def test_async_path_uses_to_thread(mock_riva, logging_obj): + mock_riva.responses["value"] = [ + _fake_response( + results=[ + _fake_result( + is_final=True, alternatives=[_fake_alternative("async ok")] + ) + ] + ) + ] + impl = NvidiaRivaAudioTranscription() + response = asyncio.run( + impl.async_audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + ) + assert response.text == "async ok" + assert ( + logging_obj.pre_call.call_args.kwargs["additional_args"]["atranscription"] + is True + ) + + +def test_timeout_is_forwarded_to_streaming_generator_when_supported( + mock_riva, logging_obj +): + """ + Without a deadline the gRPC stream can block forever on a stalled Riva + server. The handler must forward the call-level ``timeout`` to + ``streaming_response_generator`` whenever the installed riva-client + accepts a ``timeout`` kwarg. + """ + captured_kwargs = {} + + def streaming_with_timeout(self, audio_chunks, streaming_config, timeout=None): + captured_kwargs["timeout"] = timeout + list(audio_chunks) + yield from [ + _fake_response( + results=[ + _fake_result(is_final=True, alternatives=[_fake_alternative("ok")]) + ] + ) + ] + + mock_riva.client.ASRService.streaming_response_generator = streaming_with_timeout + + impl = NvidiaRivaAudioTranscription() + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=12.5, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + assert captured_kwargs["timeout"] == pytest.approx(12.5) + + +def test_grpc_error_is_wrapped_as_nvidia_riva_exception(mock_riva, logging_obj): + class FakeGrpcError(Exception): + def code(self): + return SimpleNamespace(name="UNAUTHENTICATED") + + def details(self): + return "bad token" + + def raising_streaming_response_generator(self, audio_chunks, streaming_config): + list(audio_chunks) + raise FakeGrpcError("rpc fail") + + mock_riva.client.ASRService.streaming_response_generator = ( + raising_streaming_response_generator + ) + + impl = NvidiaRivaAudioTranscription() + with pytest.raises(NvidiaRivaException) as excinfo: + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key="nvapi-xxx", + api_base="grpc.nvcf.nvidia.com:443", + ) + + assert excinfo.value.status_code == 401 + assert "UNAUTHENTICATED" in excinfo.value.message diff --git a/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py new file mode 100644 index 00000000000..c4cca8490bf --- /dev/null +++ b/tests/test_litellm/llms/nvidia_riva/audio_transcription/test_transformation.py @@ -0,0 +1,275 @@ +""" +Unit tests for NvidiaRivaAudioTranscriptionConfig. + +These tests do not require ``nvidia-riva-client`` or any audio libs to be +installed; the transformation layer is intentionally pure-Python on dicts. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, +) +from litellm.llms.nvidia_riva.audio_transcription.transformation import ( + NvidiaRivaAudioTranscriptionConfig, +) +from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException + + +@pytest.fixture +def cfg(): + return NvidiaRivaAudioTranscriptionConfig() + + +def test_supported_openai_params(cfg): + params = cfg.get_supported_openai_params(model="nvidia/parakeet-ctc-1_1b-asr") + assert "language" in params + assert "response_format" in params + assert "timestamp_granularities" in params + + +def test_map_language_normalizes_bare_codes(cfg): + out = cfg.map_openai_params( + non_default_params={"language": "en"}, + optional_params={}, + model="m", + drop_params=False, + ) + assert out["language_code"] == "en-US" + + +def test_map_language_passes_through_bcp47(cfg): + out = cfg.map_openai_params( + non_default_params={"language": "de-DE"}, + optional_params={}, + model="m", + drop_params=False, + ) + assert out["language_code"] == "de-DE" + + +def test_map_language_es_defaults_to_castilian_spain(cfg): + """ + Bare ``es`` is ISO-639 Spanish; in BCP-47 it conventionally resolves to + es-ES (Castilian / Spain), not es-US. Routing every Spanish caller to a + US-tuned Riva model would silently degrade accuracy. + """ + out = cfg.map_openai_params( + non_default_params={"language": "es"}, + optional_params={}, + model="m", + drop_params=False, + ) + assert out["language_code"] == "es-ES" + + +def test_map_timestamp_granularities_word_enables_word_offsets(cfg): + out = cfg.map_openai_params( + non_default_params={"timestamp_granularities": ["word"]}, + optional_params={}, + model="m", + drop_params=False, + ) + assert out["enable_word_time_offsets"] is True + assert out["timestamp_granularities"] == ["word"] + + +def test_map_timestamp_granularities_segment_only_does_not_enable_word_offsets(cfg): + out = cfg.map_openai_params( + non_default_params={"timestamp_granularities": ["segment"]}, + optional_params={}, + model="m", + drop_params=False, + ) + assert "enable_word_time_offsets" not in out + + +def test_transform_request_builds_recognition_config(cfg): + result = cfg.transform_audio_transcription_request( + model="nvidia/parakeet-ctc-1_1b-asr", + audio_file=b"fake-audio", + optional_params={ + "language_code": "en-US", + "enable_word_time_offsets": True, + "nvcf_function_id": "abc-123", + "use_ssl": True, + "riva_model_name": "parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer", + }, + litellm_params={ + "api_base": "grpc.nvcf.nvidia.com:443", + "api_key": "nvapi-xxx", + }, + ) + + assert isinstance(result, AudioTranscriptionRequestData) + payload = result.data + assert payload["recognition_config"]["language_code"] == "en-US" + assert payload["recognition_config"]["sample_rate_hertz"] == 16000 + assert payload["recognition_config"]["audio_channel_count"] == 1 + assert payload["recognition_config"]["encoding"] == "LINEAR_PCM" + assert payload["recognition_config"]["enable_word_time_offsets"] is True + assert ( + payload["recognition_config"]["model"] + == "parakeet-1.1b-en-US-asr-streaming-silero-vad-sortformer" + ) + assert "audio_file" not in payload + assert "auth" not in payload + + +def test_transform_request_default_riva_model_is_empty_for_auto_select(cfg): + """ + Riva auto-selects the deployed model when ``model`` is empty. This is + the right default because internal NVIDIA deployment names change + across versions/regions. + """ + result = cfg.transform_audio_transcription_request( + model="nvidia/parakeet-ctc-1_1b-asr", + audio_file=b"fake-audio", + optional_params={"language_code": "en-US"}, + litellm_params={"api_base": "grpc.nvcf.nvidia.com:443"}, + ) + assert result.data["recognition_config"]["model"] == "" + + +def test_chunking_strategy_server_vad_maps_to_endpointing_config(cfg): + result = cfg.transform_audio_transcription_request( + model="m", + audio_file=b"x", + optional_params={ + "chunking_strategy": { + "type": "server_vad", + "threshold": 0.5, + "silence_duration_ms": 700, + "prefix_padding_ms": 250, + } + }, + litellm_params={"api_base": "localhost:50051"}, + ) + ep = result.data["recognition_config"].get("endpointing_config") + assert ep is not None + assert ep["start_threshold"] == 0.5 + assert ep["stop_threshold"] == 0.5 + assert ep["stop_history"] == 700 + assert ep["stop_history_eou"] == 250 + + +def test_chunking_strategy_auto_leaves_endpointing_config_unset(cfg): + result = cfg.transform_audio_transcription_request( + model="m", + audio_file=b"x", + optional_params={"chunking_strategy": "auto"}, + litellm_params={"api_base": "localhost:50051"}, + ) + assert "endpointing_config" not in result.data["recognition_config"] + + +def test_explicit_endpointing_config_pass_through(cfg): + result = cfg.transform_audio_transcription_request( + model="m", + audio_file=b"x", + optional_params={ + "endpointing_config": {"stop_history": 1200, "start_threshold": 0.3} + }, + litellm_params={"api_base": "localhost:50051"}, + ) + ep = result.data["recognition_config"]["endpointing_config"] + assert ep == {"stop_history": 1200, "start_threshold": 0.3} + + +def test_build_transcription_response_text_format(): + final_results = [ + {"transcript": "Hello,", "words": []}, + {"transcript": " this is parakeet.", "words": []}, + ] + response = NvidiaRivaAudioTranscriptionConfig.build_transcription_response( + final_results=final_results, + response_format="json", + duration_seconds=2.4, + timestamp_granularities=None, + ) + assert response.text == "Hello, this is parakeet." + assert response["task"] == "transcribe" + # duration is only attached for verbose_json + assert "duration" not in response + + +def test_build_transcription_response_skips_empty_chunks(): + final_results = [ + {"transcript": "", "words": []}, + {"transcript": "actual content", "words": []}, + {"transcript": "", "words": []}, + ] + response = NvidiaRivaAudioTranscriptionConfig.build_transcription_response( + final_results=final_results, + response_format="json", + duration_seconds=1.0, + timestamp_granularities=None, + ) + assert response.text == "actual content" + + +def test_build_transcription_response_verbose_json_with_words(): + final_results = [ + { + "transcript": "Hello,", + "words": [ + {"word": "Hello,", "start_time_ms": 0, "end_time_ms": 320}, + ], + }, + { + "transcript": " world.", + "words": [ + {"word": "world.", "start_time_ms": 480, "end_time_ms": 870}, + ], + }, + ] + response = NvidiaRivaAudioTranscriptionConfig.build_transcription_response( + final_results=final_results, + response_format="verbose_json", + duration_seconds=2.475, + timestamp_granularities=["word"], + ) + + assert response.text == "Hello, world." + assert response["duration"] == 2.475 + words = response["words"] + assert words[0]["word"] == "Hello," + # Riva returns ms; OpenAI exposes seconds. + assert words[0]["start"] == pytest.approx(0.0) + assert words[0]["end"] == pytest.approx(0.32) + assert words[1]["start"] == pytest.approx(0.48) + assert words[1]["end"] == pytest.approx(0.87) + + +def test_build_transcription_response_verbose_json_without_word_granularity_omits_words(): + final_results = [ + { + "transcript": "Hi.", + "words": [ + {"word": "Hi.", "start_time_ms": 0, "end_time_ms": 200}, + ], + } + ] + response = NvidiaRivaAudioTranscriptionConfig.build_transcription_response( + final_results=final_results, + response_format="verbose_json", + duration_seconds=0.2, + timestamp_granularities=["segment"], + ) + assert "words" not in response + + +def test_transform_response_not_used_raises_clear_error(cfg): + with pytest.raises(NotImplementedError): + cfg.transform_audio_transcription_response(raw_response=None) # type: ignore[arg-type] + + +def test_get_error_class_returns_nvidia_riva_exception(cfg): + err = cfg.get_error_class(error_message="bad", status_code=401, headers={}) + assert isinstance(err, NvidiaRivaException) + assert err.status_code == 401 diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 4f5764e3d6e..e9798f45dce 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -213,12 +213,12 @@ async def test_async_realtime_url_contains_model(): assert called_url.startswith("wss://api.openai.com/v1/realtime?") assert f"model={model}" in called_url - # Verify proper headers were set + # Verify proper headers were set (GA default: no OpenAI-Beta unless client sent it) called_kwargs = mock_ws_connect.call_args[1] assert "additional_headers" in called_kwargs additional_headers = called_kwargs["additional_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" - assert additional_headers["OpenAI-Beta"] == "realtime=v1" + assert "OpenAI-Beta" not in additional_headers # Verify SSL is configured (should be an SSLContext or True, not None or False) assert called_kwargs["ssl"] is not None assert called_kwargs["ssl"] is not False @@ -227,6 +227,65 @@ async def test_async_realtime_url_contains_model(): mock_streaming_instance.bidirectional_forward.assert_awaited_once() +@pytest.mark.asyncio +async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it(): + """Upstream WS gets OpenAI-Beta: realtime=v1 only when the client WebSocket included it.""" + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.types.realtime import RealtimeQueryParams + + handler = OpenAIRealtime() + api_base = "https://api.openai.com/" + api_key = "test-key" + model = "gpt-4o-mini-realtime-preview" + query_params: RealtimeQueryParams = {"model": model} + + dummy_websocket = MagicMock() + dummy_websocket.scope = { + "headers": [ + (b"openai-beta", b"realtime=v1"), + ] + } + dummy_logging_obj = MagicMock() + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): + mock_streaming_instance = MagicMock() + mock_realtime_streaming.return_value = mock_streaming_instance + mock_streaming_instance.bidirectional_forward = AsyncMock() + + await handler.async_realtime( + model=model, + websocket=dummy_websocket, + logging_obj=dummy_logging_obj, + api_base=api_base, + api_key=api_key, + query_params=query_params, + ) + + mock_ws_connect.assert_called_once() + called_kwargs = mock_ws_connect.call_args[1] + additional_headers = called_kwargs["additional_headers"] + assert additional_headers["Authorization"] == f"Bearer {api_key}" + assert additional_headers["OpenAI-Beta"] == "realtime=v1" + + @pytest.mark.asyncio async def test_async_realtime_uses_max_size_parameter(): """ diff --git a/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py b/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py index 70a8d86cb1b..9d7706557b5 100644 --- a/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py +++ b/tests/test_litellm/llms/sagemaker/test_sagemaker_common_utils.py @@ -11,6 +11,102 @@ from litellm.llms.sagemaker.common_utils import AWSEventStreamDecoder from litellm.llms.sagemaker.completion.transformation import SagemakerConfig +# --------------------------------------------------------------------------- # +# SAGEMAKER_RESPONSE_STREAM_SHAPE eager-load tests # +# --------------------------------------------------------------------------- # + + +def test_sagemaker_response_stream_shape_loaded_at_import(): + """ + SAGEMAKER_RESPONSE_STREAM_SHAPE is resolved at module import time. + In a standard environment with botocore installed it must be non-None. + """ + from litellm.llms.sagemaker.common_utils import SAGEMAKER_RESPONSE_STREAM_SHAPE + + assert SAGEMAKER_RESPONSE_STREAM_SHAPE is not None + + +def test_sagemaker_response_stream_shape_load_failure_returns_none(): + """ + If botocore's Loader raises (e.g. missing data files), _load_sagemaker_response_stream_shape + should return None rather than propagating the exception, so the module + still imports cleanly. + """ + from unittest.mock import patch + + import litellm.llms.sagemaker.common_utils as mod + + with patch( + "botocore.loaders.Loader.load_service_model", + side_effect=Exception("no data"), + ): + shape = mod._load_sagemaker_response_stream_shape() + assert shape is None + + +def test_sagemaker_response_stream_shape_is_structure_shape(): + """ + The loaded shape should be the botocore StructureShape for + InvokeEndpointWithResponseStreamOutput, not a plain dict or any other type. + """ + from botocore.model import StructureShape + + from litellm.llms.sagemaker.common_utils import SAGEMAKER_RESPONSE_STREAM_SHAPE + + assert SAGEMAKER_RESPONSE_STREAM_SHAPE is not None, ( + "SAGEMAKER_RESPONSE_STREAM_SHAPE is None — botocore may not be installed" + ) + shape: StructureShape = SAGEMAKER_RESPONSE_STREAM_SHAPE # remove Optional + assert isinstance(shape, StructureShape) + assert shape.name == "InvokeEndpointWithResponseStreamOutput" + + +def test_sagemaker_response_stream_shape_not_reloaded_on_new_decoder(): + """ + Creating multiple AWSEventStreamDecoder instances must not trigger + additional botocore Loader calls — the shape is resolved once at import + time and reused. + """ + from litellm.llms.sagemaker.common_utils import SAGEMAKER_RESPONSE_STREAM_SHAPE + + decoder_a = AWSEventStreamDecoder(model="test-model-a") + decoder_b = AWSEventStreamDecoder(model="test-model-b") + + # Both decoders should use the same pre-loaded shape object (identity check) + assert "_response_stream_shape_cache" not in decoder_a.__dict__ + assert "_response_stream_shape_cache" not in decoder_b.__dict__ + # The module constant is still the same object + from litellm.llms.sagemaker.common_utils import ( + SAGEMAKER_RESPONSE_STREAM_SHAPE as shape_after, + ) + + assert SAGEMAKER_RESPONSE_STREAM_SHAPE is shape_after + + +def test_sagemaker_parse_message_from_event_raises_on_none_shape(): + """ + When SAGEMAKER_RESPONSE_STREAM_SHAPE is None (botocore unavailable), + _parse_message_from_event must raise ValueError before touching the + botocore parser — not an opaque AttributeError from inside botocore. + """ + from unittest.mock import MagicMock, patch + + import litellm.llms.sagemaker.common_utils as mod + from litellm.llms.sagemaker.common_utils import SagemakerError + + decoder = AWSEventStreamDecoder(model="test-model") + mock_event = MagicMock() + + with patch.object(mod, "SAGEMAKER_RESPONSE_STREAM_SHAPE", None): + with pytest.raises(SagemakerError) as exc_info: + decoder._parse_message_from_event(mock_event) + + assert exc_info.value.status_code == 500 + assert "botocore" in str(exc_info.value.message).lower() + # The botocore parser must never have been called + mock_event.to_response_dict.assert_not_called() + + @pytest.mark.asyncio async def test_aiter_bytes_unicode_decode_error(): """ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py new file mode 100644 index 00000000000..9ff4e01da5e --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_token_exchange.py @@ -0,0 +1,511 @@ +""" +Tests for OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers. + +Covers: exchange flow, caching, error handling, resolve_mcp_auth integration, +bearer token extraction, and config loading. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from litellm.proxy._experimental.mcp_server.auth.token_exchange import ( + TOKEN_EXCHANGE_GRANT_TYPE, + TokenExchangeHandler, +) +from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, +) +from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + resolve_mcp_auth, +) +from litellm.proxy._types import LiteLLM_MCPServerTable, MCPTransport +from litellm.types.mcp import MCPAuth +from litellm.types.mcp_server.mcp_server_manager import MCPServer + + +def _obo_server(**overrides) -> MCPServer: + defaults = dict( + server_id="srv-obo-1", + name="test-obo", + url="https://mcp.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + client_id="litellm-client-id", + client_secret="litellm-client-secret", + token_exchange_endpoint="https://idp.example.com/oauth2/token", + audience="api://mcp-server", + scopes=["mcp.tools.read", "mcp.tools.execute"], + ) + defaults.update(overrides) + return MCPServer(**defaults) + + +def _exchange_response(token="exchanged-tok-abc", expires_in=3600): + resp = MagicMock() + resp.json.return_value = { + "access_token": token, + "token_type": "Bearer", + "expires_in": expires_in, + } + resp.raise_for_status = MagicMock() + resp.text = "" + return resp + + +# ── Exchange Flow ── + + +@pytest.mark.asyncio +async def test_exchange_token_success(): + """Token exchange sends correct RFC 8693 parameters and returns access_token.""" + handler = TokenExchangeHandler() + server = _obo_server() + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response("scoped-token-1") + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ): + result = await handler.exchange_token("user-jwt-xyz", server) + + assert result == "scoped-token-1" + mock_client.post.assert_called_once() + + _, kwargs = mock_client.post.call_args + data = kwargs["data"] + assert data["grant_type"] == TOKEN_EXCHANGE_GRANT_TYPE + assert data["subject_token"] == "user-jwt-xyz" + assert data["subject_token_type"] == "urn:ietf:params:oauth:token-type:access_token" + assert data["audience"] == "api://mcp-server" + assert data["scope"] == "mcp.tools.read mcp.tools.execute" + assert data["client_id"] == "litellm-client-id" + assert data["client_secret"] == "litellm-client-secret" + + +@pytest.mark.asyncio +async def test_exchange_token_no_audience(): + """When audience is None, it is omitted from the request.""" + handler = TokenExchangeHandler() + server = _obo_server(audience=None) + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response() + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ): + await handler.exchange_token("user-jwt", server) + + _, kwargs = mock_client.post.call_args + assert "audience" not in kwargs["data"] + + +@pytest.mark.asyncio +async def test_exchange_token_no_scopes(): + """When scopes is None, scope param is omitted from the request.""" + handler = TokenExchangeHandler() + server = _obo_server(scopes=None) + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response() + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ): + await handler.exchange_token("user-jwt", server) + + _, kwargs = mock_client.post.call_args + assert "scope" not in kwargs["data"] + + +# ── Caching ── + + +@pytest.mark.asyncio +async def test_exchange_token_cached(): + """Second call with same user token uses cache — only 1 HTTP POST.""" + handler = TokenExchangeHandler() + server = _obo_server() + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response("cached-exchange-tok") + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ): + t1 = await handler.exchange_token("same-jwt", server) + t2 = await handler.exchange_token("same-jwt", server) + + assert t1 == t2 == "cached-exchange-tok" + assert mock_client.post.call_count == 1 + + +@pytest.mark.asyncio +async def test_different_user_tokens_not_shared(): + """Different user JWTs get different exchanged tokens.""" + handler = TokenExchangeHandler() + server = _obo_server() + call_count = 0 + + async def mock_post(url, data=None): + nonlocal call_count + call_count += 1 + resp = MagicMock() + resp.json.return_value = { + "access_token": f"exchanged-{call_count}", + "expires_in": 3600, + } + resp.raise_for_status = MagicMock() + return resp + + mock_client = AsyncMock() + mock_client.post = mock_post + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ): + t1 = await handler.exchange_token("user-a-jwt", server) + t2 = await handler.exchange_token("user-b-jwt", server) + + assert t1 == "exchanged-1" + assert t2 == "exchanged-2" + assert call_count == 2 + + +# ── Error Handling ── + + +@pytest.mark.asyncio +async def test_exchange_token_http_error(): + """HTTP errors from the IDP are wrapped in a ValueError.""" + handler = TokenExchangeHandler() + server = _obo_server() + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = "invalid_grant" + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Bad Request", + request=MagicMock(), + response=mock_response, + ) + mock_client = AsyncMock() + mock_client.post.return_value = mock_response + + with ( + patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ), + pytest.raises(ValueError, match="failed with status 400"), + ): + await handler.exchange_token("bad-jwt", server) + + +@pytest.mark.asyncio +async def test_exchange_token_http_error_does_not_log_response_body(): + """Raw IDP error bodies are not logged because they can contain credentials.""" + handler = TokenExchangeHandler() + server = _obo_server() + raw_response_body = "client_secret=do-not-log" + mock_response = MagicMock() + mock_response.status_code = 401 + mock_response.text = raw_response_body + mock_response.raise_for_status.side_effect = httpx.HTTPStatusError( + "Unauthorized", + request=MagicMock(), + response=mock_response, + ) + mock_client = AsyncMock() + mock_client.post.return_value = mock_response + + with ( + patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.verbose_logger.debug" + ) as mock_debug, + pytest.raises(ValueError, match="failed with status 401"), + ): + await handler.exchange_token("bad-jwt", server) + + logged_values = " ".join( + str(value) + for call in mock_debug.call_args_list + for value in [*call.args, *call.kwargs.values()] + ) + assert raw_response_body not in logged_values + + +@pytest.mark.asyncio +async def test_exchange_token_missing_access_token(): + """Response without access_token raises ValueError.""" + handler = TokenExchangeHandler() + server = _obo_server() + resp = MagicMock() + resp.json.return_value = {"token_type": "Bearer"} + resp.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.post.return_value = resp + + with ( + patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.get_async_httpx_client", + return_value=mock_client, + ), + pytest.raises(ValueError, match="missing 'access_token'"), + ): + await handler.exchange_token("jwt", server) + + +@pytest.mark.asyncio +async def test_exchange_token_missing_endpoint(): + """Missing token_exchange_endpoint and token_url raises ValueError.""" + handler = TokenExchangeHandler() + server = _obo_server(token_exchange_endpoint=None, token_url=None) + + with pytest.raises(ValueError, match="no token_exchange_endpoint or token_url"): + await handler.exchange_token("jwt", server) + + +@pytest.mark.asyncio +async def test_exchange_token_missing_credentials(): + """Missing client_id or client_secret raises ValueError.""" + handler = TokenExchangeHandler() + server = _obo_server(client_id=None, client_secret=None) + # has_token_exchange_config will be False, so we call _do_exchange directly + with pytest.raises(ValueError, match="missing client_id or client_secret"): + await handler._do_exchange("jwt", server) + + +# ── resolve_mcp_auth Integration ── + + +@pytest.mark.asyncio +async def test_resolve_mcp_auth_with_token_exchange(): + """resolve_mcp_auth delegates to token exchange when server has OBO config and subject_token provided.""" + server = _obo_server() + mock_handler = AsyncMock() + mock_handler.exchange_token.return_value = "obo-scoped-token" + + with patch( + "litellm.proxy._experimental.mcp_server.auth.token_exchange.mcp_token_exchange_handler", + mock_handler, + ): + result = await resolve_mcp_auth(server, subject_token="user-jwt") + + assert result == "obo-scoped-token" + mock_handler.exchange_token.assert_called_once_with("user-jwt", server) + + +@pytest.mark.asyncio +async def test_resolve_mcp_auth_obo_without_subject_token_falls_through(): + """Without a subject_token, resolve_mcp_auth falls through to client_credentials.""" + server = _obo_server( + token_url="https://auth.example.com/token", + ) + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response("cc-token") + + with patch( + "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + return_value=mock_client, + ): + result = await resolve_mcp_auth(server, subject_token=None) + + # Falls through to client_credentials since subject_token is None + # The server has client_id/client_secret/token_url so has_client_credentials is True + assert result == "cc-token" + + +@pytest.mark.asyncio +async def test_resolve_mcp_auth_obo_without_subject_token_uses_cached_client_credentials(): + """The M2M fallback for OBO servers reuses the client_credentials cache.""" + server = _obo_server( + server_id="srv-obo-m2m-cache", + token_url="https://auth.example.com/token", + ) + mock_client = AsyncMock() + mock_client.post.return_value = _exchange_response("cached-cc-token") + + with patch( + "litellm.proxy._experimental.mcp_server.oauth2_token_cache.get_async_httpx_client", + return_value=mock_client, + ): + first = await resolve_mcp_auth(server, subject_token=None) + second = await resolve_mcp_auth(server, subject_token=None) + + assert first == second == "cached-cc-token" + mock_client.post.assert_called_once() + + +@pytest.mark.asyncio +async def test_resolve_mcp_auth_header_beats_obo(): + """An explicit mcp_auth_header takes priority over OBO token exchange.""" + server = _obo_server() + result = await resolve_mcp_auth( + server, mcp_auth_header="Bearer override", subject_token="user-jwt" + ) + assert result == "Bearer override" + + +# ── Bearer Token Extraction ── + + +def test_extract_bearer_token_from_oauth2_headers(): + """Extracts token from oauth2_headers Authorization header.""" + result = MCPServerManager._extract_bearer_token( + oauth2_headers={"Authorization": "Bearer my-jwt-token"}, + raw_headers=None, + ) + assert result == "my-jwt-token" + + +def test_extract_bearer_token_from_raw_headers(): + """Falls back to raw_headers when oauth2_headers missing.""" + result = MCPServerManager._extract_bearer_token( + oauth2_headers=None, + raw_headers={"authorization": "Bearer raw-jwt"}, + ) + assert result == "raw-jwt" + + +def test_extract_bearer_token_no_bearer_prefix(): + """Returns token as-is when no Bearer prefix.""" + result = MCPServerManager._extract_bearer_token( + oauth2_headers={"Authorization": "some-opaque-token"}, + raw_headers=None, + ) + assert result == "some-opaque-token" + + +def test_extract_bearer_token_none(): + """Returns None when no auth headers present.""" + result = MCPServerManager._extract_bearer_token( + oauth2_headers=None, + raw_headers=None, + ) + assert result is None + + +# ── MCPServer Properties ── + + +def test_has_token_exchange_config_true(): + """has_token_exchange_config is True for a fully configured OBO server.""" + server = _obo_server() + assert server.has_token_exchange_config is True + + +def test_has_token_exchange_config_false_wrong_auth_type(): + """has_token_exchange_config is False when auth_type is not oauth2_token_exchange.""" + server = _obo_server(auth_type=MCPAuth.oauth2) + assert server.has_token_exchange_config is False + + +def test_has_token_exchange_config_false_missing_creds(): + """has_token_exchange_config is False when client_id/client_secret missing.""" + server = _obo_server(client_id=None) + assert server.has_token_exchange_config is False + + +def test_has_token_exchange_config_uses_token_url_fallback(): + """has_token_exchange_config is True when token_url is set instead of token_exchange_endpoint.""" + server = _obo_server( + token_exchange_endpoint=None, + token_url="https://idp.example.com/token", + ) + assert server.has_token_exchange_config is True + + +# ── Config Loading ── + + +@pytest.mark.asyncio +async def test_config_loading_token_exchange_fields(): + """load_servers_from_config correctly maps OBO config fields to MCPServer.""" + manager = MCPServerManager() + config = { + "my_obo_server": { + "url": "https://mcp.example.com/mcp", + "transport": "http", + "auth_type": "oauth2_token_exchange", + "client_id": "my-client", + "client_secret": "my-secret", + "token_exchange_endpoint": "https://idp.example.com/oauth2/token", + "audience": "api://my-mcp", + "scopes": ["read", "write"], + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + } + } + await manager.load_servers_from_config(config) + + servers = list(manager.config_mcp_servers.values()) + assert len(servers) == 1 + + server = servers[0] + assert server.auth_type == MCPAuth.oauth2_token_exchange + assert server.token_exchange_endpoint == "https://idp.example.com/oauth2/token" + assert server.audience == "api://my-mcp" + assert server.subject_token_type == "urn:ietf:params:oauth:token-type:jwt" + assert server.client_id == "my-client" + assert server.client_secret == "my-secret" + assert server.scopes == ["read", "write"] + assert server.has_token_exchange_config is True + + +@pytest.mark.asyncio +async def test_config_loading_default_subject_token_type(): + """subject_token_type defaults to access_token when not specified in config.""" + manager = MCPServerManager() + config = { + "obo_defaults": { + "url": "https://mcp.example.com/mcp", + "transport": "http", + "auth_type": "oauth2_token_exchange", + "client_id": "cid", + "client_secret": "csec", + "token_exchange_endpoint": "https://idp.example.com/token", + } + } + await manager.load_servers_from_config(config) + + server = list(manager.config_mcp_servers.values())[0] + assert server.subject_token_type == "urn:ietf:params:oauth:token-type:access_token" + + +@pytest.mark.asyncio +async def test_database_loading_token_exchange_scopes_from_credentials(): + """DB-loaded OBO server credentials retain configured scopes.""" + manager = MCPServerManager() + db_server = LiteLLM_MCPServerTable( + server_id="srv-obo-db", + server_name="obo_db_server", + url="https://mcp.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2_token_exchange, + credentials={ + "client_id": "db-client", + "client_secret": "db-secret", + "token_exchange_endpoint": "https://idp.example.com/oauth2/token", + "audience": "api://db-mcp", + "scopes": ["db.read", "db.write"], + }, + ) + + server = await manager.build_mcp_server_from_table( + db_server, + credentials_are_encrypted=False, + ) + + assert server.auth_type == MCPAuth.oauth2_token_exchange + assert server.client_id == "db-client" + assert server.client_secret == "db-secret" + assert server.token_exchange_endpoint == "https://idp.example.com/oauth2/token" + assert server.audience == "api://db-mcp" + assert server.scopes == ["db.read", "db.write"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py index 649a08e8744..cbea386a69c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_hook_extra_headers.py @@ -549,7 +549,11 @@ class TestHookHeaderMergePriority: captured_extra_headers: Dict[str, Any] = {} async def fake_create_mcp_client( - server, mcp_auth_header=None, extra_headers=None, stdio_env=None + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token=None, ): captured_extra_headers["value"] = extra_headers mock_client = MagicMock() @@ -589,7 +593,11 @@ class TestHookHeaderMergePriority: captured_extra_headers: Dict[str, Any] = {} async def fake_create_mcp_client( - server, mcp_auth_header=None, extra_headers=None, stdio_env=None + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token=None, ): captured_extra_headers["value"] = extra_headers mock_client = MagicMock() @@ -635,7 +643,11 @@ class TestHookHeaderMergePriority: captured_extra_headers: Dict[str, Any] = {} async def fake_create_mcp_client( - server, mcp_auth_header=None, extra_headers=None, stdio_env=None + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token=None, ): captured_extra_headers["value"] = extra_headers mock_client = MagicMock() @@ -691,7 +703,11 @@ class TestHookHeaderMergePriority: captured_extra_headers: Dict[str, Any] = {} async def fake_create_mcp_client( - server, mcp_auth_header=None, extra_headers=None, stdio_env=None + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token=None, ): captured_extra_headers["value"] = extra_headers mock_client = MagicMock() @@ -739,7 +755,11 @@ class TestHookHeaderMergePriority: captured_extra_headers: Dict[str, Any] = {} async def fake_create_mcp_client( - server, mcp_auth_header=None, extra_headers=None, stdio_env=None + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + subject_token=None, ): captured_extra_headers["value"] = extra_headers mock_client = MagicMock() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 06f95159c08..a7649502bde 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -1228,6 +1228,7 @@ async def test_oauth2_headers_passed_to_mcp_client(): mcp_auth_header=None, extra_headers=None, stdio_env=None, + subject_token=None, ): # Capture the arguments for verification captured_client_args.update( @@ -1236,6 +1237,7 @@ async def test_oauth2_headers_passed_to_mcp_client(): "mcp_auth_header": mcp_auth_header, "extra_headers": extra_headers, "stdio_env": stdio_env, + "subject_token": subject_token, } ) # Return a mock client that doesn't actually connect @@ -2282,6 +2284,144 @@ class TestMCPServerManagerReload: mock_build.assert_awaited_once_with(db_row) assert manager.registry["server-1"] is rebuilt_server + @pytest.mark.asyncio + async def test_skips_server_when_build_from_database_fails(self, caplog): + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + except ImportError: + pytest.skip("MCP server not available") + + manager = MCPServerManager() + timestamp = datetime.utcnow() + healthy_row = _make_db_mcp_server("healthy-server", timestamp) + bad_row = _make_db_mcp_server("bad-server", timestamp) + another_healthy_row = _make_db_mcp_server("another-healthy-server", timestamp) + + healthy_server = MCPServer( + server_id="healthy-server", + name="healthy", + transport=MCPTransport.http, + updated_at=timestamp, + ) + another_healthy_server = MCPServer( + server_id="another-healthy-server", + name="another-healthy", + transport=MCPTransport.http, + updated_at=timestamp, + ) + + async def build_server(db_row): + if db_row.server_id == "bad-server": + raise RuntimeError("transient build failure") + if db_row.server_id == "healthy-server": + return healthy_server + return another_healthy_server + + mock_prisma = MagicMock() + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock( + return_value=[healthy_row, bad_row, another_healthy_row] + ) + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma, + ), + patch.object( + manager, + "build_mcp_server_from_table", + AsyncMock(side_effect=build_server), + ), + patch.object(manager, "_maybe_register_openapi_tools", AsyncMock()), + caplog.at_level("ERROR", logger="LiteLLM"), + ): + await manager.reload_servers_from_database() + + assert set(manager.registry) == {"healthy-server", "another-healthy-server"} + assert manager.registry["healthy-server"] is healthy_server + assert manager.registry["another-healthy-server"] is another_healthy_server + assert "Skipping MCP server bad-server" in caplog.text + + @pytest.mark.asyncio + async def test_skips_server_when_openapi_registration_fails(self, caplog): + try: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + except ImportError: + pytest.skip("MCP server not available") + + manager = MCPServerManager() + timestamp = datetime.utcnow() + healthy_row = _make_db_mcp_server("healthy-server", timestamp) + bad_openapi_row = _make_db_mcp_server("bad-openapi-server", timestamp) + existing_server = MCPServer( + server_id="existing-server", + name="existing", + transport=MCPTransport.http, + updated_at=timestamp, + ) + manager.registry = {existing_server.server_id: existing_server} + + healthy_server = MCPServer( + server_id="healthy-server", + name="healthy", + transport=MCPTransport.http, + updated_at=timestamp, + ) + bad_openapi_server = MCPServer( + server_id="bad-openapi-server", + name="bad-openapi", + transport=MCPTransport.http, + spec_path="https://example.invalid/openapi.json", + updated_at=timestamp, + ) + + async def build_server(db_row): + if db_row.server_id == "healthy-server": + return healthy_server + return bad_openapi_server + + observed_registries = [] + + async def register_openapi_tools(server, **kwargs): + observed_registries.append(set(manager.registry)) + assert kwargs == {"initialize_mapping": False} + if server.server_id == "bad-openapi-server": + raise RuntimeError("blocked address") + + mock_prisma = MagicMock() + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock( + return_value=[healthy_row, bad_openapi_row] + ) + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=mock_prisma, + ), + patch.object( + manager, + "build_mcp_server_from_table", + AsyncMock(side_effect=build_server), + ), + patch.object( + manager, + "_maybe_register_openapi_tools", + AsyncMock(side_effect=register_openapi_tools), + ), + caplog.at_level("ERROR", logger="LiteLLM"), + ): + await manager.reload_servers_from_database() + + assert set(manager.registry) == {"healthy-server"} + assert manager.registry["healthy-server"] is healthy_server + assert observed_registries == [ + {"existing-server"}, + {"existing-server"}, + ] + assert "Skipping MCP server bad-openapi-server" in caplog.text + @pytest.mark.asyncio async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook(): @@ -2946,7 +3086,7 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow(): """ P1 Regression: list_tools path must apply _resolve_oauth2_flow to legacy DB rows where oauth2_flow is NULL but M2M credentials are present. - + Without this fix, has_client_credentials returns False and the caller's Authorization header is forwarded upstream instead of being blocked. """ @@ -3044,7 +3184,7 @@ async def test_call_tool_empty_extra_headers_returns_none(): """ P2 Regression: When all configured extra_headers are filtered out (e.g. Authorization for M2M), the resulting extra_headers should be None, not {}. - + Downstream code that checks `if extra_headers is None` will behave differently if an empty dict is passed instead. """ @@ -3071,7 +3211,10 @@ async def test_call_tool_empty_extra_headers_returns_none(): extra_headers=["Authorization"], # Will be filtered out for M2M ) - raw_headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} + raw_headers = { + "Authorization": "Bearer sk-1234", + "Content-Type": "application/json", + } captured_extra_headers = None @@ -3108,8 +3251,8 @@ async def test_call_tool_empty_extra_headers_returns_none(): pass # We only care about the captured headers # With P2 fix: extra_headers should be None (not {}) when all headers filtered - assert captured_extra_headers is None, ( - "P2 API consistency issue: expected None for empty extra_headers, got: " - + str(captured_extra_headers) + assert ( + captured_extra_headers is None + ), "P2 API consistency issue: expected None for empty extra_headers, got: " + str( + captured_extra_headers ) - diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 6cbdcd28208..11e9dbbdd57 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -450,7 +450,7 @@ class TestMCPServerManager: captured_extra_headers = None async def capture_create_mcp_client( - server, mcp_auth_header, extra_headers, stdio_env + server, mcp_auth_header, extra_headers, stdio_env, subject_token=None ): # pragma: no cover - helper nonlocal captured_extra_headers captured_extra_headers = extra_headers diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py index efe100a11dc..957dea22f3c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_openapi_to_mcp_generator.py @@ -868,3 +868,146 @@ class TestResolveOperationParams: assert "per_page" in names assert "sha" in names assert len(names) == 4 # no duplicates + + +# --------------------------------------------------------------------------- +# Tool name sanitization for OpenAPI -> MCP +# Repro: GitHub's REST OpenAPI uses tag-namespaced operationIds like +# "actions/download-job-logs-for-workflow-run". Without sanitization the +# generated MCP tool name contains '/', which Anthropic/OpenAI/Bedrock all +# reject (^[a-zA-Z0-9_-]+$). This block guards the registration + preview +# paths against that. +# --------------------------------------------------------------------------- + + +class TestSanitizeOpenAPIToolName: + def test_replaces_slashes(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + assert ( + sanitize_openapi_tool_name("actions/download-job-logs-for-workflow-run") + == "actions_download-job-logs-for-workflow-run" + ) + + def test_replaces_other_punctuation(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + assert sanitize_openapi_tool_name("foo.bar:baz qux") == "foo_bar_baz_qux" + + def test_lowercases(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + assert sanitize_openapi_tool_name("Pulls/List-Files") == "pulls_list-files" + + def test_already_valid_passes_through(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + assert sanitize_openapi_tool_name("plain-tool_name") == "plain-tool_name" + + def test_empty_string(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + assert sanitize_openapi_tool_name("") == "" + + def test_caps_at_128_chars(self): + from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( + sanitize_openapi_tool_name, + ) + + out = sanitize_openapi_tool_name("a/" * 200) + assert len(out) <= 128 + + +class TestRegisterToolsFromOpenAPI: + """Verify register_tools_from_openapi emits provider-safe tool names.""" + + def test_github_style_operation_ids_are_sanitized(self, monkeypatch): + import re + + from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator + + registered: list = [] + + def _capture(name, description, input_schema, handler): # noqa: ANN001 + registered.append(name) + + monkeypatch.setattr( + openapi_to_mcp_generator.global_mcp_tool_registry, + "register_tool", + _capture, + ) + + spec = { + "paths": { + "/repos/{owner}/{repo}/actions/jobs/{job_id}/logs": { + "get": { + "operationId": "actions/download-job-logs-for-workflow-run", + "summary": "Download job logs", + } + }, + "/repos/{owner}/{repo}/pulls/{pull_number}/files": { + "get": { + "operationId": "pulls/list-files", + "summary": "List files", + } + }, + } + } + + openapi_to_mcp_generator.register_tools_from_openapi( + spec, base_url="https://api.example.com" + ) + + assert registered, "expected at least one registered tool" + anthropic_re = re.compile(r"^[a-zA-Z0-9_-]{1,128}$") + for name in registered: + assert anthropic_re.match( + name + ), f"tool name {name!r} violates ^[a-zA-Z0-9_-]+$" + assert "actions_download-job-logs-for-workflow-run" in registered + assert "pulls_list-files" in registered + + def test_missing_operation_id_uses_sanitized_method_path_fallback( + self, monkeypatch + ): + import re + + from litellm.proxy._experimental.mcp_server import openapi_to_mcp_generator + + registered: list = [] + + def _capture(name, description, input_schema, handler): # noqa: ANN001 + registered.append(name) + + monkeypatch.setattr( + openapi_to_mcp_generator.global_mcp_tool_registry, + "register_tool", + _capture, + ) + + spec = { + "paths": { + "/foo/{bar}/baz": { + "get": {"summary": "no operationId here"}, + } + } + } + openapi_to_mcp_generator.register_tools_from_openapi( + spec, base_url="https://api.example.com" + ) + + assert registered + for name in registered: + assert re.match( + r"^[a-zA-Z0-9_-]+$", name + ), f"fallback tool name {name!r} not sanitized" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 90e504c959c..f4feac68fcc 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1376,3 +1376,164 @@ class TestEndpointRoleChecks: user_api_key_dict=user_key, ) assert result["status"] == "ok" + + +class TestPreviewOpenAPITools: + """Verify the OpenAPI preview endpoint emits provider-safe tool names. + + Regression: GitHub's OpenAPI spec uses tag-namespaced operationIds like + `actions/download-job-logs-for-workflow-run` which contain '/'. The + preview must sanitize so what the dashboard shows matches what gets + registered (and what makes it past LLM provider tool-name validation). + """ + + pytestmark = pytest.mark.asyncio + + async def test_preview_sanitizes_slash_in_operation_id(self, monkeypatch): + import re + + async def fake_load_spec(spec_path): # noqa: ANN001 + return { + "paths": { + "/repos/{owner}/{repo}/actions/jobs/{job_id}/logs": { + "get": { + "operationId": ( + "actions/download-job-logs-for-workflow-run" + ), + "summary": "Download job logs", + } + }, + "/repos/{owner}/{repo}/pulls/{pull_number}/files": { + "get": { + "operationId": "pulls/list-files", + "summary": "List files", + } + }, + } + } + + from litellm.proxy._experimental.mcp_server import ( + openapi_to_mcp_generator, + ) + + monkeypatch.setattr( + openapi_to_mcp_generator, + "load_openapi_spec_async", + fake_load_spec, + raising=False, + ) + + payload = NewMCPServerRequest( + server_name="github_openapi_mcp", + spec_path="https://example.invalid/openapi.json", + transport="http", + ) + request = _build_request() + + from litellm.proxy._types import LitellmUserRoles + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result.get("error") is None, result + names = [t["name"] for t in result["tools"]] + anthropic_re = re.compile(r"^[a-zA-Z0-9_-]{1,128}$") + for name in names: + assert anthropic_re.match( + name + ), f"preview tool name {name!r} violates ^[a-zA-Z0-9_-]+$" + assert "actions_download-job-logs-for-workflow-run" in names + assert "pulls_list-files" in names + + async def test_preview_method_order_matches_registration(self, monkeypatch): + """Preview must iterate HTTP methods in the same order as + register_tools_from_openapi, otherwise collision-disambiguation + suffixes (_2, _3, ...) get assigned to different operations and the + dashboard shows names that differ from what's actually registered. + """ + from litellm.proxy._experimental.mcp_server import ( + openapi_to_mcp_generator, + ) + + spec = { + "paths": { + "/items/{id}": { + "delete": { + "operationId": "items/delete", + "summary": "Delete item", + }, + "patch": { + "operationId": "items.delete", + "summary": "Soft-delete item", + }, + } + } + } + + async def fake_load_spec(spec_path): # noqa: ANN001 + return spec + + monkeypatch.setattr( + openapi_to_mcp_generator, + "load_openapi_spec_async", + fake_load_spec, + raising=False, + ) + + payload = NewMCPServerRequest( + server_name="collision_openapi_mcp", + spec_path="https://example.invalid/openapi.json", + transport="http", + ) + request = _build_request() + from litellm.proxy._types import LitellmUserRoles + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert result.get("error") is None, result + preview_summary_to_name = {t["description"]: t["name"] for t in result["tools"]} + + registered_summary_to_name: dict = {} + + def fake_create_tool_function( + path, method, operation, base_url + ): # noqa: ANN001 + def _f(): + return None + + return _f + + monkeypatch.setattr( + openapi_to_mcp_generator, + "create_tool_function", + fake_create_tool_function, + ) + + class _StubRegistry: + def register_tool( + self, name, description, input_schema, handler + ): # noqa: ANN001 + registered_summary_to_name[description] = name + + monkeypatch.setattr( + openapi_to_mcp_generator, + "global_mcp_tool_registry", + _StubRegistry(), + ) + + openapi_to_mcp_generator.register_tools_from_openapi( + spec, base_url="https://example.invalid" + ) + + assert preview_summary_to_name == registered_summary_to_name, ( + f"preview {preview_summary_to_name} != " + f"registered {registered_summary_to_name} — method iteration " + "order is out of sync, so collision suffixes (_2, _3, ...) " + "land on different operations" + ) diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 3433e3e6d85..8a854bcd6a8 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -906,6 +906,285 @@ def test_can_object_call_model_no_access_to_alias_or_underlying(): assert "my-fake-gpt" in str(exc_info.value.message) +# -- Team-member access-group resolution with team-scoped DB models ----------- + + +def _make_team_scoped_router(team_id: str = "team-a"): + """ + Build a Router whose model_list looks like what the proxy creates for + team-scoped BYOK DB models: the internal model_name is + ``__`` and the public name lives in + ``model_info.team_public_model_name``. Two models belong to the + access group ``fast-models``; one (``mock-power``) does not. + """ + from litellm import Router + + model_list = [ + { + "model_name": f"mock-fast-1_{team_id}_aaa", + "litellm_params": { + "model": "openai/mock-fast-1", + "api_key": "fake", + }, + "model_info": { + "id": f"demo-mock-fast-1-{team_id}", + "team_id": team_id, + "team_public_model_name": "mock-fast-1", + "access_groups": ["fast-models"], + }, + }, + { + "model_name": f"mock-fast-2_{team_id}_bbb", + "litellm_params": { + "model": "openai/mock-fast-2", + "api_key": "fake", + }, + "model_info": { + "id": f"demo-mock-fast-2-{team_id}", + "team_id": team_id, + "team_public_model_name": "mock-fast-2", + "access_groups": ["fast-models"], + }, + }, + { + "model_name": f"mock-power_{team_id}_ccc", + "litellm_params": { + "model": "openai/mock-power", + "api_key": "fake", + }, + "model_info": { + "id": f"demo-mock-power-{team_id}", + "team_id": team_id, + "team_public_model_name": "mock-power", + }, + }, + ] + return Router(model_list=model_list) + + +def test_can_object_call_model_access_group_with_team_id(): + """ + When team_id is passed, _can_object_call_model should resolve + model_info.access_groups for team-scoped DB models and allow + access via group name. + """ + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_team_scoped_router() + + result = _can_object_call_model( + model="mock-fast-1", + llm_router=router, + models=["fast-models", "mock-power"], + object_type="team", + team_id="team-a", + ) + assert result is True + + +def test_can_object_call_model_access_group_without_team_id_fails(): + """ + Without team_id the router cannot find team-scoped DB models, so + access group resolution fails and the call is denied. + This is the pre-fix behavior. + """ + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_team_scoped_router() + + with pytest.raises(ProxyException): + _can_object_call_model( + model="mock-fast-1", + llm_router=router, + models=["fast-models", "mock-power"], + object_type="team", + # team_id intentionally omitted + ) + + +def test_can_object_call_model_literal_name_with_team_id(): + """ + Literal model name matching should still work when team_id is + passed — no regression from adding team_id. + """ + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_team_scoped_router() + + result = _can_object_call_model( + model="mock-power", + llm_router=router, + models=["fast-models", "mock-power"], + object_type="team", + team_id="team-a", + ) + assert result is True + + +def test_can_object_call_model_denied_model_with_team_id(): + """ + A model not in the allowed list (by name or access group) should + still be denied even when team_id is passed. + """ + from litellm.proxy._types import ProxyException + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_team_scoped_router() + + with pytest.raises(ProxyException): + _can_object_call_model( + model="mock-vision", + llm_router=router, + models=["fast-models", "mock-power"], + object_type="team", + team_id="team-a", + ) + + +def test_can_object_call_model_second_group_member_with_team_id(): + """ + Both models in the access group should be reachable, not just + the first one. + """ + from litellm.proxy.auth.auth_checks import _can_object_call_model + + router = _make_team_scoped_router() + + result = _can_object_call_model( + model="mock-fast-2", + llm_router=router, + models=["fast-models"], + object_type="team", + team_id="team-a", + ) + assert result is True + + +@pytest.mark.asyncio +async def test_check_team_member_model_access_with_access_group(): + """ + End-to-end test of _check_team_member_model_access: a member whose + allowed_models contains an access group name should be allowed to + call models in that group for team-scoped DB models. + """ + from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.auth_checks import _check_team_member_model_access + + router = _make_team_scoped_router() + team = LiteLLM_TeamTable(team_id="team-a") + token = UserAPIKeyAuth(token="sk-test", user_id="alice", team_id="team-a") + membership = LiteLLM_TeamMembership( + user_id="alice", + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable( + allowed_models=["fast-models", "mock-power"], + ), + ) + + with patch( + "litellm.proxy.auth.auth_checks.get_team_membership", + return_value=membership, + ): + # Should not raise — mock-fast-1 is in the fast-models group + await _check_team_member_model_access( + model="mock-fast-1", + team_object=team, + valid_token=token, + llm_router=router, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + + +@pytest.mark.asyncio +async def test_check_team_member_model_access_denied_model(): + """ + A member with per-member allowed_models should be denied access to + a model that is neither listed by name nor covered by an access group. + """ + from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + ProxyException, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.auth_checks import _check_team_member_model_access + + router = _make_team_scoped_router() + team = LiteLLM_TeamTable(team_id="team-a") + token = UserAPIKeyAuth(token="sk-test", user_id="alice", team_id="team-a") + membership = LiteLLM_TeamMembership( + user_id="alice", + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable( + allowed_models=["fast-models", "mock-power"], + ), + ) + + with patch( + "litellm.proxy.auth.auth_checks.get_team_membership", + return_value=membership, + ): + with pytest.raises(ProxyException) as exc_info: + await _check_team_member_model_access( + model="mock-vision", + team_object=team, + valid_token=token, + llm_router=router, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + assert exc_info.value.type == ProxyErrorTypes.team_model_access_denied + + +@pytest.mark.asyncio +async def test_check_team_member_model_access_no_override_inherits_team(): + """ + When a member has no allowed_models (empty budget table), the function + should return without raising — the team-level check applies instead. + """ + from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + UserAPIKeyAuth, + ) + from litellm.proxy.auth.auth_checks import _check_team_member_model_access + + router = _make_team_scoped_router() + team = LiteLLM_TeamTable(team_id="team-a") + token = UserAPIKeyAuth(token="sk-test", user_id="bob", team_id="team-a") + membership = LiteLLM_TeamMembership( + user_id="bob", + team_id="team-a", + litellm_budget_table=LiteLLM_BudgetTable(), + ) + + with patch( + "litellm.proxy.auth.auth_checks.get_team_membership", + return_value=membership, + ): + # Should return without raising — no per-member restriction + await _check_team_member_model_access( + model="mock-vision", + team_object=team, + valid_token=token, + llm_router=router, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + ) + + # Tag Budget Enforcement Tests diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index cf6feabf85f..268cdff1f7d 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -53,6 +53,61 @@ def test_non_admin_config_update_route_rejected(): assert "Your role=internal_user" in str(exc_info.value) +@pytest.mark.parametrize( + "route", + ["/compliance/eu-ai-act", "/compliance/gdpr"], +) +def test_compliance_routes_open_to_internal_user(route): + """Compliance routes are stateless validators on caller-supplied log data + - non-admin internal_user roles can call them.""" + role = LitellmUserRoles.INTERNAL_USER.value + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + +@pytest.mark.parametrize( + "route", + ["/compliance/eu-ai-act", "/compliance/gdpr"], +) +def test_compliance_routes_blocked_for_internal_user_view_only(route): + """Deprecated internal_user_viewer role must not gain compliance route access.""" + role = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=role, + ) + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=role) + request = MagicMock(spec=Request) + request.query_params = {} + + with pytest.raises(Exception) as exc_info: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + assert "Only proxy admin can be used" in str(exc_info.value) + + def test_proxy_admin_viewer_config_update_route_rejected(): """Test that proxy admin viewer users are rejected when trying to call /config/update""" diff --git a/tests/test_litellm/proxy/auth/test_team_member_budget.py b/tests/test_litellm/proxy/auth/test_team_member_budget.py index 11a28106e31..b38a953d189 100644 --- a/tests/test_litellm/proxy/auth/test_team_member_budget.py +++ b/tests/test_litellm/proxy/auth/test_team_member_budget.py @@ -306,6 +306,79 @@ async def test_team_member_budget_check_no_team_membership(): assert result is True +@pytest.mark.asyncio +async def test_team_member_budget_check_blocks_regenerated_key_after_old_key_exhausts_budget(): + """Deleting an exhausted key and creating a new key must not reset a user's team budget.""" + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + team_object = LiteLLM_TeamTable( + team_id="test-team-1", + team_alias="Test Team", + spend=0.0, + max_budget=None, + ) + # The spend below represents usage accumulated by an earlier key that was + # later deleted. The new key must still be checked against the same + # user/team membership spend instead of receiving a fresh per-key budget. + regenerated_token = UserAPIKeyAuth( + token="new-regenerated-token", + user_id="test-user-1", + team_id="test-team-1", + models=["gpt-3.5-turbo"], + ) + team_membership = LiteLLM_TeamMembership( + user_id="test-user-1", + team_id="test-team-1", + spend=0.0000002, + litellm_budget_table=LiteLLM_BudgetTable( + max_budget=0.0000001, + ), + ) + + mock_request = MagicMock(spec=Request) + mock_prisma_client = MagicMock() + mock_user_api_key_cache = MagicMock() + mock_proxy_logging_obj = MagicMock() + + with ( + patch( + "litellm.proxy.auth.auth_checks.get_team_membership", + new_callable=AsyncMock, + return_value=team_membership, + ) as mock_get_team_membership, + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), + patch("litellm.proxy.proxy_server.user_api_key_cache", mock_user_api_key_cache), + ): + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body=request_body, + team_object=team_object, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging_obj, + valid_token=regenerated_token, + request=mock_request, + ) + + mock_get_team_membership.assert_any_await( + user_id="test-user-1", + team_id="test-team-1", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_user_api_key_cache, + proxy_logging_obj=mock_proxy_logging_obj, + ) + assert "Budget has been exceeded" in str(exc_info.value) + assert "test-user-1" in str(exc_info.value) + assert "test-team-1" in str(exc_info.value) + + @pytest.mark.asyncio async def test_team_member_budget_check_personal_key_not_team(): """Test that team member budget check is skipped for personal keys (no team).""" diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 2e5eef2a0aa..95b3d746c66 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -556,22 +556,7 @@ async def test_enterprise_custom_auth_runs_post_custom_auth_checks_when_opt_in() litellm.enable_post_custom_auth_checks = original_flag -@pytest.mark.parametrize( - "custom_litellm_key_header, api_key, passed_in_key", - [ - ("Bearer sk-12345678", "sk-12345678", "Bearer sk-12345678"), - ("Basic sk-12345678", "sk-12345678", "Basic sk-12345678"), - ("bearer sk-12345678", "sk-12345678", "bearer sk-12345678"), - ("sk-12345678", "sk-12345678", "sk-12345678"), - # AWS Signature V4 format (LangChain AWS SDK) - ( - "AWS4-HMAC-SHA256 Credential=Bearer sk-12345678/20260210/us-east-1/bedrock/aws4_request, SignedHeaders=host, Signature=abc123", - "sk-12345678", - "AWS4-HMAC-SHA256 Credential=Bearer sk-12345678/20260210/us-east-1/bedrock/aws4_request, SignedHeaders=host, Signature=abc123", - ), - ], -) -def test_get_api_key_with_custom_litellm_key_header( +def _assert_get_api_key_with_custom_litellm_key_header( custom_litellm_key_header, api_key, passed_in_key ): assert get_api_key( @@ -587,6 +572,49 @@ def test_get_api_key_with_custom_litellm_key_header( ) == (api_key, passed_in_key) +def test_get_api_key_with_custom_litellm_key_header_bearer_prefix(): + token = "sk-" + "1" * 8 + header = f"Bearer {token}" + _assert_get_api_key_with_custom_litellm_key_header( + custom_litellm_key_header=header, api_key=token, passed_in_key=header + ) + + +def test_get_api_key_with_custom_litellm_key_header_basic_prefix(): + token = "sk-" + "1" * 8 + header = f"Basic {token}" + _assert_get_api_key_with_custom_litellm_key_header( + custom_litellm_key_header=header, api_key=token, passed_in_key=header + ) + + +def test_get_api_key_with_custom_litellm_key_header_lowercase_bearer_prefix(): + token = "sk-" + "1" * 8 + header = f"bearer {token}" + _assert_get_api_key_with_custom_litellm_key_header( + custom_litellm_key_header=header, api_key=token, passed_in_key=header + ) + + +def test_get_api_key_with_custom_litellm_key_header_no_prefix(): + token = "sk-" + "1" * 8 + _assert_get_api_key_with_custom_litellm_key_header( + custom_litellm_key_header=token, api_key=token, passed_in_key=token + ) + + +def test_get_api_key_with_custom_litellm_key_header_aws_sigv4(): + """AWS Signature V4 format (LangChain AWS SDK).""" + token = "sk-" + "1" * 8 + header = ( + f"AWS4-HMAC-SHA256 Credential=Bearer {token}/20260210/us-east-1/bedrock/" + "aws4_request, SignedHeaders=host, Signature=abc123" + ) + _assert_get_api_key_with_custom_litellm_key_header( + custom_litellm_key_header=header, api_key=token, passed_in_key=header + ) + + def test_team_metadata_with_tags_flows_through_jwt_auth(): """ Test that team_metadata (specifically tags) flows through JWT authentication. diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index e7fec2c5279..00ed7e8cd6c 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -1677,3 +1677,98 @@ async def test_async_log_success_event_uses_team_priority_from_auth_metadata(): assert ( "default_pool" not in priority_keys[0] ), f"Priority key should NOT use 'default_pool', should use team's priority. Got: {priority_keys[0]}" + + +@pytest.mark.asyncio +async def test_priority_429_includes_model_name_and_configured_limits(): + """ + The priority-based 429 should tell operators which model was hit and what + the model's configured TPM/RPM are, so they can decide whether to tune the + priority allocation or the model limits. + + Regression test for the previous message that read: + "Priority-based rate limit exceeded. Priority: prod, + Rate limit type: tokens, Remaining: -664145, + Model saturation: 86.3%" + -- with no indication of which model was hit. + """ + from fastapi import HTTPException + + os.environ["LITELLM_LICENSE"] = "test-license-key" + litellm.priority_reservation = {"prod": 0.5} + + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + + model = "gpt-4o-test" + total_tpm = 1_000_000 + total_rpm = 10_000 + + llm_router = Router( + model_list=[ + { + "model_name": model, + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "test-key", + "api_base": "test-base", + "tpm": total_tpm, + "rpm": total_rpm, + }, + } + ] + ) + handler.update_variables(llm_router=llm_router) + + user = UserAPIKeyAuth() + user.metadata = {"priority": "prod"} + user.user_id = "prod_user" + + model_group_info = handler.llm_router.get_model_group_info(model_group=model) + + # Force the atomic check+increment to return OVER_LIMIT for the + # priority_model descriptor. saturation=0.95 keeps us above the + # default saturation threshold so priority limits are enforced. + over_limit_response = { + "overall_code": "OVER_LIMIT", + "statuses": [ + { + "code": "OVER_LIMIT", + "descriptor_key": "priority_model", + "rate_limit_type": "tokens", + "limit_remaining": -664145, + "current_limit": int(total_tpm * 0.5), + } + ], + } + + with patch.object( + handler.v3_limiter, + "atomic_check_and_increment_by_n", + new=AsyncMock(return_value=over_limit_response), + ): + with pytest.raises(HTTPException) as exc_info: + await handler._check_rate_limits( + model=model, + model_group_info=model_group_info, + user_api_key_dict=user, + priority="prod", + saturation=0.95, + data={"model": model}, + ) + + assert exc_info.value.status_code == 429 + detail = exc_info.value.detail + assert isinstance(detail, dict) + error_msg = detail["error"] + + # New fields added by this change -- the whole point of the fix. + assert f"Model: {model}" in error_msg, error_msg + assert f"Model TPM: {total_tpm}" in error_msg, error_msg + assert f"Model RPM: {total_rpm}" in error_msg, error_msg + + # Existing fields must still be present (no regression). + assert "Priority-based rate limit exceeded" in error_msg, error_msg + assert "Priority: prod" in error_msg, error_msg + assert "Rate limit type: tokens" in error_msg, error_msg + assert "Model saturation:" in error_msg, error_msg diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 370477c3605..e9ac1794ac9 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -363,11 +363,28 @@ async def test_normal_router_call_tpm_v3( rate_limit_object, value, "tokens" ) - # First request should succeed + # First request should succeed. Include messages + a tight max_tokens so + # the atomic reserve_tpm_tokens path populates the :tokens counter with a + # predictable amount — the pre-call hook no longer touches :tokens via + # should_rate_limit. + # Estimate: input ~ 1 token (`"hi"`), max_tokens = 5 → reservation = 6, + # which fits under the tpm_limit of 10. + pre_call_data = { + "model": "azure-model", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 5, + } + expected_reservation = parallel_request_handler._estimate_tokens_for_request( + data=pre_call_data + ) + assert ( + expected_reservation < 10 + ), "Test premise: reservation must fit under tpm_limit=10" + await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, - data={"model": "azure-model"}, + data=pre_call_data, call_type="", ) @@ -386,7 +403,7 @@ async def test_normal_router_call_tpm_v3( await asyncio.sleep(0) time_controller.advance(1) - # Verify the token count is tracked + # Verify the token count is tracked (populated by reserve_tpm_tokens). counter_value = await local_cache.async_get_cache(key=counter_key) print(f"local_cache: {local_cache.in_memory_cache.cache_dict}") @@ -405,7 +422,7 @@ async def test_normal_router_call_tpm_v3( await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, - data={"model": "azure-model"}, + data=pre_call_data, call_type="", ) @@ -416,14 +433,18 @@ async def test_normal_router_call_tpm_v3( await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, - data={"model": "azure-model"}, + data=pre_call_data, call_type="", ) - # Verify new window and reset counter + # Verify new window — counter resets and is repopulated to the new + # reservation amount (no longer the +1-per-request inflation artifact). final_counter_value = await local_cache.async_get_cache(key=counter_key) - assert final_counter_value == 1, "Counter should reset to 1 after window expiry" + assert final_counter_value == expected_reservation, ( + f"Counter should reset to a fresh reservation ({expected_reservation}) " + f"after window expiry, got {final_counter_value}" + ) @pytest.mark.parametrize( @@ -1977,18 +1998,10 @@ async def test_async_log_success_event_with_dict_usage_missing_fields(monkeypatc end_time=datetime.now(), ) - # Find the TPM increment operation - tpm_operation = None - for op in captured_operations: - if op["key"].endswith(":tokens"): - tpm_operation = op - break - - assert tpm_operation is not None, "Should have a TPM increment operation" - # Should default to 0 when field is missing - assert ( - tpm_operation["increment_value"] == 0 - ), "Should default to 0 when completion_tokens is missing" + # When total_tokens resolves to 0 (missing fields) and there's no reservation, + # the reconciliation delta is 0 — no TPM increment should be emitted. + tpm_ops = [op for op in captured_operations if op["key"].endswith(":tokens")] + assert tpm_ops == [], f"Expected no TPM ops when usage is empty, got: {tpm_ops}" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py new file mode 100644 index 00000000000..297d18d1ab3 --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_tpm_concurrent.py @@ -0,0 +1,999 @@ +""" +Unit tests for TPM rate limit for concurrent requests + +Verifies token-reservation pattern: +- Concurrent requests cannot all observe "under limit" before any of them + has incremented the counter (atomic reservation via + ``atomic_check_and_increment_by_n``). +- After a successful request, the counter is reconciled to actual usage. +- After a failed request, the full reservation is released. + +The reservation path delegates atomicity to ``atomic_check_and_increment_by_n``, +which uses Redis Lua when available and an asyncio-locked in-memory check +otherwise. These tests exercise the in-memory fallback so they run without +Redis. +""" + +import asyncio +from datetime import datetime +from typing import Any, Dict + +import pytest + +from litellm.caching.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.hooks.parallel_request_limiter_v3 import ( + TPM_RESERVATION_RELEASED_KEY, + TPM_RESERVED_MODEL_KEY, + TPM_RESERVED_SCOPES_KEY, + TPM_RESERVED_TOKENS_KEY, + _PROXY_MaxParallelRequestsHandler_v3 as RateLimitHandler, +) +from litellm.proxy.utils import InternalUsageCache, hash_token +from litellm.types.utils import ModelResponse, Usage + + +@pytest.fixture +def rate_limiter(): + cache = DualCache() + handler = RateLimitHandler(internal_usage_cache=InternalUsageCache(cache)) + return handler, cache + + +@pytest.mark.asyncio +async def test_token_reservation_prevents_concurrent_bypass(rate_limiter): + """ + With a 100 TPM limit and 5 concurrent requests each estimated at ~50+ tokens, + upfront reservation must reject the late arrivals — not let all 5 through. + Exercises the in-memory fallback in ``atomic_check_and_increment_by_n``. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-test-key") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=100, + ) + + request_data = { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hello, this is a test message for concurrent bypass testing.", + } + ], + "max_tokens": 50, + } + + async def make_request(request_id: int) -> Dict[str, Any]: + data = request_data.copy() + try: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + return { + "request_id": request_id, + "success": True, + "reserved_tokens": data.get(TPM_RESERVED_TOKENS_KEY, 0), + } + except Exception as e: + return { + "request_id": request_id, + "success": False, + "error": str(e), + "status_code": getattr(e, "status_code", None), + } + + tasks = [make_request(i) for i in range(5)] + results = await asyncio.gather(*tasks) + + successful = [r for r in results if r["success"]] + failed = [r for r in results if not r["success"]] + rate_limited = [r for r in failed if r.get("status_code") == 429] + + assert len(rate_limited) > 0, ( + f"Expected some rate-limited requests but all {len(successful)} succeeded — " + f"the concurrent bypass bug is still present." + ) + + +@pytest.mark.asyncio +async def test_no_leak_on_over_limit_rejection(rate_limiter): + """ + When a reservation would exceed the TPM limit, the counter must NOT be + bumped. Otherwise rejected requests would silently consume quota with no + path to refund (the failure callback only fires after the reservation + was successfully stashed). + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-no-leak"), + tpm_limit=10, # tiny limit, easy to blow past + ) + counter_key = handler.create_rate_limit_keys( + key="api_key", value=user_api_key_dict.api_key, rate_limit_type="tokens" + ) + + # Reservation will estimate >> 10 tokens, so this should be rejected. + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "x" * 200}], + "max_tokens": 200, + } + + estimated = handler._estimate_tokens_for_request(data=data) + assert estimated > user_api_key_dict.tpm_limit, ( + "Test assumes the reservation amount blows past the limit; " + f"estimated={estimated}, limit={user_api_key_dict.tpm_limit}" + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429 + + # The reservation bump (estimated_tokens) must NOT have committed. The + # counter may carry a tiny pre-existing bump from should_rate_limit's + # per-request +1 sliding-window logic, but it must be far below the + # reservation amount — proving the all-or-nothing primitive rolled back + # cleanly on rejection. + cached_value = await cache.async_get_cache(key=counter_key, local_only=True) + cached_int = int(cached_value or 0) + assert cached_int < estimated, ( + f"Reservation leaked: counter={cached_int} after rejection of an " + f"estimated_tokens={estimated} reservation." + ) + + +@pytest.mark.asyncio +async def test_token_adjustment_on_success(rate_limiter): + """ + On success a reserved scope's counter is reconciled to actual via + `actual - reserved`. With actual=50 and reserved=100, the api_key + counter should see a -50 delta — and only because api_key was + reserved against. Unreserved scopes get the full +actual instead. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-test-adjust") + + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], + } + }, + "model": "gpt-3.5-turbo", + } + + mock_response = ModelResponse( + id="test", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-3.5-turbo", + usage=Usage(prompt_tokens=20, completion_tokens=30, total_tokens=50), + choices=[], + ) + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append( + { + "key": op["key"], + "increment": op["increment_value"], + } + ) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_success_event( + kwargs=mock_kwargs, + response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + token_adjustments = [i for i in increments if "tokens" in i["key"]] + + assert any(i["increment"] == -50 for i in token_adjustments), ( + f"Expected a -50 token adjustment (50 actual - 100 reserved) but got: " + f"{token_adjustments}" + ) + + +@pytest.mark.asyncio +async def test_token_release_on_failure(rate_limiter): + """On failure the entire reservation must be refunded — but only against + scopes that were actually charged at pre-call. Unreserved scopes were + never incremented and must not receive a -reserved op (would drift + negative).""" + handler, _cache = rate_limiter + + api_key = hash_token("sk-test-fail") + + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], + } + }, + } + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append( + { + "key": op["key"], + "increment": op["increment_value"], + } + ) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_failure_event( + kwargs=mock_kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + token_releases = [i for i in increments if "tokens" in i["key"]] + + assert any( + i["increment"] == -100 for i in token_releases + ), f"Expected the full reservation (-100) to be released, got: {token_releases}" + + +@pytest.mark.asyncio +async def test_model_scope_refund_targets_reserved_model(rate_limiter): + """ + The pre-call reservation is charged against ``data["model"]`` but the + router later writes ``model_group`` into ``litellm_params.metadata``, + which can be ``None`` or a different value. Reconciliation MUST refund the + same model-scoped counter that was incremented; otherwise model-level + counters (model_per_team / model_per_key / etc.) drift up forever. + + This test makes ``model_group`` absent from kwargs (the failure mode in + the Greptile P1) and asserts the refund still targets the model the + reservation used. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-test-model-mismatch") + team_id = "team-abc" + reserved_model = "gpt-4o-mini" + + mock_kwargs = { + # NOTE: no litellm_params.metadata.model_group — get_model_group_from_litellm_kwargs + # returns None on this kwargs dict. + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + "user_api_key_team_id": team_id, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_MODEL_KEY: reserved_model, + TPM_RESERVED_SCOPES_KEY: [ + ["model_per_team", f"{team_id}:{reserved_model}"] + ], + } + }, + } + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_failure_event( + kwargs=mock_kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + expected_model_per_team_key = handler.create_rate_limit_keys( + key="model_per_team", + value=f"{team_id}:{reserved_model}", + rate_limit_type="tokens", + ) + matching = [i for i in increments if i["key"] == expected_model_per_team_key] + assert matching, ( + f"Expected a refund on the reserved model_per_team counter " + f"({expected_model_per_team_key}) but got: " + f"{[i['key'] for i in increments]}" + ) + assert matching[0]["increment"] == -100, ( + f"Expected full -100 refund on model_per_team counter, got " + f"{matching[0]['increment']}" + ) + + +@pytest.mark.asyncio +async def test_should_rate_limit_does_not_inflate_tokens_counter(rate_limiter): + """ + The pre-call sliding-window check (`should_rate_limit`) must not bump the + `:tokens` counter. That counter is owned exclusively by the atomic + `reserve_tpm_tokens` path; double-handling shrinks the effective TPM + budget by 1 per concurrent in-flight request. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-no-tokens-inflation") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + rpm_limit=100, + tpm_limit=10_000, + ) + + tokens_counter_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 10, + } + + estimated = handler._estimate_tokens_for_request(data=data) + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + cached = int( + await cache.async_get_cache(key=tokens_counter_key, local_only=True) or 0 + ) + + # The :tokens counter should reflect ONLY the reservation amount — not + # an additional +1 from the should_rate_limit pre-pass. + assert cached == estimated, ( + f"Expected :tokens counter to equal the reservation ({estimated}) " + f"with no +1 inflation from should_rate_limit, got {cached}" + ) + + +@pytest.mark.asyncio +async def test_concurrent_burst_within_tpm_budget_all_succeed(rate_limiter): + """ + With a TPM limit comfortably above (N concurrent × per-request reservation), + all N requests must succeed. Pre-fix the should_rate_limit +1-per-key + inflation could 429 late arrivals on tight budgets. + """ + handler, cache = rate_limiter + + user_api_key_dict = UserAPIKeyAuth( + api_key=hash_token("sk-burst-budget"), + tpm_limit=1000, + rpm_limit=100, + ) + + request_data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "x" * 40}], # ~10 input tokens + "max_tokens": 100, + } + + estimated_per_request = handler._estimate_tokens_for_request(data=request_data) + n_concurrent = 3 + # Sanity: total reservation must fit within tpm_limit and we want enough + # headroom that any +1 inflation would NOT push us over. + assert estimated_per_request * n_concurrent < user_api_key_dict.tpm_limit + + async def make_request(request_id: int): + data = request_data.copy() + try: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + return True + except Exception: + return False + + results = await asyncio.gather(*[make_request(i) for i in range(n_concurrent)]) + + assert all(results), ( + f"All {n_concurrent} requests should fit within tpm_limit=" + f"{user_api_key_dict.tpm_limit} (estimated_per_request=" + f"{estimated_per_request}), but only {sum(results)} succeeded — " + f"the should_rate_limit :tokens-counter inflation bug is back." + ) + + +@pytest.mark.asyncio +async def test_org_scope_refund_on_failure(rate_limiter): + """ + The plain `organization` scope is reserved upfront (it carries + tokens_per_unit) — so on failure, the full reservation must be released + against {organization:org_id}:tokens. Pre-fix this scope was missing + from `_build_tpm_scope_pipeline_operations`, leaking forever. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-org-refund") + org_id = "org-acme" + + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + "user_api_key_org_id": org_id, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], + } + }, + } + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_failure_event( + kwargs=mock_kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + expected_org_key = handler.create_rate_limit_keys( + key="organization", value=org_id, rate_limit_type="tokens" + ) + matching = [i for i in increments if i["key"] == expected_org_key] + assert matching, ( + f"Expected a refund on the org tokens counter ({expected_org_key}) " + f"but got keys: {[i['key'] for i in increments]}" + ) + assert ( + matching[0]["increment"] == -100 + ), f"Expected full -100 refund on org counter, got {matching[0]['increment']}" + + +@pytest.mark.asyncio +async def test_org_scope_reconciled_on_success(rate_limiter): + """ + On success the org tokens counter must be reconciled to actual usage. + With reserved=100 and actual=50, the org scope should see a -50 delta. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-org-success") + org_id = "org-acme" + + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + "user_api_key_org_id": org_id, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["organization", org_id]], + } + }, + "model": "gpt-3.5-turbo", + } + + mock_response = ModelResponse( + id="test", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-3.5-turbo", + usage=Usage(prompt_tokens=20, completion_tokens=30, total_tokens=50), + choices=[], + ) + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_success_event( + kwargs=mock_kwargs, + response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + expected_org_key = handler.create_rate_limit_keys( + key="organization", value=org_id, rate_limit_type="tokens" + ) + matching = [i for i in increments if i["key"] == expected_org_key] + assert matching, ( + f"Expected a reconciliation op on the org tokens counter " + f"({expected_org_key}), got keys: {[i['key'] for i in increments]}" + ) + assert matching[0]["increment"] == -50, ( + f"Expected -50 delta on org counter (50 actual - 100 reserved), got " + f"{matching[0]['increment']}" + ) + + +@pytest.mark.asyncio +async def test_estimate_tokens_uses_max_tokens_when_explicit(rate_limiter): + """When max_tokens is set explicitly, reservation should equal input + max_tokens.""" + handler, _cache = rate_limiter + + estimate = handler._estimate_tokens_for_request( + data={ + "messages": [ + {"role": "user", "content": "abcd" * 4} + ], # 16 chars ~ 4 tokens + "max_tokens": 25, + } + ) + # input ~= 16/4 = 4 tokens; max_tokens = 25; total ~= 29 + assert estimate == 4 + 25 + + +@pytest.mark.asyncio +async def test_estimate_tokens_zero_for_empty_embeddings(rate_limiter): + """Embeddings have no output budget — reservation should equal input only.""" + handler, _cache = rate_limiter + + estimate = handler._estimate_tokens_for_request( + data={"input": "hello world"} # 11 chars + ) + # input ~= 11/4 = 2 tokens (max(1, 11//4)); max_tokens = 0 + assert estimate == 2 + + +@pytest.mark.asyncio +async def test_contentless_request_reserves_minimum(rate_limiter): + """ + A contentless request (no messages/prompt/input — e.g. /responses, + tool-call continuations) must still hit the atomic counter so concurrent + contentless requests don't all observe "under limit". Pre-fix the + `has_estimable_content` short-circuit skipped the reservation entirely + and post-call reconciliation provided no backpressure. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-contentless") + user_api_key_dict = UserAPIKeyAuth(api_key=api_key, tpm_limit=2) + + counter_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + + # Two contentless requests should consume two slots of the 2-token + # budget. The third must 429. + for _ in range(2): + data = {"model": "gpt-3.5-turbo"} + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + assert ( + data.get(TPM_RESERVED_TOKENS_KEY) == 1 + ), "Contentless request should reserve the floor of 1 token" + + counter_after_two = int( + await cache.async_get_cache(key=counter_key, local_only=True) or 0 + ) + assert counter_after_two == 2, ( + f"After two contentless requests at the floor, the api_key tokens " + f"counter should be 2, got {counter_after_two}" + ) + + with pytest.raises(Exception) as exc_info: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data={"model": "gpt-3.5-turbo"}, + call_type="", + ) + assert getattr(exc_info.value, "status_code", None) == 429, ( + "Third contentless request must be rate-limited; pre-fix it would " + "have bypassed the TPM check entirely." + ) + + +@pytest.mark.asyncio +async def test_atomic_keys_share_hash_tag_per_descriptor(rate_limiter): + """ + Cluster safety: every key in a single descriptor's Lua payload must + share a `{key:value}` hash tag so the call lands on a single Redis + Cluster slot. Otherwise the Lua script raises CROSSSLOT in cluster mode. + """ + handler, _cache = rate_limiter + + descriptors = [ + { + "key": "api_key", + "value": "abc", + "rate_limit": { + "requests_per_unit": 10, + "tokens_per_unit": 100, + "window_size": 60, + }, + }, + { + "key": "user", + "value": "xyz", + "rate_limit": {"tokens_per_unit": 200, "window_size": 60}, + }, + ] + increments = [{"requests": 1, "tokens": 10}, {"tokens": 10}] + + for descriptor, inc in zip(descriptors, increments): + keys, _args, _meta = handler._build_descriptor_atomic_payload( + descriptor=descriptor, + increment_amounts=inc, + ) + # All keys in a descriptor's payload must share the same {tag} + # — that's the prefix between the first '{' and '}'. + tags = {k[: k.index("}") + 1] for k in keys} + assert len(tags) == 1, ( + f"Descriptor {descriptor['key']}:{descriptor['value']} produced " + f"keys spanning multiple hash tags: {tags}. Redis Cluster would " + f"reject this Lua call with CROSSSLOT." + ) + expected_tag = f"{{{descriptor['key']}:{descriptor['value']}}}" + assert tags == {expected_tag}, f"Expected hash tag {expected_tag}, got {tags}" + + +@pytest.mark.asyncio +async def test_reservation_released_on_proxy_rejection(rate_limiter): + """ + If the request is rejected after the pre-call reservation succeeds but + before the LLM call (e.g. a downstream guardrail/auth hook raises), + `async_post_call_failure_hook` must release the reservation. Otherwise + the tokens leak — `async_log_failure_event` is a litellm completion + callback and never fires for proxy-side rejections. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-leak-fix") + user_api_key_dict = UserAPIKeyAuth(api_key=api_key, tpm_limit=1000) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + reserved = data[TPM_RESERVED_TOKENS_KEY] + assert reserved > 0 + + counter_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + counter_after_reserve = int( + await cache.async_get_cache(key=counter_key, local_only=True) or 0 + ) + assert counter_after_reserve == reserved + + # Simulate a downstream guardrail rejecting the request. + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("guardrail rejected"), + user_api_key_dict=user_api_key_dict, + ) + + counter_after_release = int( + await cache.async_get_cache(key=counter_key, local_only=True) or 0 + ) + assert counter_after_release == 0, ( + f"Reservation leaked: counter={counter_after_release} after " + f"proxy-level rejection refund (expected 0)." + ) + assert data.get(TPM_RESERVATION_RELEASED_KEY) is True, ( + "Released marker must be stamped to prevent async_log_failure_event " + "from double-refunding." + ) + + +@pytest.mark.asyncio +async def test_reservation_release_idempotent(rate_limiter): + """ + If both `async_post_call_failure_hook` and `async_log_failure_event` end + up firing for the same request, only the first refund applies — the + second sees the released marker and no-ops. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-idempotent") + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + # Shared metadata dict simulates the propagation between + # request_data["metadata"] and kwargs["litellm_params"]["metadata"] — + # the post-call-failure-hook stamps the released marker there, and the + # log-failure-event reads it. + shared_metadata = { + "user_api_key_hash": api_key, + TPM_RESERVED_TOKENS_KEY: 100, + } + + request_data = { + "metadata": shared_metadata, + TPM_RESERVED_TOKENS_KEY: 100, + "_litellm_rate_limit_descriptors": [ + { + "key": "api_key", + "value": api_key, + "rate_limit": {"tokens_per_unit": 10000, "window_size": 60}, + } + ], + } + + await handler.async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("rejected"), + user_api_key_dict=UserAPIKeyAuth(api_key=api_key), + ) + + first_refund_count = len([i for i in increments if "tokens" in i["key"]]) + assert first_refund_count > 0, "First refund should have applied" + + # Now simulate async_log_failure_event firing afterwards. It must see + # the released marker (via shared metadata) and not double-refund. + await handler.async_log_failure_event( + kwargs={ + "litellm_params": {"metadata": shared_metadata}, + "standard_logging_object": {"metadata": shared_metadata}, + }, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + second_refund_count = len([i for i in increments if "tokens" in i["key"]]) + assert second_refund_count == first_refund_count, ( + f"Idempotency violated: refund count went from {first_refund_count} " + f"to {second_refund_count} after second hook fired." + ) + + +@pytest.mark.asyncio +async def test_unreserved_scopes_charged_actual_not_delta_on_success(rate_limiter): + """ + Counter-drift fix: a scope present in metadata but NOT reserved at + pre-call (no configured TPM limit for it) must be charged the full + `actual_tokens` on success — never the `delta = actual - reserved`. + Otherwise that scope's counter goes negative whenever `actual < reserved` + (the common case, since the reservation includes a conservative output + pad). + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-mixed-scopes") + team_id = "team-no-tpm-limit" + + # Reservation ONLY hit api_key — team had no TPM limit configured. + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + "user_api_key_team_id": team_id, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], + } + }, + "model": "gpt-3.5-turbo", + } + + mock_response = ModelResponse( + id="t", + object="chat.completion", + created=int(datetime.now().timestamp()), + model="gpt-3.5-turbo", + usage=Usage(prompt_tokens=20, completion_tokens=30, total_tokens=50), + choices=[], + ) + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_success_event( + kwargs=mock_kwargs, + response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + api_key_token_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + team_token_key = handler.create_rate_limit_keys( + key="team", value=team_id, rate_limit_type="tokens" + ) + + api_key_ops = [i for i in increments if i["key"] == api_key_token_key] + team_ops = [i for i in increments if i["key"] == team_token_key] + + assert api_key_ops and api_key_ops[0]["increment"] == -50, ( + f"Reserved api_key scope must reconcile via delta (50-100=-50), " + f"got {api_key_ops}" + ) + assert team_ops and team_ops[0]["increment"] == 50, ( + f"Unreserved team scope must be charged full actual (+50), not the " + f"-50 delta (which would drift its counter negative). Got {team_ops}" + ) + + +@pytest.mark.asyncio +async def test_unreserved_scopes_not_refunded_on_failure(rate_limiter): + """ + Failure refund must only emit ops against scopes the reservation + actually charged. Refunding an unreserved scope (which was never + incremented at pre-call) would drive its counter to -reserved. + """ + handler, _cache = rate_limiter + + api_key = hash_token("sk-mixed-fail") + team_id = "team-no-tpm" + + mock_kwargs = { + "standard_logging_object": { + "metadata": { + "user_api_key_hash": api_key, + "user_api_key_team_id": team_id, + TPM_RESERVED_TOKENS_KEY: 100, + TPM_RESERVED_SCOPES_KEY: [["api_key", api_key]], + } + }, + } + + increments = [] + + async def mock_increment(increment_list, **kwargs): + for op in increment_list: + increments.append({"key": op["key"], "increment": op["increment_value"]}) + + handler.internal_usage_cache.dual_cache.async_increment_cache_pipeline = ( + mock_increment + ) + + await handler.async_log_failure_event( + kwargs=mock_kwargs, + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + team_token_key = handler.create_rate_limit_keys( + key="team", value=team_id, rate_limit_type="tokens" + ) + api_key_token_key = handler.create_rate_limit_keys( + key="api_key", value=api_key, rate_limit_type="tokens" + ) + + team_ops = [i for i in increments if i["key"] == team_token_key] + api_key_ops = [i for i in increments if i["key"] == api_key_token_key] + + assert not team_ops, ( + f"Unreserved team scope must NOT be refunded (would drift negative), " + f"got {team_ops}" + ) + assert ( + api_key_ops and api_key_ops[0]["increment"] == -100 + ), f"Reserved api_key scope must be refunded -100, got {api_key_ops}" + + +@pytest.mark.asyncio +async def test_token_rate_limit_headers_present_in_stored_response(rate_limiter): + """ + With `skip_tpm_check=True` on the RPM sliding-window pass, token statuses + only come from `reserve_tpm_tokens`. They must be merged into + `data["litellm_proxy_rate_limit_response"]` so the post-call hook can + emit `x-ratelimit-{key}-remaining-tokens` / `-limit-tokens` headers to + the client. + """ + handler, cache = rate_limiter + + api_key = hash_token("sk-headers") + user_api_key_dict = UserAPIKeyAuth( + api_key=api_key, + rpm_limit=100, + tpm_limit=10_000, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 20, + } + + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="", + ) + + response = data.get("litellm_proxy_rate_limit_response") + assert isinstance( + response, dict + ), "Expected litellm_proxy_rate_limit_response to be set after pre-call" + + statuses = response.get("statuses") or [] + token_statuses = [s for s in statuses if s.get("rate_limit_type") == "tokens"] + request_statuses = [s for s in statuses if s.get("rate_limit_type") == "requests"] + + assert token_statuses, ( + f"Token rate-limit status missing from stored response. Without it, " + f"x-ratelimit-*-tokens headers never reach the client. Got " + f"statuses: {[(s.get('descriptor_key'), s.get('rate_limit_type')) for s in statuses]}" + ) + assert request_statuses, ( + "RPM rate-limit status was clobbered by the TPM merge — both must " + "coexist in the stored response." + ) + + # The token status carries the limit and a positive remaining budget. + api_key_tokens = next( + (s for s in token_statuses if s.get("descriptor_key") == "api_key"), + None, + ) + assert api_key_tokens is not None, f"api_key token status absent: {token_statuses}" + assert api_key_tokens["current_limit"] == 10_000 + assert api_key_tokens["limit_remaining"] >= 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "-s"]) diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index ad53e87e555..ad893012807 100644 --- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -4,6 +4,7 @@ import pytest from fastapi import HTTPException from litellm.proxy._types import ( + LiteLLM_UserTable, LitellmUserRoles, NewUserRequest, NewUserResponse, @@ -16,8 +17,8 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( _process_group_patch_operations, create_group, create_user, + get_users, get_service_provider_config, - patch_group, patch_user, update_group, update_user, @@ -259,6 +260,124 @@ async def test_scim_create_user_respects_default_role_set_via_ui(mocker, monkeyp ) +@pytest.mark.asyncio +async def test_get_users_filters_username_by_exposed_scim_username_for_okta(mocker): + """ + Okta deprovisioning first locates a user with `userName eq ""`. + LiteLLM exposes SCIM userName from user_email, so the lookup must match + user_email even when the internal user_id is a UUID. + """ + user = LiteLLM_UserTable( + user_id="internal-user-id", + user_email="okta.user@example.com", + user_alias="Okta User", + teams=[], + metadata={}, + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[user]) + mock_prisma_client.db.litellm_usertable.count = AsyncMock(return_value=1) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock( + return_value=SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + id="internal-user-id", + userName="okta.user@example.com", + emails=[SCIMUserEmail(value="okta.user@example.com")], + ) + ), + ) + + response = await get_users( + startIndex=1, + count=10, + filter='userName eq "okta.user@example.com"', + ) + + expected_where = { + "OR": [ + {"user_email": "okta.user@example.com"}, + {"user_id": "okta.user@example.com"}, + ] + } + mock_prisma_client.db.litellm_usertable.find_many.assert_awaited_once_with( + where=expected_where, + skip=0, + take=10, + order={"created_at": "desc"}, + ) + mock_prisma_client.db.litellm_usertable.count.assert_awaited_once_with( + where=expected_where + ) + assert response.totalResults == 1 + assert response.Resources[0].id == "internal-user-id" + + +@pytest.mark.asyncio +async def test_get_users_filters_email_value_by_user_email(mocker): + """ + SCIM clients can locate users with `emails.value eq ""`; keep that + filter as a direct user_email lookup alongside the userName fallback query. + """ + user = LiteLLM_UserTable( + user_id="internal-user-id", + user_email="scim.user@example.com", + user_alias="SCIM User", + teams=[], + metadata={}, + ) + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=[user]) + mock_prisma_client.db.litellm_usertable.count = AsyncMock(return_value=1) + + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", + AsyncMock(return_value=mock_prisma_client), + ) + mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user", + AsyncMock( + return_value=SCIMUser( + schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], + id="internal-user-id", + userName="scim.user@example.com", + emails=[SCIMUserEmail(value="scim.user@example.com")], + ) + ), + ) + + response = await get_users( + startIndex=1, + count=10, + filter='emails.value eq "scim.user@example.com"', + ) + + expected_where = {"user_email": "scim.user@example.com"} + mock_prisma_client.db.litellm_usertable.find_many.assert_awaited_once_with( + where=expected_where, + skip=0, + take=10, + order={"created_at": "desc"}, + ) + mock_prisma_client.db.litellm_usertable.count.assert_awaited_once_with( + where=expected_where + ) + assert response.totalResults == 1 + assert response.Resources[0].id == "internal-user-id" + + @pytest.mark.asyncio async def test_handle_existing_user_by_email_no_email(mocker): """Should return None when new_user_request has no email""" @@ -1337,7 +1456,7 @@ async def test_create_group_with_nonexistent_users_creates_when_flag_true( ) # Execute the create_group function - should succeed - result = await create_group(group=scim_group) + await create_group(group=scim_group) # Verify users were created assert mock_create_user.call_count == 2 diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index 54fbac1264b..dc983aa26fd 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -83,41 +83,100 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): mock_prisma = MagicMock() mock_prisma.db = MagicMock() - # query_raw returns list of dicts (pre-aggregated by GROUP BY) + # query_raw now returns rollup rows produced by GROUPING SETS, each + # tagged with its grouping level via GROUPING_ID(). The dispatcher + # places each row directly in its bucket without Python-side summing. + # GROUPING_ID values for relevant levels (date, api_key, model, + # model_group, custom_llm_provider, mcp, endpoint): + # () grand total = 127 + # (date) = 63 + # (date, endpoint) = 62 + # (date, endpoint, api_key) = 30 + base = { + "model": None, + "model_group": None, + "custom_llm_provider": None, + "mcp_namespaced_tool_name": None, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "failed_requests": 0, + } mock_rows = [ + # (date, endpoint) — rolls up across api_keys and models { + **base, "date": "2024-01-01", "endpoint": "/v1/chat/completions", - "api_key": "key-1", - "model": "gpt-4", - "model_group": None, - "custom_llm_provider": "openai", - "mcp_namespaced_tool_name": None, + "api_key": None, + "group_level": 62, "spend": 15.0, "prompt_tokens": 150, "completion_tokens": 75, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, "api_requests": 2, "successful_requests": 2, - "failed_requests": 0, }, { + **base, "date": "2024-01-01", "endpoint": "/v1/embeddings", - "api_key": "key-2", - "model": "text-embedding-ada-002", - "model_group": None, - "custom_llm_provider": "openai", - "mcp_namespaced_tool_name": None, + "api_key": None, + "group_level": 62, "spend": 3.0, "prompt_tokens": 30, "completion_tokens": 0, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, "api_requests": 1, "successful_requests": 1, - "failed_requests": 0, + }, + # (date, endpoint, api_key) — populates the per-key sub-bucket + { + **base, + "date": "2024-01-01", + "endpoint": "/v1/chat/completions", + "api_key": "key-1", + "group_level": 30, + "spend": 15.0, + "prompt_tokens": 150, + "completion_tokens": 75, + "api_requests": 2, + "successful_requests": 2, + }, + { + **base, + "date": "2024-01-01", + "endpoint": "/v1/embeddings", + "api_key": "key-2", + "group_level": 30, + "spend": 3.0, + "prompt_tokens": 30, + "completion_tokens": 0, + "api_requests": 1, + "successful_requests": 1, + }, + # (date) — per-date totals + { + **base, + "date": "2024-01-01", + "endpoint": None, + "api_key": None, + "group_level": 63, + "spend": 18.0, + "prompt_tokens": 180, + "completion_tokens": 75, + "api_requests": 3, + "successful_requests": 3, + }, + # () — grand total + { + **base, + "date": None, + "endpoint": None, + "api_key": None, + "group_level": 127, + "spend": 18.0, + "prompt_tokens": 180, + "completion_tokens": 75, + "api_requests": 3, + "successful_requests": 3, }, ] @@ -449,24 +508,43 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): mock_prisma = MagicMock() mock_prisma.db = MagicMock() - # query_raw returns list of dicts (pre-aggregated by GROUP BY) + # GROUPING SETS rollup rows. The api_key metadata lookup is driven + # by any non-NULL api_key in the result set, so the (date, endpoint, + # api_key) row at level 30 is what ensures get_api_key_metadata is + # called for "deleted-key-hash". + base = { + "model": None, + "model_group": None, + "custom_llm_provider": None, + "mcp_namespaced_tool_name": None, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "failed_requests": 0, + } mock_rows = [ { + **base, "date": "2024-01-01", "endpoint": "/v1/chat/completions", - "api_key": "deleted-key-hash", - "model": "gpt-4", - "model_group": None, - "custom_llm_provider": "openai", - "mcp_namespaced_tool_name": None, + "api_key": None, + "group_level": 62, + "spend": 10.0, + "prompt_tokens": 100, + "completion_tokens": 50, + "api_requests": 1, + "successful_requests": 1, + }, + { + **base, + "date": "2024-01-01", + "endpoint": "/v1/chat/completions", + "api_key": "deleted-key-hash", + "group_level": 30, "spend": 10.0, "prompt_tokens": 100, "completion_tokens": 50, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, "api_requests": 1, "successful_requests": 1, - "failed_requests": 0, }, ] diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 24b8e1595b4..f0909afcbf6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -1553,6 +1553,98 @@ class TestTemporaryMCPSessionEndpoints: assert "permission" in str(exc_info.value) + @pytest.mark.asyncio + async def test_mcp_oauth_user_api_key_auth_falls_back_to_token_cookie(self): + """ + When the Authorization header is absent but a valid 'token' cookie is + present (browser navigation), _mcp_oauth_user_api_key_auth should + decode the cookie JWT and authenticate via the API key stored in it. + """ + import jwt + + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _mcp_oauth_user_api_key_auth, + ) + + master_key = "test-master-key" + api_key_in_cookie = "sk-test-cookie-key" + token_cookie = jwt.encode( + { + "user_id": "user@example.com", + "key": api_key_in_cookie, + "user_role": "proxy_admin", + "login_method": "sso", + }, + master_key, + algorithm="HS256", + ) + + mock_request = MagicMock() + mock_request.headers = {} + mock_request.cookies = {"token": token_cookie} + + expected_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key=api_key_in_cookie + ) + fake_proxy_server = types.SimpleNamespace(master_key=master_key) + + with ( + patch.dict(sys.modules, {"litellm.proxy.proxy_server": fake_proxy_server}), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_api_key_auth_builder", + AsyncMock(return_value=expected_auth), + ) as auth_builder_mock, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._read_request_body", + AsyncMock(return_value={}), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.populate_request_with_path_params", + side_effect=lambda request_data, request: request_data, + ), + ): + result = await _mcp_oauth_user_api_key_auth(mock_request) + + assert result is expected_auth + _, call_kwargs = auth_builder_mock.call_args + assert call_kwargs["api_key"] == f"Bearer {api_key_in_cookie}" + + @pytest.mark.asyncio + async def test_mcp_oauth_user_api_key_auth_uses_authorization_header_when_present( + self, + ): + """When Authorization header is present it takes priority over the cookie.""" + from litellm.proxy.management_endpoints.mcp_management_endpoints import ( + _mcp_oauth_user_api_key_auth, + ) + + expected_auth = generate_mock_user_api_key_auth( + user_role=LitellmUserRoles.PROXY_ADMIN + ) + mock_request = MagicMock() + mock_request.headers = {"Authorization": "Bearer sk-header-key"} + mock_request.cookies = {} + + with ( + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._user_api_key_auth_builder", + AsyncMock(return_value=expected_auth), + ) as auth_builder_mock, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints._read_request_body", + AsyncMock(return_value={}), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.populate_request_with_path_params", + side_effect=lambda request_data, request: request_data, + ), + ): + result = await _mcp_oauth_user_api_key_auth(mock_request) + + assert result is expected_auth + _, call_kwargs = auth_builder_mock.call_args + assert call_kwargs["api_key"] == "Bearer sk-header-key" + @pytest.mark.asyncio async def test_mcp_authorize_proxies_to_discoverable_endpoint(self): from litellm.proxy.management_endpoints.mcp_management_endpoints import ( diff --git a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py index 987f17fe075..48e4353966b 100644 --- a/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py +++ b/tests/test_litellm/proxy/management_helpers/test_audit_log_callbacks.py @@ -346,3 +346,126 @@ class TestS3LoggerAuditLogEvent: element = logger.log_queue[0] assert element.s3_object_key.startswith("audit_logs/") assert "audit-456" in element.s3_object_key + + +class TestS3AuditCallbackParamsDecoupling: + """`s3_audit_callback_params` should give the audit-log path its own + S3Logger instance, distinct from the singleton serving normal logs.""" + + @pytest.fixture(autouse=True) + def _isolate_caches_and_globals(self): + from litellm.litellm_core_utils import litellm_logging as ll_logging + from litellm.proxy.management_helpers import audit_logs as ll_audit_logs + + original_s3 = litellm.s3_callback_params + original_audit = getattr(litellm, "s3_audit_callback_params", None) + ll_audit_logs._audit_log_callback_cache.clear() + ll_logging._in_memory_loggers.clear() + yield + litellm.s3_callback_params = original_s3 + litellm.s3_audit_callback_params = original_audit + ll_audit_logs._audit_log_callback_cache.clear() + ll_logging._in_memory_loggers.clear() + + def test_opt_in_constructs_separate_instance_with_audit_config(self): + """Audit config set → audit resolver returns a fresh S3Logger pointing + at the audit bucket, distinct from the normal-log singleton.""" + from litellm.integrations.s3_v2 import S3Logger + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + from litellm.proxy.management_helpers.audit_logs import ( + _resolve_audit_log_callback, + ) + + litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} + litellm.s3_audit_callback_params = {"s3_bucket_name": "audit-bucket"} + + with patch("asyncio.create_task"): + audit_instance = _resolve_audit_log_callback("s3_v2") + normal_instance = _init_custom_logger_compatible_class( + logging_integration="s3_v2", + internal_usage_cache=None, + llm_router=None, + ) + + assert isinstance(audit_instance, S3Logger) + assert isinstance(normal_instance, S3Logger) + assert id(audit_instance) != id(normal_instance) + assert audit_instance.s3_bucket_name == "audit-bucket" + assert normal_instance.s3_bucket_name == "normal-bucket" + + def test_opt_out_preserves_singleton_behavior(self): + """No `s3_audit_callback_params` → audit and normal share the singleton + (existing behavior, regression guard).""" + from litellm.integrations.s3_v2 import S3Logger + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + from litellm.proxy.management_helpers.audit_logs import ( + _resolve_audit_log_callback, + ) + + litellm.s3_callback_params = {"s3_bucket_name": "shared-bucket"} + litellm.s3_audit_callback_params = None + + with patch("asyncio.create_task"): + normal_instance = _init_custom_logger_compatible_class( + logging_integration="s3_v2", + internal_usage_cache=None, + llm_router=None, + ) + audit_instance = _resolve_audit_log_callback("s3_v2") + + assert isinstance(audit_instance, S3Logger) + assert id(audit_instance) == id(normal_instance) + assert audit_instance.s3_bucket_name == "shared-bucket" + + def test_empty_dict_opts_in(self): + """`s3_audit_callback_params = {}` is opt-in (truthy-by-presence) and + produces a separate instance with no bucket configured (env/IAM-only).""" + from litellm.integrations.s3_v2 import S3Logger + from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + ) + from litellm.proxy.management_helpers.audit_logs import ( + _resolve_audit_log_callback, + ) + + litellm.s3_callback_params = {"s3_bucket_name": "normal-bucket"} + litellm.s3_audit_callback_params = {} + + with patch("asyncio.create_task"): + audit_instance = _resolve_audit_log_callback("s3_v2") + normal_instance = _init_custom_logger_compatible_class( + logging_integration="s3_v2", + internal_usage_cache=None, + llm_router=None, + ) + + assert id(audit_instance) != id(normal_instance) + assert audit_instance.s3_bucket_name is None + assert normal_instance.s3_bucket_name == "normal-bucket" + + def test_reset_audit_log_callback_cache_clears_audit_instance(self): + """`reset_audit_log_callback_cache()` must drop the cached audit + instance so a config reload picks up the new params.""" + from litellm.proxy.management_helpers.audit_logs import ( + _audit_log_callback_cache, + _resolve_audit_log_callback, + reset_audit_log_callback_cache, + ) + + litellm.s3_audit_callback_params = {"s3_bucket_name": "first"} + with patch("asyncio.create_task"): + first = _resolve_audit_log_callback("s3_v2") + assert first is not None and "s3_v2" in _audit_log_callback_cache + + reset_audit_log_callback_cache() + assert "s3_v2" not in _audit_log_callback_cache + + litellm.s3_audit_callback_params = {"s3_bucket_name": "second"} + second = _resolve_audit_log_callback("s3_v2") + assert second is not None + assert id(second) != id(first) + assert second.s3_bucket_name == "second" diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index d4b4617730b..31a25916e2e 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -3,8 +3,9 @@ import datetime from typing import AsyncGenerator from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest -from fastapi import HTTPException, Request, status +from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse import litellm @@ -26,6 +27,48 @@ from litellm.proxy.utils import ProxyLogging class TestProxyBaseLLMRequestProcessing: + @pytest.mark.asyncio + async def test_base_passthrough_process_llm_request_preserves_litellm_headers_for_non_streaming_response( + self, monkeypatch + ): + processing_obj = ProxyBaseLLMRequestProcessing(data={}) + + async def fake_base_process_llm_request(**kwargs): + passthrough_response = kwargs["fastapi_response"] + passthrough_response.headers["x-litellm-call-id"] = "test-call-id" + passthrough_response.headers["x-litellm-version"] = "test-version" + return httpx.Response( + status_code=200, + content=b'{"ok":true}', + headers={ + "content-type": "application/json", + "x-amzn-requestid": "bedrock-request-id", + }, + ) + + monkeypatch.setattr( + processing_obj, + "base_process_llm_request", + fake_base_process_llm_request, + ) + + result = await processing_obj.base_passthrough_process_llm_request( + request=MagicMock(spec=Request), + fastapi_response=Response(), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + proxy_logging_obj=MagicMock(spec=ProxyLogging), + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=MagicMock(), + model="bedrock-test-model", + ) + + assert result.status_code == 200 + assert result.body == b'{"ok":true}' + assert result.headers["x-amzn-requestid"] == "bedrock-request-id" + assert result.headers["x-litellm-call-id"] == "test-call-id" + assert result.headers["x-litellm-version"] == "test-version" + @pytest.mark.asyncio async def test_common_processing_pre_call_logic_pre_call_hook_receives_litellm_call_id( self, monkeypatch diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index 383f6886d17..92611431a15 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1143,6 +1143,155 @@ async def test_add_litellm_data_to_request_preserves_user_tags_when_team_opts_in assert updated["metadata"].get("tags") == ["team-allowed"] +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_unions_caller_header_tags_with_static_key_tags(): + """Caller-supplied `x-litellm-tags` must union with static key-level + tags, not overwrite them, when `allow_client_tags=True`.""" + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = { + "Content-Type": "application/json", + "x-litellm-tags": "tenant:1681", + } + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "allow_client_tags": True, + "tags": ["team:platform", "env:prod"], + }, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + final_tags = updated["metadata"].get("tags") or [] + assert "team:platform" in final_tags + assert "env:prod" in final_tags + assert "tenant:1681" in final_tags + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_unions_caller_header_tags_with_static_team_tags(): + """Same union behavior must hold for team-level static tags.""" + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = { + "Content-Type": "application/json", + "x-litellm-tags": "tenant:42", + } + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={}, + team_metadata={ + "allow_client_tags": True, + "tags": ["team:eng", "owner:platform"], + }, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + final_tags = updated["metadata"].get("tags") or [] + assert "team:eng" in final_tags + assert "owner:platform" in final_tags + assert "tenant:42" in final_tags + + +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_unions_dedups_overlapping_caller_and_static_tags(): + """A tag that appears in both the static set and the caller header + must show up exactly once in the merged list.""" + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/v1/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = { + "Content-Type": "application/json", + "x-litellm-tags": "env:prod,tenant:7", + } + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = {"model": "gpt-3.5-turbo"} + + user_api_key_dict = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "allow_client_tags": True, + "tags": ["env:prod", "team:platform"], + }, + team_metadata={}, + spend=0.0, + max_budget=100.0, + model_max_budget={}, + team_spend=0.0, + team_max_budget=200.0, + ) + + updated = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + final_tags = updated["metadata"].get("tags") or [] + assert final_tags.count("env:prod") == 1 + assert "team:platform" in final_tags + assert "tenant:7" in final_tags + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_user_spend_and_budget(): from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request diff --git a/tests/test_litellm/proxy/test_model_level_guardrails.py b/tests/test_litellm/proxy/test_model_level_guardrails.py index e83f8c67caa..3d74edd772b 100644 --- a/tests/test_litellm/proxy/test_model_level_guardrails.py +++ b/tests/test_litellm/proxy/test_model_level_guardrails.py @@ -294,3 +294,134 @@ async def test_post_call_success_hook_skips_guardrail_not_on_model(): ) assert guardrail.was_called is False + + +# --------------------------------------------------------------------------- +# Integration test: async_post_call_streaming_iterator_hook with model-level guardrails +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_runs_model_level_guardrail(): + """ + Model-level guardrails configured on a deployment should execute in + async_post_call_streaming_iterator_hook (streaming path) — even when + `default_on: false` and the guardrail is not in the request body. + """ + from litellm.caching.caching import DualCache + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import ProxyLogging + from litellm.types.guardrails import GuardrailEventHooks + + class TestStreamingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="test-model-guardrail", + event_hook=GuardrailEventHooks.post_call, + ) + self.was_called = False + + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + self.was_called = True + async for chunk in response: + yield chunk + + guardrail = TestStreamingGuardrail() + + mock_router = MagicMock() + mock_deployment = MagicMock() + mock_deployment.litellm_params.get.return_value = ["test-model-guardrail"] + mock_router.get_deployment.return_value = mock_deployment + + async def fake_response(): + yield "chunk-1" + yield "chunk-2" + + with ( + patch("litellm.callbacks", [guardrail]), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + ): + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + request_data = { + "model": "gpt-4", + "metadata": {"model_info": {"id": "model-uuid-123"}}, + } + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + chunks = [] + async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_response(), + user_api_key_dict=user_api_key_dict, + request_data=request_data, + ): + chunks.append(chunk) + + assert guardrail.was_called is True + assert chunks == ["chunk-1", "chunk-2"] + + +@pytest.mark.asyncio +async def test_streaming_iterator_hook_skips_guardrail_not_on_model(): + """ + Streaming guardrails NOT configured on the model (and not in the request + body / key / team) should not execute, even after the dispatcher merge + runs. Confirms the gate stays closed for unrelated guardrails. + """ + from litellm.caching.caching import DualCache + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import ProxyLogging + from litellm.types.guardrails import GuardrailEventHooks + + class TestStreamingGuardrail(CustomGuardrail): + def __init__(self): + super().__init__( + guardrail_name="unrelated-guardrail", + event_hook=GuardrailEventHooks.post_call, + ) + self.was_called = False + + async def async_post_call_streaming_iterator_hook( + self, user_api_key_dict, response, request_data + ): + self.was_called = True + async for chunk in response: + yield chunk + + guardrail = TestStreamingGuardrail() + + # Deployment has a DIFFERENT guardrail configured + mock_router = MagicMock() + mock_deployment = MagicMock() + mock_deployment.litellm_params.get.return_value = ["some-other-guardrail"] + mock_router.get_deployment.return_value = mock_deployment + + async def fake_response(): + yield "chunk-1" + + with ( + patch("litellm.callbacks", [guardrail]), + patch("litellm.proxy.proxy_server.llm_router", mock_router), + ): + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + request_data = { + "model": "gpt-4", + "metadata": {"model_info": {"id": "model-uuid-123"}}, + } + user_api_key_dict = UserAPIKeyAuth(api_key="test-key") + + chunks = [] + async for chunk in proxy_logging.async_post_call_streaming_iterator_hook( + response=fake_response(), + user_api_key_dict=user_api_key_dict, + request_data=request_data, + ): + chunks.append(chunk) + + assert guardrail.was_called is False + assert chunks == ["chunk-1"] diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 6fbce4a5458..59b43330a25 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2,7 +2,6 @@ import os import sys from unittest.mock import MagicMock, patch -import fastapi import pytest sys.path.insert( @@ -12,7 +11,6 @@ sys.path.insert( import builtins import types -from litellm.proxy.health_endpoints.health_app_factory import build_health_app from litellm.proxy.proxy_cli import ProxyInitializationHelpers @@ -133,6 +131,87 @@ class TestProxyInitializationHelpers: ) assert args["timeout_worker_healthcheck"] == 15 + def test_get_reload_options_no_config(self): + opts = ProxyInitializationHelpers._get_reload_options(None) + assert opts == {"reload": True} + + def test_get_reload_options_with_config_in_cwd(self, tmp_path, monkeypatch): + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + monkeypatch.chdir(tmp_path) + + opts = ProxyInitializationHelpers._get_reload_options("config.yaml") + + assert opts["reload"] is True + assert opts["reload_dirs"] == [str(tmp_path)] + assert opts["reload_includes"] == ["*.py", "config.yaml"] + + def test_get_reload_options_with_config_outside_cwd(self, tmp_path, monkeypatch): + cwd_dir = tmp_path / "work" + cwd_dir.mkdir() + elsewhere = tmp_path / "configs" + elsewhere.mkdir() + config_file = elsewhere / "proxy.yaml" + config_file.write_text("model_list: []\n") + monkeypatch.chdir(cwd_dir) + + opts = ProxyInitializationHelpers._get_reload_options(str(config_file)) + + assert opts["reload"] is True + assert opts["reload_dirs"] == [str(cwd_dir), str(elsewhere)] + assert opts["reload_includes"] == ["*.py", "proxy.yaml"] + + def test_patch_statreload_for_config_yields_yaml(self, tmp_path): + from pathlib import Path + + from uvicorn.supervisors.statreload import StatReload + + if hasattr(StatReload, "_litellm_patched_config_paths"): + StatReload._litellm_patched_config_paths.clear() + + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + py_file = tmp_path / "module.py" + py_file.write_text("x = 1\n") + + applied = ProxyInitializationHelpers._patch_statreload_for_config( + str(config_file) + ) + assert applied is True + + fake_self = types.SimpleNamespace( + config=types.SimpleNamespace(reload_dirs=[tmp_path]) + ) + yielded_paths = {Path(p).resolve() for p in StatReload.iter_py_files(fake_self)} + + assert config_file.resolve() in yielded_paths + assert py_file.resolve() in yielded_paths + + def test_patch_statreload_for_config_is_idempotent(self, tmp_path): + from pathlib import Path + + from uvicorn.supervisors.statreload import StatReload + + if hasattr(StatReload, "_litellm_patched_config_paths"): + StatReload._litellm_patched_config_paths.clear() + + config_file = tmp_path / "config.yaml" + config_file.write_text("model_list: []\n") + py_file = tmp_path / "only.py" + py_file.write_text("x = 1\n") + + for _ in range(3): + ProxyInitializationHelpers._patch_statreload_for_config(str(config_file)) + + fake_self = types.SimpleNamespace( + config=types.SimpleNamespace(reload_dirs=[tmp_path]) + ) + yielded = list(StatReload.iter_py_files(fake_self)) + assert len(yielded) == len(set(map(str, yielded))) + yielded_paths = {Path(p).resolve() for p in yielded} + assert config_file.resolve() in yielded_paths + assert py_file.resolve() in yielded_paths + @patch("asyncio.run") @patch("builtins.print") def test_init_hypercorn_server(self, mock_print, mock_asyncio_run): @@ -690,62 +769,8 @@ class TestProxyInitializationHelpers: mock_uvicorn_run.assert_called_once() -class TestHealthAppFactory: - """Test cases for the health app factory module""" - - def test_build_health_app(self): - """Test that build_health_app creates a FastAPI app with the correct title and includes the health router""" - # Execute - health_app = build_health_app() - - # Assert - assert health_app.title == "LiteLLM Health Endpoints" - assert isinstance(health_app, fastapi.FastAPI) - - # Verify that the app has the expected health endpoints by checking route paths - # When a router is included, its routes are flattened into the main app's routes - route_paths = [] - for route in health_app.routes: - if hasattr(route, "path"): - route_paths.append(route.path) - - # Check for some expected health endpoints - expected_paths = [ - "/test", - "/health/services", - "/health", - "/health/history", - "/health/latest", - "/settings", - "/active/callbacks", - "/health/readiness", - "/health/liveliness", - "/health/liveness", - "/health/test_connection", - ] - - # At least some of the expected health endpoints should be present - found_paths = [path for path in expected_paths if path in route_paths] - assert ( - len(found_paths) > 0 - ), f"Expected to find health endpoints, but found: {route_paths}" - - # Verify that the app has routes (indicating the router was included) - assert ( - len(health_app.routes) > 0 - ), "Health app should have routes from the included router" - - def test_build_health_app_returns_different_instances(self): - """Test that build_health_app returns different FastAPI instances on each call""" - # Execute - health_app_1 = build_health_app() - health_app_2 = build_health_app() - - # Assert - assert health_app_1 is not health_app_2 - assert health_app_1.title == health_app_2.title - assert isinstance(health_app_1, fastapi.FastAPI) - assert isinstance(health_app_2, fastapi.FastAPI) +class TestRunServerDbSetup: + """Tests for run_server's prisma setup_database behavior.""" @patch("subprocess.run") @patch("atexit.register") diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index 4923d70a437..42bb919295f 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -331,6 +331,172 @@ async def test_delete_old_logs_continues_on_valid_int_return(): assert total_deleted == 800 +@pytest.mark.asyncio +async def test_delete_old_logs_continues_after_single_batch_failure(monkeypatch): + """A single batch failure (e.g. DB timeout) must not abort the whole run — + subsequent batches should still execute and their counts accumulate.""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + # Zero out the failure backoff so the test doesn't take ~0.5s of real sleep. + monkeypatch.setattr( + cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 + ) + + mock_prisma_client = MagicMock() + mock_db = MagicMock() + # batch 1 succeeds, batch 2 raises (one-off DB timeout), batches 3-4 succeed, + # batch 5 returns 0 → loop exits naturally. + mock_db.execute_raw = AsyncMock( + side_effect=[100, TimeoutError("simulated DB timeout"), 200, 50, 0] + ) + mock_prisma_client.db = mock_db + + cleaner = cleanup_module.SpendLogCleanup( + general_settings={"maximum_spend_logs_retention_period": "7d"} + ) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) + total_deleted = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date) + + # All 5 batches should have been attempted; 100 + 200 + 50 = 350 deleted. + assert mock_db.execute_raw.call_count == 5 + assert total_deleted == 350 + + +@pytest.mark.asyncio +async def test_delete_old_logs_aborts_after_consecutive_failures(monkeypatch): + """If batch failures persist for SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES + in a row (e.g. DB is down), the loop must abort instead of hot-looping.""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + # Lower the threshold so the test is fast and deterministic. + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) + monkeypatch.setattr( + cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 + ) + + mock_prisma_client = MagicMock() + mock_db = MagicMock() + # Every batch raises — must abort after exactly 3 attempts, not loop forever. + mock_db.execute_raw = AsyncMock( + side_effect=ConnectionError("simulated persistent DB outage") + ) + mock_prisma_client.db = mock_db + + cleaner = cleanup_module.SpendLogCleanup( + general_settings={"maximum_spend_logs_retention_period": "7d"} + ) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) + total_deleted = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date) + + assert mock_db.execute_raw.call_count == 3 + assert total_deleted == 0 + + +@pytest.mark.asyncio +async def test_delete_old_logs_resets_consecutive_failures_on_success(monkeypatch): + """A success between failures must reset the consecutive-failure counter so + intermittent timeouts don't trip the abort threshold.""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) + monkeypatch.setattr( + cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 + ) + + mock_prisma_client = MagicMock() + mock_db = MagicMock() + # Pattern: fail, fail, success (resets counter), fail, fail, success, done. + # Without reset, three of these would trip abort; with reset, they don't. + mock_db.execute_raw = AsyncMock( + side_effect=[ + TimeoutError("t1"), + TimeoutError("t2"), + 100, + TimeoutError("t3"), + TimeoutError("t4"), + 50, + 0, + ] + ) + mock_prisma_client.db = mock_db + + cleaner = cleanup_module.SpendLogCleanup( + general_settings={"maximum_spend_logs_retention_period": "7d"} + ) + + cutoff_date = datetime.now(timezone.utc) - timedelta(days=7) + total_deleted = await cleaner._delete_old_logs(mock_prisma_client, cutoff_date) + + assert mock_db.execute_raw.call_count == 7 + assert total_deleted == 150 + + +@pytest.mark.asyncio +async def test_cleanup_uses_logger_exception_for_full_traceback(monkeypatch): + """The outer error handler must call logger.exception() (not .error(str(e))) + so Prisma/DB timeouts surface a full traceback and exception type.""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + mock_logger = MagicMock() + monkeypatch.setattr(cleanup_module, "verbose_proxy_logger", mock_logger) + + mock_prisma_client = MagicMock() + # Force the outer try/except to fire by making _should_delete_spend_logs raise. + cleaner = cleanup_module.SpendLogCleanup( + general_settings={"maximum_spend_logs_retention_period": "7d"} + ) + cleaner.pod_lock_manager = None + + def boom(): + raise RuntimeError("simulated prisma timeout") + + cleaner._should_delete_spend_logs = boom # type: ignore[assignment] + + await cleaner.cleanup_old_spend_logs(mock_prisma_client) + + assert mock_logger.exception.called, "expected logger.exception() to be called" + # The exception type name must appear in the formatted args so operators can + # tell *what* failed, not just "Error during cleanup:". + call_args = mock_logger.exception.call_args + formatted = call_args[0][0] % call_args[0][1:] + assert "RuntimeError" in formatted + assert "simulated prisma timeout" in formatted + + +@pytest.mark.asyncio +async def test_cleanup_releases_lock_after_persistent_batch_failures(monkeypatch): + """Even when batch deletion aborts due to consecutive failures, the pod lock + must still be released so the next scheduled run isn't permanently blocked.""" + import litellm.proxy.db.db_transaction_queue.spend_log_cleanup as cleanup_module + + monkeypatch.setattr(cleanup_module, "SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 2) + monkeypatch.setattr( + cleanup_module, "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.0 + ) + + mock_prisma_client = MagicMock() + mock_db = MagicMock() + mock_db.execute_raw = AsyncMock(side_effect=TimeoutError("DB down")) + mock_prisma_client.db = mock_db + + mock_pod_lock_manager = MagicMock() + mock_pod_lock_manager.redis_cache = MagicMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + + cleaner = cleanup_module.SpendLogCleanup( + general_settings={"maximum_spend_logs_retention_period": "7d"} + ) + cleaner.pod_lock_manager = mock_pod_lock_manager + + await cleaner.cleanup_old_spend_logs(mock_prisma_client) + + # Cleanup didn't crash; the abort-after-failures path returned cleanly. + mock_pod_lock_manager.release_lock.assert_awaited_once() + + def test_cleanup_batch_size_env_var(monkeypatch): """Ensure batch size is configurable via environment variable""" import importlib diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index ca4e469de08..b33b2a69bee 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -231,23 +231,23 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-4.1.1.tgz", - "integrity": "sha512-B0Hv6G3gWGMn0xKJ0txEi/jM5iFpT3MfDxmhZFb4W047GvytCf1DHQ1D69W3zHI4yWe2aTZAA0JnbMZ7Xc8DuQ==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-4.1.2.tgz", + "integrity": "sha512-NfBUvBaYgKIuq6E/RBLY1m0IohzNHAYyaJGuTK79Z23uNwmz2jl1mPsC5ZxCCxylinKhT1Amn5oNTlx1wN8cQg==", "dev": true, "license": "MIT", "dependencies": { - "@csstools/css-calc": "^2.1.4", - "@csstools/css-color-parser": "^3.1.0", - "@csstools/css-parser-algorithms": "^3.0.5", - "@csstools/css-tokenizer": "^3.0.4", - "lru-cache": "^11.2.4" + "@csstools/css-calc": "^3.0.0", + "@csstools/css-color-parser": "^4.0.1", + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0", + "lru-cache": "^11.2.5" } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "6.7.7", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.7.7.tgz", - "integrity": "sha512-8CO/UQ4tzDd7ula+/CVimJIVWez99UJlbMyIgk8xOnhAVPKLnBZmUFYVgugS441v2ZqUq5EnSh6B0Ua0liSFAA==", + "version": "6.8.1", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-6.8.1.tgz", + "integrity": "sha512-MvRz1nCqW0fsy8Qz4dnLIvhOlMzqDVBabZx6lH+YywFDdjXhMY37SmpV1XFX3JzG5GWHn63j6HX6QPr3lZXHvQ==", "dev": true, "license": "MIT", "dependencies": { @@ -255,7 +255,7 @@ "bidi-js": "^1.0.3", "css-tree": "^3.1.0", "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.2.5" + "lru-cache": "^11.2.6" } }, "node_modules/@asamuzakjp/nwsapi": { @@ -301,9 +301,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.0.tgz", - "integrity": "sha512-IyDgFV5GeDUVX4YdF/3CPULtVGSXXMLh1xVIgdCgxApktqnQV0r7/8Nqthg+8YLGaAtdyIlo2qIdZrbCv4+7ww==", + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", + "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", "dev": true, "license": "MIT", "dependencies": { @@ -317,9 +317,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", - "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", "license": "MIT", "engines": { "node": ">=6.9.0" @@ -350,9 +350,9 @@ } }, "node_modules/@csstools/color-helpers": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", - "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", + "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", "dev": true, "funding": [ { @@ -366,13 +366,13 @@ ], "license": "MIT-0", "engines": { - "node": ">=18" + "node": ">=20.19.0" } }, "node_modules/@csstools/css-calc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", - "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", + "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", "dev": true, "funding": [ { @@ -386,17 +386,17 @@ ], "license": "MIT", "engines": { - "node": ">=18" + "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^3.0.5", - "@csstools/css-tokenizer": "^3.0.4" + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" } }, "node_modules/@csstools/css-color-parser": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", - "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", + "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", "dev": true, "funding": [ { @@ -410,21 +410,21 @@ ], "license": "MIT", "dependencies": { - "@csstools/color-helpers": "^5.1.0", - "@csstools/css-calc": "^2.1.4" + "@csstools/color-helpers": "^6.0.2", + "@csstools/css-calc": "^3.2.0" }, "engines": { - "node": ">=18" + "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-parser-algorithms": "^3.0.5", - "@csstools/css-tokenizer": "^3.0.4" + "@csstools/css-parser-algorithms": "^4.0.0", + "@csstools/css-tokenizer": "^4.0.0" } }, "node_modules/@csstools/css-parser-algorithms": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", - "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-4.0.0.tgz", + "integrity": "sha512-+B87qS7fIG3L5h3qwJ/IFbjoVoOe/bpOdh9hAjXbvx0o8ImEmUsGXN0inFOnk2ChCFgqkkGFQ+TpM5rbhkKe4w==", "dev": true, "funding": [ { @@ -438,16 +438,16 @@ ], "license": "MIT", "engines": { - "node": ">=18" + "node": ">=20.19.0" }, "peerDependencies": { - "@csstools/css-tokenizer": "^3.0.4" + "@csstools/css-tokenizer": "^4.0.0" } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.0.26", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.0.26.tgz", - "integrity": "sha512-6boXK0KkzT5u5xOgF6TKB+CLq9SOpEGmkZw0g5n9/7yg85wab3UzSxB8TxhLJ31L4SGJ6BCFRw/iftTha1CJXA==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", + "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", "dev": true, "funding": [ { @@ -459,12 +459,20 @@ "url": "https://opencollective.com/csstools" } ], - "license": "MIT-0" + "license": "MIT-0", + "peerDependencies": { + "css-tree": "^3.2.1" + }, + "peerDependenciesMeta": { + "css-tree": { + "optional": true + } + } }, "node_modules/@csstools/css-tokenizer": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", - "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-4.0.0.tgz", + "integrity": "sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==", "dev": true, "funding": [ { @@ -478,25 +486,25 @@ ], "license": "MIT", "engines": { - "node": ">=18" + "node": ">=20.19.0" } }, "node_modules/@emnapi/core": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", - "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", "license": "MIT", "optional": true, "dependencies": { @@ -504,9 +512,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "dev": true, "license": "MIT", "optional": true, @@ -527,9 +535,9 @@ "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", - "integrity": "sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", "cpu": [ "ppc64" ], @@ -544,9 +552,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.2.tgz", - "integrity": "sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", "cpu": [ "arm" ], @@ -561,9 +569,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.2.tgz", - "integrity": "sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", "cpu": [ "arm64" ], @@ -578,9 +586,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.2.tgz", - "integrity": "sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", "cpu": [ "x64" ], @@ -595,9 +603,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.2.tgz", - "integrity": "sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", "cpu": [ "arm64" ], @@ -612,9 +620,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.2.tgz", - "integrity": "sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", "cpu": [ "x64" ], @@ -629,9 +637,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.2.tgz", - "integrity": "sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", "cpu": [ "arm64" ], @@ -646,9 +654,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.2.tgz", - "integrity": "sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", "cpu": [ "x64" ], @@ -663,9 +671,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.2.tgz", - "integrity": "sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", "cpu": [ "arm" ], @@ -680,9 +688,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.2.tgz", - "integrity": "sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", "cpu": [ "arm64" ], @@ -697,9 +705,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.2.tgz", - "integrity": "sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", "cpu": [ "ia32" ], @@ -714,9 +722,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.2.tgz", - "integrity": "sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", "cpu": [ "loong64" ], @@ -731,9 +739,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.2.tgz", - "integrity": "sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", "cpu": [ "mips64el" ], @@ -748,9 +756,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.2.tgz", - "integrity": "sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", "cpu": [ "ppc64" ], @@ -765,9 +773,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.2.tgz", - "integrity": "sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", "cpu": [ "riscv64" ], @@ -782,9 +790,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.2.tgz", - "integrity": "sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", "cpu": [ "s390x" ], @@ -799,9 +807,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.2.tgz", - "integrity": "sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", "cpu": [ "x64" ], @@ -816,9 +824,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.2.tgz", - "integrity": "sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", "cpu": [ "arm64" ], @@ -833,9 +841,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.2.tgz", - "integrity": "sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", "cpu": [ "x64" ], @@ -850,9 +858,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.2.tgz", - "integrity": "sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", "cpu": [ "arm64" ], @@ -867,9 +875,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.2.tgz", - "integrity": "sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", "cpu": [ "x64" ], @@ -884,9 +892,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.2.tgz", - "integrity": "sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", "cpu": [ "arm64" ], @@ -901,9 +909,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.2.tgz", - "integrity": "sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", "cpu": [ "x64" ], @@ -918,9 +926,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.2.tgz", - "integrity": "sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", "cpu": [ "arm64" ], @@ -935,9 +943,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.2.tgz", - "integrity": "sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", "cpu": [ "ia32" ], @@ -952,9 +960,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.2.tgz", - "integrity": "sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", "cpu": [ "x64" ], @@ -1011,15 +1019,15 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.1.tgz", - "integrity": "sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA==", + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", "dev": true, "license": "Apache-2.0", "dependencies": { "@eslint/object-schema": "^2.1.7", "debug": "^4.3.1", - "minimatch": "^3.1.2" + "minimatch": "^3.1.5" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1052,20 +1060,20 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.3.tgz", - "integrity": "sha512-Kr+LPIUVKz2qkx1HAMH8q1q6azbqBAsXJUxBl/ODDuVPX45Z9DfwB8tPjTi6nNZ8BuM3nbJxC5zCAg5elnBUTQ==", + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", "dev": true, "license": "MIT", "dependencies": { - "ajv": "^6.12.4", + "ajv": "^6.14.0", "debug": "^4.3.2", "espree": "^10.0.1", "globals": "^14.0.0", "ignore": "^5.2.0", "import-fresh": "^3.2.1", "js-yaml": "^4.1.1", - "minimatch": "^3.1.2", + "minimatch": "^3.1.5", "strip-json-comments": "^3.1.1" }, "engines": { @@ -1113,9 +1121,9 @@ } }, "node_modules/@exodus/bytes": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.10.0.tgz", - "integrity": "sha512-tf8YdcbirXdPnJ+Nd4UN1EXnz+IP2DI45YVEr3vvzcVTOyrApkmIB4zvOQVd3XPr7RXnfBtAx+PXImXOIU0Ajg==", + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", + "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", "dev": true, "license": "MIT", "engines": { @@ -1131,22 +1139,22 @@ } }, "node_modules/@floating-ui/core": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.4.tgz", - "integrity": "sha512-C3HlIdsBxszvm5McXlB8PeOEWfBhcGBTZGkGlWc2U0KFY5IwG5OQEuQ8rq52DZmcHDlPLd+YFBK+cZcytwIFWg==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", + "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==", "license": "MIT", "dependencies": { - "@floating-ui/utils": "^0.2.10" + "@floating-ui/utils": "^0.2.11" } }, "node_modules/@floating-ui/dom": { - "version": "1.7.5", - "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.5.tgz", - "integrity": "sha512-N0bD2kIPInNHUHehXhMke1rBGs1dwqvC9O9KYMyyjK7iXt7GAhnro7UlcuYcGdS/yYOlq0MAVgrow8IbWJwyqg==", + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", + "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==", "license": "MIT", "dependencies": { - "@floating-ui/core": "^1.7.4", - "@floating-ui/utils": "^0.2.10" + "@floating-ui/core": "^1.7.5", + "@floating-ui/utils": "^0.2.11" } }, "node_modules/@floating-ui/react": { @@ -1178,9 +1186,9 @@ } }, "node_modules/@floating-ui/utils": { - "version": "0.2.10", - "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz", - "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", + "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", "license": "MIT" }, "node_modules/@headlessui/react": { @@ -1218,12 +1226,12 @@ } }, "node_modules/@headlessui/react/node_modules/@floating-ui/react-dom": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.7.tgz", - "integrity": "sha512-0tLRojf/1Go2JgEVm+3Frg9A3IW8bJgKgdO0BN5RkF//ufuz2joZM63Npau2ff3J6lUVYgDSNzNkR+aH3IVfjg==", + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", + "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", "license": "MIT", "dependencies": { - "@floating-ui/dom": "^1.7.5" + "@floating-ui/dom": "^1.7.6" }, "peerDependencies": { "react": ">=16.8.0", @@ -1252,29 +1260,43 @@ } }, "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, "engines": { "node": ">=18.18.0" } }, "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@humanfs/core": "^0.19.1", + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", "@humanwhocodes/retry": "^0.4.0" }, "engines": { "node": ">=18.18.0" } }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, "node_modules/@humanwhocodes/module-importer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", @@ -1304,9 +1326,9 @@ } }, "node_modules/@img/colour": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", - "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", "optional": true, "engines": { @@ -1769,10 +1791,37 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@internationalized/date": { + "version": "3.12.1", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.12.1.tgz", + "integrity": "sha512-6IedsVWXyq4P9Tj+TxuU8WGWM70hYLl12nbYU8jkikVpa6WXapFazPUcHUMDMoWftIDE2ILDkFFte6W2nFCkRQ==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.0" + } + }, + "node_modules/@internationalized/number": { + "version": "3.6.6", + "resolved": "https://registry.npmjs.org/@internationalized/number/-/number-3.6.6.tgz", + "integrity": "sha512-iFgmQaXHE0vytNfpLZWOC2mEJCBRzcUxt53Xf/yCXG93lRvqas237i3r7X4RKMwO3txiyZD4mQjKAByFv6UGSQ==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.0" + } + }, + "node_modules/@internationalized/string": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/@internationalized/string/-/string-3.2.8.tgz", + "integrity": "sha512-NdbMQUSfXLYIQol5VyMtinm9pZDciiMfN7RtmSuSB78io1hqwJ0naYfxyW6vgxWBkzWymQa/3uLDlbfmshtCaA==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.0" + } + }, "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", "dev": true, "license": "MIT", "engines": { @@ -1815,16 +1864,22 @@ } }, "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" } }, "node_modules/@next/env": { @@ -2029,9 +2084,9 @@ } }, "node_modules/@oxc-resolver/binding-android-arm-eabi": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm-eabi/-/binding-android-arm-eabi-11.17.0.tgz", - "integrity": "sha512-kVnY21v0GyZ/+LG6EIO48wK3mE79BUuakHUYLIqobO/Qqq4mJsjuYXMSn3JtLcKZpN1HDVit4UHpGJHef1lrlw==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm-eabi/-/binding-android-arm-eabi-11.19.1.tgz", + "integrity": "sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg==", "cpu": [ "arm" ], @@ -2043,9 +2098,9 @@ ] }, "node_modules/@oxc-resolver/binding-android-arm64": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm64/-/binding-android-arm64-11.17.0.tgz", - "integrity": "sha512-Pf8e3XcsK9a8RHInoAtEcrwf2vp7V9bSturyUUYxw9syW6E7cGi7z9+6ADXxm+8KAevVfLA7pfBg8NXTvz/HOw==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm64/-/binding-android-arm64-11.19.1.tgz", + "integrity": "sha512-oolbkRX+m7Pq2LNjr/kKgYeC7bRDMVTWPgxBGMjSpZi/+UskVo4jsMU3MLheZV55jL6c3rNelPl4oD60ggYmqA==", "cpu": [ "arm64" ], @@ -2057,9 +2112,9 @@ ] }, "node_modules/@oxc-resolver/binding-darwin-arm64": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.17.0.tgz", - "integrity": "sha512-lVSgKt3biecofXVr8e1hnfX0IYMd4A6VCxmvOmHsFt5Zbmt0lkO4S2ap2bvQwYDYh5ghUNamC7M2L8K6vishhQ==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.19.1.tgz", + "integrity": "sha512-nUC6d2i3R5B12sUW4O646qD5cnMXf2oBGPLIIeaRfU9doJRORAbE2SGv4eW6rMqhD+G7nf2Y8TTJTLiiO3Q/dQ==", "cpu": [ "arm64" ], @@ -2071,9 +2126,9 @@ ] }, "node_modules/@oxc-resolver/binding-darwin-x64": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-x64/-/binding-darwin-x64-11.17.0.tgz", - "integrity": "sha512-+/raxVJE1bo7R4fA9Yp0wm3slaCOofTEeUzM01YqEGcRDLHB92WRGjRhagMG2wGlvqFuSiTp81DwSbBVo/g6AQ==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-x64/-/binding-darwin-x64-11.19.1.tgz", + "integrity": "sha512-cV50vE5+uAgNcFa3QY1JOeKDSkM/9ReIcc/9wn4TavhW/itkDGrXhw9jaKnkQnGbjJ198Yh5nbX/Gr2mr4Z5jQ==", "cpu": [ "x64" ], @@ -2085,9 +2140,9 @@ ] }, "node_modules/@oxc-resolver/binding-freebsd-x64": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-freebsd-x64/-/binding-freebsd-x64-11.17.0.tgz", - "integrity": "sha512-x9Ks56n+n8h0TLhzA6sJXa2tGh3uvMGpBppg6PWf8oF0s5S/3p/J6k1vJJ9lIUtTmenfCQEGKnFokpRP4fLTLg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-freebsd-x64/-/binding-freebsd-x64-11.19.1.tgz", + "integrity": "sha512-xZOQiYGFxtk48PBKff+Zwoym7ScPAIVp4c14lfLxizO2LTTTJe5sx9vQNGrBymrf/vatSPNMD4FgsaaRigPkqw==", "cpu": [ "x64" ], @@ -2099,9 +2154,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-arm-gnueabihf": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-11.17.0.tgz", - "integrity": "sha512-Wf3w07Ow9kXVJrS0zmsaFHKOGhXKXE8j1tNyy+qIYDsQWQ4UQZVx5SjlDTcqBnFerlp3Z3Is0RjmVzgoLG3qkA==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-11.19.1.tgz", + "integrity": "sha512-lXZYWAC6kaGe/ky2su94e9jN9t6M0/6c+GrSlCqL//XO1cxi5lpAhnJYdyrKfm0ZEr/c7RNyAx3P7FSBcBd5+A==", "cpu": [ "arm" ], @@ -2113,9 +2168,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-arm-musleabihf": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-11.17.0.tgz", - "integrity": "sha512-N0OKA1al1gQ5Gm7Fui1RWlXaHRNZlwMoBLn3TVtSXX+WbnlZoVyDqqOqFL8+pVEHhhxEA2LR8kmM0JO6FAk6dg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-11.19.1.tgz", + "integrity": "sha512-veG1kKsuK5+t2IsO9q0DErYVSw2azvCVvWHnfTOS73WE0STdLLB7Q1bB9WR+yHPQM76ASkFyRbogWo1GR1+WbQ==", "cpu": [ "arm" ], @@ -2127,9 +2182,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-arm64-gnu": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-11.17.0.tgz", - "integrity": "sha512-wdcQ7Niad9JpjZIGEeqKJnTvczVunqlZ/C06QzR5zOQNeLVRScQ9S5IesKWUAPsJQDizV+teQX53nTK+Z5Iy+g==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-11.19.1.tgz", + "integrity": "sha512-heV2+jmXyYnUrpUXSPugqWDRpnsQcDm2AX4wzTuvgdlZfoNYO0O3W2AVpJYaDn9AG4JdM6Kxom8+foE7/BcSig==", "cpu": [ "arm64" ], @@ -2141,9 +2196,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-arm64-musl": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-musl/-/binding-linux-arm64-musl-11.17.0.tgz", - "integrity": "sha512-65B2/t39HQN5AEhkLsC+9yBD1iRUkKOIhfmJEJ7g6wQ9kylra7JRmNmALFjbsj0VJsoSQkpM8K07kUZuNJ9Kxw==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-musl/-/binding-linux-arm64-musl-11.19.1.tgz", + "integrity": "sha512-jvo2Pjs1c9KPxMuMPIeQsgu0mOJF9rEb3y3TdpsrqwxRM+AN6/nDDwv45n5ZrUnQMsdBy5gIabioMKnQfWo9ew==", "cpu": [ "arm64" ], @@ -2155,9 +2210,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-ppc64-gnu": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-11.17.0.tgz", - "integrity": "sha512-kExgm3TLK21dNMmcH+xiYGbc6BUWvT03PUZ2aYn8mUzGPeeORklBhg3iYcaBI3ZQHB25412X1Z6LLYNjt4aIaA==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-11.19.1.tgz", + "integrity": "sha512-vLmdNxWCdN7Uo5suays6A/+ywBby2PWBBPXctWPg5V0+eVuzsJxgAn6MMB4mPlshskYbppjpN2Zg83ArHze9gQ==", "cpu": [ "ppc64" ], @@ -2169,9 +2224,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-riscv64-gnu": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-11.17.0.tgz", - "integrity": "sha512-1utUJC714/ydykZQE8c7QhpEyM4SaslMfRXxN9G61KYazr6ndt85LaubK3EZCSD50vVEfF4PVwFysCSO7LN9uA==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-11.19.1.tgz", + "integrity": "sha512-/b+WgR+VTSBxzgOhDO7TlMXC1ufPIMR6Vj1zN+/x+MnyXGW7prTLzU9eW85Aj7Th7CCEG9ArCbTeqxCzFWdg2w==", "cpu": [ "riscv64" ], @@ -2183,9 +2238,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-riscv64-musl": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-11.17.0.tgz", - "integrity": "sha512-mayiYOl3LMmtO2CLn4I5lhanfxEo0LAqlT/EQyFbu1ZN3RS+Xa7Q3JEM0wBpVIyfO/pqFrjvC5LXw/mHNDEL7A==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-11.19.1.tgz", + "integrity": "sha512-YlRdeWb9j42p29ROh+h4eg/OQ3dTJlpHSa+84pUM9+p6i3djtPz1q55yLJhgW9XfDch7FN1pQ/Vd6YP+xfRIuw==", "cpu": [ "riscv64" ], @@ -2197,9 +2252,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-s390x-gnu": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-11.17.0.tgz", - "integrity": "sha512-Ow/yI+CrUHxIIhn/Y1sP/xoRKbCC3x9O1giKr3G/pjMe+TCJ5ZmfqVWU61JWwh1naC8X5Xa7uyLnbzyYqPsHfg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-11.19.1.tgz", + "integrity": "sha512-EDpafVOQWF8/MJynsjOGFThcqhRHy417sRyLfQmeiamJ8qVhSKAn2Dn2VVKUGCjVB9C46VGjhNo7nOPUi1x6uA==", "cpu": [ "s390x" ], @@ -2211,9 +2266,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-x64-gnu": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-gnu/-/binding-linux-x64-gnu-11.17.0.tgz", - "integrity": "sha512-Z4J7XlPMQOLPANyu6y3B3V417Md4LKH5bV6bhqgaG99qLHmU5LV2k9ErV14fSqoRc/GU/qOpqMdotxiJqN/YWg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-gnu/-/binding-linux-x64-gnu-11.19.1.tgz", + "integrity": "sha512-NxjZe+rqWhr+RT8/Ik+5ptA3oz7tUw361Wa5RWQXKnfqwSSHdHyrw6IdcTfYuml9dM856AlKWZIUXDmA9kkiBQ==", "cpu": [ "x64" ], @@ -2225,9 +2280,9 @@ ] }, "node_modules/@oxc-resolver/binding-linux-x64-musl": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-musl/-/binding-linux-x64-musl-11.17.0.tgz", - "integrity": "sha512-0effK+8lhzXsgsh0Ny2ngdnTPF30v6QQzVFApJ1Ctk315YgpGkghkelvrLYYgtgeFJFrzwmOJ2nDvCrUFKsS2Q==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-musl/-/binding-linux-x64-musl-11.19.1.tgz", + "integrity": "sha512-cM/hQwsO3ReJg5kR+SpI69DMfvNCp+A/eVR4b4YClE5bVZwz8rh2Nh05InhwI5HR/9cArbEkzMjcKgTHS6UaNw==", "cpu": [ "x64" ], @@ -2239,9 +2294,9 @@ ] }, "node_modules/@oxc-resolver/binding-openharmony-arm64": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-openharmony-arm64/-/binding-openharmony-arm64-11.17.0.tgz", - "integrity": "sha512-kFB48dRUW6RovAICZaxHKdtZe+e94fSTNA2OedXokzMctoU54NPZcv0vUX5PMqyikLIKJBIlW7laQidnAzNrDA==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-openharmony-arm64/-/binding-openharmony-arm64-11.19.1.tgz", + "integrity": "sha512-QF080IowFB0+9Rh6RcD19bdgh49BpQHUW5TajG1qvWHvmrQznTZZjYlgE2ltLXyKY+qs4F/v5xuX1XS7Is+3qA==", "cpu": [ "arm64" ], @@ -2253,9 +2308,9 @@ ] }, "node_modules/@oxc-resolver/binding-wasm32-wasi": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-wasm32-wasi/-/binding-wasm32-wasi-11.17.0.tgz", - "integrity": "sha512-a3elKSBLPT0OoRPxTkCIIc+4xnOELolEBkPyvdj01a6PSdSmyJ1NExWjWLaXnT6wBMblvKde5RmSwEi3j+jZpg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-wasm32-wasi/-/binding-wasm32-wasi-11.19.1.tgz", + "integrity": "sha512-w8UCKhX826cP/ZLokXDS6+milN8y4X7zidsAttEdWlVoamTNf6lhBJldaWr3ukTDiye7s4HRcuPEPOXNC432Vg==", "cpu": [ "wasm32" ], @@ -2269,27 +2324,10 @@ "node": ">=14.0.0" } }, - "node_modules/@oxc-resolver/binding-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.1.tgz", - "integrity": "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1", - "@tybys/wasm-util": "^0.10.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - } - }, "node_modules/@oxc-resolver/binding-win32-arm64-msvc": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-11.17.0.tgz", - "integrity": "sha512-4eszUsSDb9YVx0RtYkPWkxxtSZIOgfeiX//nG5cwRRArg178w4RCqEF1kbKPud9HPrp1rXh7gE4x911OhvTnPg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-11.19.1.tgz", + "integrity": "sha512-nJ4AsUVZrVKwnU/QRdzPCCrO0TrabBqgJ8pJhXITdZGYOV28TIYystV1VFLbQ7DtAcaBHpocT5/ZJnF78YJPtQ==", "cpu": [ "arm64" ], @@ -2301,9 +2339,9 @@ ] }, "node_modules/@oxc-resolver/binding-win32-ia32-msvc": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-11.17.0.tgz", - "integrity": "sha512-t946xTXMmR7yGH0KAe9rB055/X4EPIu93JUvjchl2cizR5QbuwkUV7vLS2BS6x6sfvDoQb6rWYnV1HCci6tBSg==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-11.19.1.tgz", + "integrity": "sha512-EW+ND5q2Tl+a3pH81l1QbfgbF3HmqgwLfDfVithRFheac8OTcnbXt/JxqD2GbDkb7xYEqy1zNaVFRr3oeG8npA==", "cpu": [ "ia32" ], @@ -2315,9 +2353,9 @@ ] }, "node_modules/@oxc-resolver/binding-win32-x64-msvc": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-x64-msvc/-/binding-win32-x64-msvc-11.17.0.tgz", - "integrity": "sha512-pX6s2kMXLQg+hlqKk5UqOW09iLLxnTkvn8ohpYp2Mhsm2yzDPCx9dyOHiB/CQixLzTkLQgWWJykN4Z3UfRKW4Q==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-x64-msvc/-/binding-win32-x64-msvc-11.19.1.tgz", + "integrity": "sha512-6hIU3RQu45B+VNTY4Ru8ppFwjVS/S5qwYyGhBotmjxfEKk41I2DlGtRfGJndZ5+6lneE2pwloqunlOyZuX/XAw==", "cpu": [ "x64" ], @@ -2394,9 +2432,9 @@ } }, "node_modules/@rc-component/mini-decimal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.0.tgz", - "integrity": "sha512-jS4E7T9Li2GuYwI6PyiVXmxTiM6b07rlD9Ge8uGZSCz3WlzcG5ZK7g5bbuKNeZ9pgUuPK/5guV781ujdVpm4HQ==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rc-component/mini-decimal/-/mini-decimal-1.1.3.tgz", + "integrity": "sha512-bk/FJ09fLf+NLODMAFll6CfYrHPBioTedhW6lxDBuuWucJEqFUd4l/D/5JgIi3dina6sYahB8iuPAZTNz2pMxw==", "license": "MIT", "dependencies": { "@babel/runtime": "^7.18.0" @@ -2499,16 +2537,13 @@ } }, "node_modules/@react-aria/focus": { - "version": "3.21.3", - "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.21.3.tgz", - "integrity": "sha512-FsquWvjSCwC2/sBk4b+OqJyONETUIXQ2vM0YdPAuC+QFQh2DT6TIBo6dOZVSezlhudDla69xFBd6JvCFq1AbUw==", + "version": "3.22.0", + "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.22.0.tgz", + "integrity": "sha512-ZfDOVuVhqDsM9mkNji3QUZ/d40JhlVgXrDkrfXylM1035QCrcTHN7m2DpbE95sU2A8EQb4wikvt5jM6K/73BPg==", "license": "Apache-2.0", "dependencies": { - "@react-aria/interactions": "^3.26.0", - "@react-aria/utils": "^3.32.0", - "@react-types/shared": "^3.32.1", "@swc/helpers": "^0.5.0", - "clsx": "^2.0.0" + "react-aria": "3.48.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", @@ -2516,80 +2551,24 @@ } }, "node_modules/@react-aria/interactions": { - "version": "3.26.0", - "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.26.0.tgz", - "integrity": "sha512-AAEcHiltjfbmP1i9iaVw34Mb7kbkiHpYdqieWufldh4aplWgsF11YQZOfaCJW4QoR2ML4Zzoa9nfFwLXA52R7Q==", + "version": "3.28.0", + "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.28.0.tgz", + "integrity": "sha512-OXwdU1EWFdMxmr/K1CXNGJzmNlCClByb+PuCaqUyzBymHPCGVhawirLIon/CrIN5psh3AiWpHSh4H0WeJdVpng==", "license": "Apache-2.0", "dependencies": { - "@react-aria/ssr": "^3.9.10", - "@react-aria/utils": "^3.32.0", - "@react-stately/flags": "^3.1.2", - "@react-types/shared": "^3.32.1", - "@swc/helpers": "^0.5.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", - "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/ssr": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.10.tgz", - "integrity": "sha512-hvTm77Pf+pMBhuBm760Li0BVIO38jv1IBws1xFm1NoL26PU+fe+FMW5+VZWyANR6nYL65joaJKZqOdTQMkO9IQ==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - }, - "engines": { - "node": ">= 12" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, - "node_modules/@react-aria/utils": { - "version": "3.32.0", - "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.32.0.tgz", - "integrity": "sha512-/7Rud06+HVBIlTwmwmJa2W8xVtgxgzm0+kLbuFooZRzKDON6hhozS1dOMR/YLMxyJOaYOTpImcP4vRR9gL1hEg==", - "license": "Apache-2.0", - "dependencies": { - "@react-aria/ssr": "^3.9.10", - "@react-stately/flags": "^3.1.2", - "@react-stately/utils": "^3.11.0", - "@react-types/shared": "^3.32.1", + "@react-types/shared": "^3.34.0", "@swc/helpers": "^0.5.0", - "clsx": "^2.0.0" + "react-aria": "3.48.0" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" } }, - "node_modules/@react-stately/flags": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@react-stately/flags/-/flags-3.1.2.tgz", - "integrity": "sha512-2HjFcZx1MyQXoPqcBGALwWWmgFVUk2TuKVIQxCbRq7fPyWXIl6VHcakCLurdtYC2Iks7zizvz0Idv48MQ38DWg==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - } - }, - "node_modules/@react-stately/utils": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.11.0.tgz", - "integrity": "sha512-8LZpYowJ9eZmmYLpudbo/eclIRnbhWIJZ994ncmlKlouNzKohtM8qTC6B1w1pwUbiwGdUoyzLuQbeaIor5Dvcw==", - "license": "Apache-2.0", - "dependencies": { - "@swc/helpers": "^0.5.0" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" - } - }, "node_modules/@react-types/shared": { - "version": "3.32.1", - "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.32.1.tgz", - "integrity": "sha512-famxyD5emrGGpFuUlgOP6fVW2h/ZaF405G5KDi3zPHzyjAWys/8W6NAVJtNbkCkhedmvL0xOhvt8feGXyXaw5w==", + "version": "3.34.0", + "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.34.0.tgz", + "integrity": "sha512-gp6xo/s2lX54AlTjOiqwDnxA7UW79BNvI9dB9pr3LZTzRKCd1ZA+ZbgKw/ReIiWuvvVw/8QFJpnqeeFyLocMcQ==", "license": "Apache-2.0", "peerDependencies": { "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" @@ -2605,9 +2584,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.59.0.tgz", - "integrity": "sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.3.tgz", + "integrity": "sha512-x35CNW/ANXG3hE/EZpRU8MXX1JDN86hBb2wMGAtltkz7pc6cxgjpy1OMMfDosOQ+2hWqIkag/fGok1Yady9nGw==", "cpu": [ "arm" ], @@ -2619,9 +2598,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.59.0.tgz", - "integrity": "sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.3.tgz", + "integrity": "sha512-xw3xtkDApIOGayehp2+Rz4zimfkaX65r4t47iy+ymQB2G4iJCBBfj0ogVg5jpvjpn8UWn/+q9tprxleYeNp3Hw==", "cpu": [ "arm64" ], @@ -2633,9 +2612,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.59.0.tgz", - "integrity": "sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.3.tgz", + "integrity": "sha512-vo6Y5Qfpx7/5EaamIwi0WqW2+zfiusVihKatLvtN1VFVy3D13uERk/6gZLU1UiHRL6fDXqj/ELIeVRGnvcTE1g==", "cpu": [ "arm64" ], @@ -2647,9 +2626,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.59.0.tgz", - "integrity": "sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.3.tgz", + "integrity": "sha512-D+0QGcZhBzTN82weOnsSlY7V7+RMmPuF1CkbxyMAGE8+ZHeUjyb76ZiWmBlCu//AQQONvxcqRbwZTajZKqjuOw==", "cpu": [ "x64" ], @@ -2661,9 +2640,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.59.0.tgz", - "integrity": "sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.3.tgz", + "integrity": "sha512-6HnvHCT7fDyj6R0Ph7A6x8dQS/S38MClRWeDLqc0MdfWkxjiu1HSDYrdPhqSILzjTIC/pnXbbJbo+ft+gy/9hQ==", "cpu": [ "arm64" ], @@ -2675,9 +2654,9 @@ ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.59.0.tgz", - "integrity": "sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.3.tgz", + "integrity": "sha512-KHLgC3WKlUYW3ShFKnnosZDOJ0xjg9zp7au3sIm2bs/tGBeC2ipmvRh/N7JKi0t9Ue20C0dpEshi8WUubg+cnA==", "cpu": [ "x64" ], @@ -2689,9 +2668,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.59.0.tgz", - "integrity": "sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.3.tgz", + "integrity": "sha512-DV6fJoxEYWJOvaZIsok7KrYl0tPvga5OZ2yvKHNNYyk/2roMLqQAbGhr78EQ5YhHpnhLKJD3S1WFusAkmUuV5g==", "cpu": [ "arm" ], @@ -2703,9 +2682,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.59.0.tgz", - "integrity": "sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.3.tgz", + "integrity": "sha512-mQKoJAzvuOs6F+TZybQO4GOTSMUu7v0WdxEk24krQ/uUxXoPTtHjuaUuPmFhtBcM4K0ons8nrE3JyhTuCFtT/w==", "cpu": [ "arm" ], @@ -2717,9 +2696,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.59.0.tgz", - "integrity": "sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.3.tgz", + "integrity": "sha512-Whjj2qoiJ6+OOJMGptTYazaJvjOJm+iKHpXQM1P3LzGjt7Ff++Tp7nH4N8J/BUA7R9IHfDyx4DJIflifwnbmIA==", "cpu": [ "arm64" ], @@ -2731,9 +2710,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.59.0.tgz", - "integrity": "sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.3.tgz", + "integrity": "sha512-4YTNHKqGng5+yiZt3mg77nmyuCfmNfX4fPmyUapBcIk+BdwSwmCWGXOUxhXbBEkFHtoN5boLj/5NON+u5QC9tg==", "cpu": [ "arm64" ], @@ -2745,9 +2724,9 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.59.0.tgz", - "integrity": "sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.3.tgz", + "integrity": "sha512-SU3kNlhkpI4UqlUc2VXPGK9o886ZsSeGfMAX2ba2b8DKmMXq4AL7KUrkSWVbb7koVqx41Yczx6dx5PNargIrEA==", "cpu": [ "loong64" ], @@ -2759,9 +2738,9 @@ ] }, "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.59.0.tgz", - "integrity": "sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.3.tgz", + "integrity": "sha512-6lDLl5h4TXpB1mTf2rQWnAk/LcXrx9vBfu/DT5TIPhvMhRWaZ5MxkIc8u4lJAmBo6klTe1ywXIUHFjylW505sg==", "cpu": [ "loong64" ], @@ -2773,9 +2752,9 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.59.0.tgz", - "integrity": "sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.3.tgz", + "integrity": "sha512-BMo8bOw8evlup/8G+cj5xWtPyp93xPdyoSN16Zy90Q2QZ0ZYRhCt6ZJSwbrRzG9HApFabjwj2p25TUPDWrhzqQ==", "cpu": [ "ppc64" ], @@ -2787,9 +2766,9 @@ ] }, "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.59.0.tgz", - "integrity": "sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.3.tgz", + "integrity": "sha512-E0L8X1dZN1/Rph+5VPF6Xj2G7JJvMACVXtamTJIDrVI44Y3K+G8gQaMEAavbqCGTa16InptiVrX6eM6pmJ+7qA==", "cpu": [ "ppc64" ], @@ -2801,9 +2780,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.59.0.tgz", - "integrity": "sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.3.tgz", + "integrity": "sha512-oZJ/WHaVfHUiRAtmTAeo3DcevNsVvH8mbvodjZy7D5QKvCefO371SiKRpxoDcCxB3PTRTLayWBkvmDQKTcX/sw==", "cpu": [ "riscv64" ], @@ -2815,9 +2794,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.59.0.tgz", - "integrity": "sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.3.tgz", + "integrity": "sha512-Dhbyh7j9FybM3YaTgaHmVALwA8AkUwTPccyCQ79TG9AJUsMQqgN1DDEZNr4+QUfwiWvLDumW5vdwzoeUF+TNxQ==", "cpu": [ "riscv64" ], @@ -2829,9 +2808,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.59.0.tgz", - "integrity": "sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.3.tgz", + "integrity": "sha512-cJd1X5XhHHlltkaypz1UcWLA8AcoIi1aWhsvaWDskD1oz2eKCypnqvTQ8ykMNI0RSmm7NkTdSqSSD7zM0xa6Ig==", "cpu": [ "s390x" ], @@ -2843,9 +2822,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.59.0.tgz", - "integrity": "sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.3.tgz", + "integrity": "sha512-DAZDBHQfG2oQuhY7mc6I3/qB4LU2fQCjRvxbDwd/Jdvb9fypP4IJ4qmtu6lNjes6B531AI8cg1aKC2di97bUxA==", "cpu": [ "x64" ], @@ -2857,9 +2836,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.59.0.tgz", - "integrity": "sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.3.tgz", + "integrity": "sha512-cRxsE8c13mZOh3vP+wLDxpQBRrOHDIGOWyDL93Sy0Ga8y515fBcC2pjUfFwUe5T7tqvTvWbCpg1URM/AXdWIXA==", "cpu": [ "x64" ], @@ -2871,9 +2850,9 @@ ] }, "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.59.0.tgz", - "integrity": "sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.3.tgz", + "integrity": "sha512-QaWcIgRxqEdQdhJqW4DJctsH6HCmo5vHxY0krHSX4jMtOqfzC+dqDGuHM87bu4H8JBeibWx7jFz+h6/4C8wA5Q==", "cpu": [ "x64" ], @@ -2885,9 +2864,9 @@ ] }, "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.59.0.tgz", - "integrity": "sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.3.tgz", + "integrity": "sha512-AaXwSvUi3QIPtroAUw1t5yHGIyqKEXwH54WUocFolZhpGDruJcs8c+xPNDRn4XiQsS7MEwnYsHW2l0MBLDMkWg==", "cpu": [ "arm64" ], @@ -2899,9 +2878,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.59.0.tgz", - "integrity": "sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.3.tgz", + "integrity": "sha512-65LAKM/bAWDqKNEelHlcHvm2V+Vfb8C6INFxQXRHCvaVN1rJfwr4NvdP4FyzUaLqWfaCGaadf6UbTm8xJeYfEg==", "cpu": [ "arm64" ], @@ -2913,9 +2892,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.59.0.tgz", - "integrity": "sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.3.tgz", + "integrity": "sha512-EEM2gyhBF5MFnI6vMKdX1LAosE627RGBzIoGMdLloPZkXrUN0Ckqgr2Qi8+J3zip/8NVVro3/FjB+tjhZUgUHA==", "cpu": [ "ia32" ], @@ -2927,9 +2906,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.59.0.tgz", - "integrity": "sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.3.tgz", + "integrity": "sha512-E5Eb5H/DpxaoXH++Qkv28RcUJboMopmdDUALBczvHMf7hNIxaDZqwY5lK12UK1BHacSmvupoEWGu+n993Z0y1A==", "cpu": [ "x64" ], @@ -2941,9 +2920,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.59.0.tgz", - "integrity": "sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.3.tgz", + "integrity": "sha512-hPt/bgL5cE+Qp+/TPHBqptcAgPzgj46mPcg/16zNUmbQk0j+mOEQV/+Lqu8QRtDV3Ek95Q6FeFITpuhl6OTsAA==", "cpu": [ "x64" ], @@ -2962,16 +2941,16 @@ "license": "MIT" }, "node_modules/@rushstack/eslint-patch": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.15.0.tgz", - "integrity": "sha512-ojSshQPKwVvSMR8yT2L/QtUkV5SXi/IfDiJ4/8d6UbTPjiHVmxZzUAzGD8Tzks1b9+qQkZa0isUOvYObedITaw==", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.16.1.tgz", + "integrity": "sha512-TvZbIpeKqGQQ7X0zSCvPH9riMSFQFSggnfBjFZ1mEoILW+UuXCKwOoPcgjMwiUtRqFZ8jWhPJc4um14vC6I4ag==", "dev": true, "license": "MIT" }, "node_modules/@swc/helpers": { - "version": "0.5.18", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.18.tgz", - "integrity": "sha512-TXTnIcNJQEKwThMMqBXsZ4VGAza6bvN4pa41Rkqoio6QBKMvo+5lexeTMScGCIxtzgQJzElcvIltani+adC5PQ==", + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.21.tgz", + "integrity": "sha512-jI/VAmtdjB/RnI8GTnokyX7Ug8c+g+ffD6QRLa6XQewtnGyukKkKSk3wLTM3b5cjt1jNh9x0jfVlagdN2gDKQg==", "license": "Apache-2.0", "dependencies": { "tslib": "^2.8.0" @@ -3070,12 +3049,12 @@ } }, "node_modules/@tanstack/react-virtual": { - "version": "3.13.18", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.18.tgz", - "integrity": "sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A==", + "version": "3.13.24", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.24.tgz", + "integrity": "sha512-aIJvz5OSkhNIhZIpYivrxrPTKYsjW9Uzy+sP/mx0S3sev2HyvPb7xmjbYvokzEpfgYHy/HjzJ2zFAETuUfgCpg==", "license": "MIT", "dependencies": { - "@tanstack/virtual-core": "3.13.18" + "@tanstack/virtual-core": "3.14.0" }, "funding": { "type": "github", @@ -3100,9 +3079,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.13.18", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.13.18.tgz", - "integrity": "sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.14.0.tgz", + "integrity": "sha512-JLANqGy/D6k4Ujmh8Tr25lGimuOXNiaVyXaCAZS0W+1390sADdGnyUdSWNIfd49gebtIxGMij4IktRVzrdr12Q==", "license": "MIT", "funding": { "type": "github", @@ -3228,9 +3207,9 @@ } }, "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", "dev": true, "license": "MIT", "optional": true, @@ -3330,9 +3309,9 @@ "license": "MIT" }, "node_modules/@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", "license": "MIT", "dependencies": { "@types/ms": "*" @@ -3500,20 +3479,20 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.54.0.tgz", - "integrity": "sha512-hAAP5io/7csFStuOmR782YmTthKBJ9ND3WVL60hcOjvtGFb+HJxH4O5huAcmcZ9v9G8P+JETiZ/G1B8MALnWZQ==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.2.tgz", + "integrity": "sha512-j/bwmkBvHUtPNxzuWe5z6BEk3q54YRyGlBXkSsmfoih7zNrBvl5A9A98anlp/7JbyZcWIJ8KXo/3Tq/DjFLtuQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.54.0", - "@typescript-eslint/type-utils": "8.54.0", - "@typescript-eslint/utils": "8.54.0", - "@typescript-eslint/visitor-keys": "8.54.0", + "@typescript-eslint/scope-manager": "8.59.2", + "@typescript-eslint/type-utils": "8.59.2", + "@typescript-eslint/utils": "8.59.2", + "@typescript-eslint/visitor-keys": "8.59.2", "ignore": "^7.0.5", "natural-compare": "^1.4.0", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3523,9 +3502,9 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.54.0", - "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <6.0.0" + "@typescript-eslint/parser": "^8.59.2", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { @@ -3539,16 +3518,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.54.0.tgz", - "integrity": "sha512-BtE0k6cjwjLZoZixN0t5AKP0kSzlGu7FctRXYuPAm//aaiZhmfq1JwdYpYr1brzEspYyFeF+8XF5j2VK6oalrA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.2.tgz", + "integrity": "sha512-plR3pp6D+SSUn1HM7xvSkx12/DhoHInI2YF35KAcVFNZvlC0gtrWqx7Qq1oH2Ssgi0vlFRCTbP+DZc7B9+TtsQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.54.0", - "@typescript-eslint/types": "8.54.0", - "@typescript-eslint/typescript-estree": "8.54.0", - "@typescript-eslint/visitor-keys": "8.54.0", + "@typescript-eslint/scope-manager": "8.59.2", + "@typescript-eslint/types": "8.59.2", + "@typescript-eslint/typescript-estree": "8.59.2", + "@typescript-eslint/visitor-keys": "8.59.2", "debug": "^4.4.3" }, "engines": { @@ -3559,19 +3538,19 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <6.0.0" + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.54.0.tgz", - "integrity": "sha512-YPf+rvJ1s7MyiWM4uTRhE4DvBXrEV+d8oC3P9Y2eT7S+HBS0clybdMIPnhiATi9vZOYDc7OQ1L/i6ga6NFYK/g==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.2.tgz", + "integrity": "sha512-+2hqvEkeyf/0FBor67duF0Ll7Ot8jyKzDQOSrxazF/danillRq2DwR9dLptsXpoZQqxE1UisSmoZewrlPas9Vw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.54.0", - "@typescript-eslint/types": "^8.54.0", + "@typescript-eslint/tsconfig-utils": "^8.59.2", + "@typescript-eslint/types": "^8.59.2", "debug": "^4.4.3" }, "engines": { @@ -3582,18 +3561,18 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.54.0.tgz", - "integrity": "sha512-27rYVQku26j/PbHYcVfRPonmOlVI6gihHtXFbTdB5sb6qA0wdAQAbyXFVarQ5t4HRojIz64IV90YtsjQSSGlQg==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.2.tgz", + "integrity": "sha512-JzfyEpEtOU89CcFSwyNS3mu4MLvLSXqnmX05+aKBDM+TdR5jzcGOEBwxwGNxrEQ7p/z6kK2WyioCGBf2zZBnvg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.54.0", - "@typescript-eslint/visitor-keys": "8.54.0" + "@typescript-eslint/types": "8.59.2", + "@typescript-eslint/visitor-keys": "8.59.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3604,9 +3583,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.54.0.tgz", - "integrity": "sha512-dRgOyT2hPk/JwxNMZDsIXDgyl9axdJI3ogZ2XWhBPsnZUv+hPesa5iuhdYt2gzwA9t8RE5ytOJ6xB0moV0Ujvw==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.2.tgz", + "integrity": "sha512-BKK4alN7oi4C/zv4VqHQ+uRU+lTa6JGIZ7s1juw7b3RHo9OfKB+bKX3u0iVZetdsUCBBkSbdWbarJbmN0fTeSw==", "dev": true, "license": "MIT", "engines": { @@ -3617,21 +3596,21 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.54.0.tgz", - "integrity": "sha512-hiLguxJWHjjwL6xMBwD903ciAwd7DmK30Y9Axs/etOkftC3ZNN9K44IuRD/EB08amu+Zw6W37x9RecLkOo3pMA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.2.tgz", + "integrity": "sha512-nhqaj1nmTdVVl/BP5omXNRGO38jn5iosis2vbdmupF2txCf8ylWT8lx+JlvMYYVqzGVKtjojUFoQ3JRWK+mfzQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.54.0", - "@typescript-eslint/typescript-estree": "8.54.0", - "@typescript-eslint/utils": "8.54.0", + "@typescript-eslint/types": "8.59.2", + "@typescript-eslint/typescript-estree": "8.59.2", + "@typescript-eslint/utils": "8.59.2", "debug": "^4.4.3", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3641,14 +3620,14 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <6.0.0" + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.54.0.tgz", - "integrity": "sha512-PDUI9R1BVjqu7AUDsRBbKMtwmjWcn4J3le+5LpcFgWULN3LvHC5rkc9gCVxbrsrGmO1jfPybN5s6h4Jy+OnkAA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.2.tgz", + "integrity": "sha512-e82GVOE8Ps3E++Egvb6Y3Dw0S10u8NkQ9KXmtRhCWJJ8kDhOJTvtMAWnFL16kB1583goCWXsr0NieKCZMs2/0Q==", "dev": true, "license": "MIT", "engines": { @@ -3660,21 +3639,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.54.0.tgz", - "integrity": "sha512-BUwcskRaPvTk6fzVWgDPdUndLjB87KYDrN5EYGetnktoeAvPtO4ONHlAZDnj5VFnUANg0Sjm7j4usBlnoVMHwA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.2.tgz", + "integrity": "sha512-o0XPGNwcWw+FIwStOWn+BwBuEmL6QXP0rsvAFg7ET1dey1Nr6Wb1ac8p5HEsK0ygO/6mUxlk+YWQD9xcb/nnXg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.54.0", - "@typescript-eslint/tsconfig-utils": "8.54.0", - "@typescript-eslint/types": "8.54.0", - "@typescript-eslint/visitor-keys": "8.54.0", + "@typescript-eslint/project-service": "8.59.2", + "@typescript-eslint/tsconfig-utils": "8.59.2", + "@typescript-eslint/types": "8.59.2", + "@typescript-eslint/visitor-keys": "8.59.2", "debug": "^4.4.3", - "minimatch": "^9.0.5", + "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.4.0" + "ts-api-utils": "^2.5.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3684,20 +3663,20 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "typescript": ">=4.8.4 <6.0.0" + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/utils": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.54.0.tgz", - "integrity": "sha512-9Cnda8GS57AQakvRyG0PTejJNlA2xhvyNtEVIMlDWOOeEyBkYWhGPnfrIAnqxLMTSTo6q8g12XVjjev5l1NvMA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.2.tgz", + "integrity": "sha512-Juw3EinkXqjaffxz6roowvV7GZT/kET5vSKKZT6upl5TXdWkLkYmNPXwDDL2Vkt2DPn0nODIS4egC/0AGxKo/Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.54.0", - "@typescript-eslint/types": "8.54.0", - "@typescript-eslint/typescript-estree": "8.54.0" + "@typescript-eslint/scope-manager": "8.59.2", + "@typescript-eslint/types": "8.59.2", + "@typescript-eslint/typescript-estree": "8.59.2" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3707,19 +3686,19 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", - "typescript": ">=4.8.4 <6.0.0" + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.54.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.54.0.tgz", - "integrity": "sha512-VFlhGSl4opC0bprJiItPQ1RfUhGDIBokcPwaFH4yiBCaNPeld/9VeXbiPO1cLyorQi1G1vL+ecBk1x8o1axORA==", + "version": "8.59.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.2.tgz", + "integrity": "sha512-NwjLUnGy8/Zfx23fl50tRC8rYaYnM52xNRYFAXvmiil9yh1+K6aRVQMnzW6gQB/1DLgWt977lYQn7C+wtgXZiA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.54.0", - "eslint-visitor-keys": "^4.2.1" + "@typescript-eslint/types": "8.59.2", + "eslint-visitor-keys": "^5.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3729,6 +3708,19 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -3962,6 +3954,19 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", + "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.10.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.11.1.tgz", @@ -4188,9 +4193,9 @@ } }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", "bin": { @@ -4233,9 +4238,9 @@ } }, "node_modules/ajv": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", - "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", "dev": true, "license": "MIT", "dependencies": { @@ -4584,9 +4589,9 @@ "license": "MIT" }, "node_modules/ast-v8-to-istanbul": { - "version": "0.3.11", - "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.11.tgz", - "integrity": "sha512-Qya9fkoofMjCBNVdWINMjB5KZvkYfaO9/anwkWnjxibpWUxo5iHl2sOdP7/uAqaRuUYuoo8rDwnbaaKVFxoUvw==", + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.12.tgz", + "integrity": "sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==", "dev": true, "license": "MIT", "dependencies": { @@ -4672,9 +4677,9 @@ } }, "node_modules/axe-core": { - "version": "4.11.1", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.11.1.tgz", - "integrity": "sha512-BASOg+YwO2C+346x3LZOeoovTIoTrRqEsqMa6fmfAV0P+U9mFr9NsyOEpiYvFjbc64NMrSswhV50WdXzdb/Z5A==", + "version": "4.11.4", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.11.4.tgz", + "integrity": "sha512-KunSNx+TVpkAw/6ULfhnx+HWRecjqZGTOyquAoWHYLRSdK1tB5Ihce1ZW+UY3fj33bYAFWPu7W/GRSmmrCGuxA==", "dev": true, "license": "MPL-2.0", "engines": { @@ -4712,12 +4717,15 @@ } }, "node_modules/baseline-browser-mapping": { - "version": "2.9.19", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", - "integrity": "sha512-ipDqC8FrAl/76p2SSWKSI+H9tFwm7vYqXQrItCuiVPt26Km0jS+NzSsBWAaBusvSbQcfJG+JitdMm+wZAgTYqg==", + "version": "2.10.27", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.27.tgz", + "integrity": "sha512-zEs/ufmZoUd7WftKpKyXaT6RFxpQ5Qm9xytKRHvJfxFV9DFJkZph9RvJ1LcOUi0Z1ZVijMte65JbILeV+8QQEA==", "license": "Apache-2.0", "bin": { - "baseline-browser-mapping": "dist/cli.js" + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" } }, "node_modules/bidi-js": { @@ -4768,9 +4776,9 @@ } }, "node_modules/browserslist": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", - "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "version": "4.28.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", "dev": true, "funding": [ { @@ -4788,11 +4796,11 @@ ], "license": "MIT", "dependencies": { - "baseline-browser-mapping": "^2.9.0", - "caniuse-lite": "^1.0.30001759", - "electron-to-chromium": "^1.5.263", - "node-releases": "^2.0.27", - "update-browserslist-db": "^1.2.0" + "baseline-browser-mapping": "^2.10.12", + "caniuse-lite": "^1.0.30001782", + "electron-to-chromium": "^1.5.328", + "node-releases": "^2.0.36", + "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" @@ -4812,15 +4820,15 @@ } }, "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", "dev": true, "license": "MIT", "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", "set-function-length": "^1.2.2" }, "engines": { @@ -4880,9 +4888,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001766", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001766.tgz", - "integrity": "sha512-4C0lfJ0/YPjJQHagaE9x2Elb69CIqEPZeG0anQt9SIvIoOH4a4uaRl73IavyO+0qZh6MDLH//DrXThEYKHkmYA==", + "version": "1.0.30001791", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001791.tgz", + "integrity": "sha512-yk0l/YSrOnFZk3UROpDLQD9+kC1l4meK/wed583AXrzoarMGJcbRi2Q4RaUYbKxYAsZ8sWmaSa/DsLmdBeI1vQ==", "funding": [ { "type": "opencollective", @@ -5132,14 +5140,14 @@ } }, "node_modules/css-tree": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.1.0.tgz", - "integrity": "sha512-0eW44TGN5SQXU1mWSkKwFstI/22X2bG1nYzZTYMAWjylYURhse752YgbE4Cx46AC+bAvI+/dYTPRk1LqSUnu6w==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", + "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", "dev": true, "license": "MIT", "dependencies": { - "mdn-data": "2.12.2", - "source-map-js": "^1.0.1" + "mdn-data": "2.27.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" @@ -5635,9 +5643,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.283", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.283.tgz", - "integrity": "sha512-3vifjt1HgrGW/h76UEeny+adYApveS9dH2h3p57JYzBSXJIKUJAvtmIytDKjcSCt9xHfrNCFJ7gts6vkhuq++w==", + "version": "1.5.349", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.349.tgz", + "integrity": "sha512-QsWVGyRuY07Aqb234QytTfwd5d9AJlfNIQ5wIOl1L+PZDzI9d9+Fn0FRale/QYlFxt/bUnB0/nLd1jFPGxGK1A==", "dev": true, "license": "ISC" }, @@ -5649,22 +5657,22 @@ "license": "MIT" }, "node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", "dev": true, "license": "BSD-2-Clause", "engines": { - "node": ">=0.12" + "node": ">=20.19.0" }, "funding": { "url": "https://github.com/fb55/entities?sponsor=1" } }, "node_modules/es-abstract": { - "version": "1.24.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.24.1.tgz", - "integrity": "sha512-zHXBLhP+QehSSbsS9Pt23Gg964240DPd6QCf8WpkqEXxQ7fhdZzYsocOr5u7apWonsS5EjZDmTF+/slGMyasvw==", + "version": "1.24.2", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.24.2.tgz", + "integrity": "sha512-2FpH9Q5i2RRwyEP1AylXe6nYLR5OhaJTZwmlcP0dL/+JCbgg7yyEo/sEK6HeGZRf3dFpWwThaRHVApXSkW3xeg==", "dev": true, "license": "MIT", "dependencies": { @@ -5749,16 +5757,16 @@ } }, "node_modules/es-iterator-helpers": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.2.2.tgz", - "integrity": "sha512-BrUQ0cPTB/IwXj23HtwHjS9n7O4h9FX94b4xc5zlTHxeLgTAdzYUDyy6KdExAl9lbN5rtfe44xpjpmj9grxs5w==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.3.2.tgz", + "integrity": "sha512-HVLACW1TppGYjJ8H6/jqH/pqOtKRw6wMlrB23xfExmFWxFquAIWCmwoLsOyN96K4a5KbmOf5At9ZUO3GZbetAw==", "dev": true, "license": "MIT", "dependencies": { - "call-bind": "^1.0.8", + "call-bind": "^1.0.9", "call-bound": "^1.0.4", "define-properties": "^1.2.1", - "es-abstract": "^1.24.1", + "es-abstract": "^1.24.2", "es-errors": "^1.3.0", "es-set-tostringtag": "^2.1.0", "function-bind": "^1.1.2", @@ -5770,7 +5778,7 @@ "has-symbols": "^1.1.0", "internal-slot": "^1.1.0", "iterator.prototype": "^1.1.5", - "safe-array-concat": "^1.1.3" + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -5842,9 +5850,9 @@ } }, "node_modules/esbuild": { - "version": "0.27.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", - "integrity": "sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==", + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -5855,32 +5863,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.2", - "@esbuild/android-arm": "0.27.2", - "@esbuild/android-arm64": "0.27.2", - "@esbuild/android-x64": "0.27.2", - "@esbuild/darwin-arm64": "0.27.2", - "@esbuild/darwin-x64": "0.27.2", - "@esbuild/freebsd-arm64": "0.27.2", - "@esbuild/freebsd-x64": "0.27.2", - "@esbuild/linux-arm": "0.27.2", - "@esbuild/linux-arm64": "0.27.2", - "@esbuild/linux-ia32": "0.27.2", - "@esbuild/linux-loong64": "0.27.2", - "@esbuild/linux-mips64el": "0.27.2", - "@esbuild/linux-ppc64": "0.27.2", - "@esbuild/linux-riscv64": "0.27.2", - "@esbuild/linux-s390x": "0.27.2", - "@esbuild/linux-x64": "0.27.2", - "@esbuild/netbsd-arm64": "0.27.2", - "@esbuild/netbsd-x64": "0.27.2", - "@esbuild/openbsd-arm64": "0.27.2", - "@esbuild/openbsd-x64": "0.27.2", - "@esbuild/openharmony-arm64": "0.27.2", - "@esbuild/sunos-x64": "0.27.2", - "@esbuild/win32-arm64": "0.27.2", - "@esbuild/win32-ia32": "0.27.2", - "@esbuild/win32-x64": "0.27.2" + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" } }, "node_modules/escalade": { @@ -6011,15 +6019,15 @@ } }, "node_modules/eslint-import-resolver-node": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", - "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==", + "version": "0.3.10", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.10.tgz", + "integrity": "sha512-tRrKqFyCaKict5hOd244sL6EQFNycnMQnBe+j8uqGNXYzsImGbGUU4ibtoaBmv5FLwJwcFJNeg1GeVjQfbMrDQ==", "dev": true, "license": "MIT", "dependencies": { "debug": "^3.2.7", - "is-core-module": "^2.13.0", - "resolve": "^1.22.4" + "is-core-module": "^2.16.1", + "resolve": "^2.0.0-next.6" } }, "node_modules/eslint-import-resolver-node/node_modules/debug": { @@ -6235,24 +6243,6 @@ "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" } }, - "node_modules/eslint-plugin-react/node_modules/resolve": { - "version": "2.0.0-next.5", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz", - "integrity": "sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-core-module": "^2.13.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/eslint-plugin-react/node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -6812,9 +6802,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.1.tgz", - "integrity": "sha512-EoY1N2xCn44xU6750Sx7OjOIT59FkmstNc3X6y5xpz7D5cBtZRe/3pSlTkDJgqsOk3WwZPkWfonhhUJfttQo3w==", + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", "dev": true, "license": "MIT", "dependencies": { @@ -6976,9 +6966,9 @@ } }, "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz", + "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==", "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7392,12 +7382,12 @@ } }, "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", + "integrity": "sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==", "license": "MIT", "dependencies": { - "hasown": "^2.0.2" + "hasown": "^2.0.3" }, "engines": { "node": ">= 0.4" @@ -7821,12 +7811,13 @@ } }, "node_modules/jiti": { - "version": "1.21.7", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", - "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", + "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", + "devOptional": true, "license": "MIT", "bin": { - "jiti": "bin/jiti.js" + "jiti": "lib/jiti-cli.mjs" } }, "node_modules/js-tokens": { @@ -8051,16 +8042,6 @@ "node": ">= 6" } }, - "node_modules/knip/node_modules/jiti": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.6.1.tgz", - "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", - "dev": true, - "license": "MIT", - "bin": { - "jiti": "lib/jiti-cli.mjs" - } - }, "node_modules/knip/node_modules/strip-json-comments": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz", @@ -8075,9 +8056,9 @@ } }, "node_modules/knip/node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "dev": true, "license": "MIT", "funding": { @@ -8209,9 +8190,9 @@ } }, "node_modules/lru-cache": { - "version": "11.2.5", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.5.tgz", - "integrity": "sha512-vFrFJkWtJvJnD5hg+hJvVE8Lh/TcMzKnTgCWmtBipwI5yLX/iX+5UB2tfuyODF5E7k9xEzMdYgGqaSb1c0c5Yw==", + "version": "11.3.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.6.tgz", + "integrity": "sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==", "dev": true, "license": "BlueOak-1.0.0", "engines": { @@ -8323,9 +8304,9 @@ } }, "node_modules/mdast-util-from-markdown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz", - "integrity": "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.3.tgz", + "integrity": "sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==", "license": "MIT", "dependencies": { "@types/mdast": "^4.0.0", @@ -8577,9 +8558,9 @@ } }, "node_modules/mdn-data": { - "version": "2.12.2", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.12.2.tgz", - "integrity": "sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==", + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", + "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", "dev": true, "license": "CC0-1.0" }, @@ -9248,11 +9229,11 @@ } }, "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" } @@ -9294,9 +9275,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", "funding": [ { "type": "github", @@ -9416,6 +9397,35 @@ "node": ">=10.5.0" } }, + "node_modules/node-exports-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.0.tgz", + "integrity": "sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "array.prototype.flatmap": "^1.3.3", + "es-errors": "^1.3.0", + "object.entries": "^1.1.9", + "semver": "^6.3.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/node-exports-info/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -9459,9 +9469,9 @@ } }, "node_modules/node-releases": { - "version": "2.0.27", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", - "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "version": "2.0.38", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.38.tgz", + "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", "dev": true, "license": "MIT" }, @@ -9687,35 +9697,35 @@ } }, "node_modules/oxc-resolver": { - "version": "11.17.0", - "resolved": "https://registry.npmjs.org/oxc-resolver/-/oxc-resolver-11.17.0.tgz", - "integrity": "sha512-R5P2Tw6th+nQJdNcZGfuppBS/sM0x1EukqYffmlfX2xXLgLGCCPwu4ruEr9Sx29mrpkHgITc130Qps2JR90NdQ==", + "version": "11.19.1", + "resolved": "https://registry.npmjs.org/oxc-resolver/-/oxc-resolver-11.19.1.tgz", + "integrity": "sha512-qE/CIg/spwrTBFt5aKmwe3ifeDdLfA2NESN30E42X/lII5ClF8V7Wt6WIJhcGZjp0/Q+nQ+9vgxGk//xZNX2hg==", "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/Boshen" }, "optionalDependencies": { - "@oxc-resolver/binding-android-arm-eabi": "11.17.0", - "@oxc-resolver/binding-android-arm64": "11.17.0", - "@oxc-resolver/binding-darwin-arm64": "11.17.0", - "@oxc-resolver/binding-darwin-x64": "11.17.0", - "@oxc-resolver/binding-freebsd-x64": "11.17.0", - "@oxc-resolver/binding-linux-arm-gnueabihf": "11.17.0", - "@oxc-resolver/binding-linux-arm-musleabihf": "11.17.0", - "@oxc-resolver/binding-linux-arm64-gnu": "11.17.0", - "@oxc-resolver/binding-linux-arm64-musl": "11.17.0", - "@oxc-resolver/binding-linux-ppc64-gnu": "11.17.0", - "@oxc-resolver/binding-linux-riscv64-gnu": "11.17.0", - "@oxc-resolver/binding-linux-riscv64-musl": "11.17.0", - "@oxc-resolver/binding-linux-s390x-gnu": "11.17.0", - "@oxc-resolver/binding-linux-x64-gnu": "11.17.0", - "@oxc-resolver/binding-linux-x64-musl": "11.17.0", - "@oxc-resolver/binding-openharmony-arm64": "11.17.0", - "@oxc-resolver/binding-wasm32-wasi": "11.17.0", - "@oxc-resolver/binding-win32-arm64-msvc": "11.17.0", - "@oxc-resolver/binding-win32-ia32-msvc": "11.17.0", - "@oxc-resolver/binding-win32-x64-msvc": "11.17.0" + "@oxc-resolver/binding-android-arm-eabi": "11.19.1", + "@oxc-resolver/binding-android-arm64": "11.19.1", + "@oxc-resolver/binding-darwin-arm64": "11.19.1", + "@oxc-resolver/binding-darwin-x64": "11.19.1", + "@oxc-resolver/binding-freebsd-x64": "11.19.1", + "@oxc-resolver/binding-linux-arm-gnueabihf": "11.19.1", + "@oxc-resolver/binding-linux-arm-musleabihf": "11.19.1", + "@oxc-resolver/binding-linux-arm64-gnu": "11.19.1", + "@oxc-resolver/binding-linux-arm64-musl": "11.19.1", + "@oxc-resolver/binding-linux-ppc64-gnu": "11.19.1", + "@oxc-resolver/binding-linux-riscv64-gnu": "11.19.1", + "@oxc-resolver/binding-linux-riscv64-musl": "11.19.1", + "@oxc-resolver/binding-linux-s390x-gnu": "11.19.1", + "@oxc-resolver/binding-linux-x64-gnu": "11.19.1", + "@oxc-resolver/binding-linux-x64-musl": "11.19.1", + "@oxc-resolver/binding-openharmony-arm64": "11.19.1", + "@oxc-resolver/binding-wasm32-wasi": "11.19.1", + "@oxc-resolver/binding-win32-arm64-msvc": "11.19.1", + "@oxc-resolver/binding-win32-ia32-msvc": "11.19.1", + "@oxc-resolver/binding-win32-x64-msvc": "11.19.1" } }, "node_modules/p-limit": { @@ -9795,13 +9805,13 @@ "license": "MIT" }, "node_modules/parse5": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.0.tgz", - "integrity": "sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==", + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", "dev": true, "license": "MIT", "dependencies": { - "entities": "^6.0.0" + "entities": "^8.0.0" }, "funding": { "url": "https://github.com/inikulin/parse5?sponsor=1" @@ -9834,9 +9844,9 @@ "license": "MIT" }, "node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { @@ -9844,7 +9854,7 @@ "minipass": "^7.1.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -9990,6 +10000,27 @@ "postcss": "^8.0.0" } }, + "node_modules/postcss-import/node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/postcss-js": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.1.0.tgz", @@ -10845,6 +10876,27 @@ "node": ">=0.10.0" } }, + "node_modules/react-aria": { + "version": "3.48.0", + "resolved": "https://registry.npmjs.org/react-aria/-/react-aria-3.48.0.tgz", + "integrity": "sha512-jQjd4rBEIMqecBaAKYJbVGK6EqIHLa5znVQ7jwFyK5vCyljoj6KhgtiahmcIPsG5vG5vEDLw+ba+bEWn6A2P4w==", + "license": "Apache-2.0", + "dependencies": { + "@internationalized/date": "^3.12.1", + "@internationalized/number": "^3.6.6", + "@internationalized/string": "^3.2.8", + "@react-types/shared": "^3.34.0", + "@swc/helpers": "^0.5.0", + "aria-hidden": "^1.2.3", + "clsx": "^2.0.0", + "react-stately": "3.46.0", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1", + "react-dom": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" + } + }, "node_modules/react-copy-to-clipboard": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/react-copy-to-clipboard/-/react-copy-to-clipboard-5.1.1.tgz", @@ -10859,9 +10911,9 @@ } }, "node_modules/react-day-picker": { - "version": "8.10.1", - "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.1.tgz", - "integrity": "sha512-TMx7fNbhLk15eqcMt+7Z7S2KF7mfTId/XJDjKE8f+IUcFn0l08/kI4FiYTL/0yuOLmEcbR4Fwe3GJf/NiiMnPA==", + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.2.tgz", + "integrity": "sha512-LK68OTbHB3oJNhl9cA0qVizzp3o26w61YSjAFkYi67N86iro32wx86kSNeFU/hq+gI8m1yzWhnomMLfZ041RzQ==", "license": "MIT", "funding": { "type": "individual", @@ -10869,7 +10921,7 @@ }, "peerDependencies": { "date-fns": "^2.28.0 || ^3.0.0", - "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "node_modules/react-dom": { @@ -10946,6 +10998,23 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/react-stately": { + "version": "3.46.0", + "resolved": "https://registry.npmjs.org/react-stately/-/react-stately-3.46.0.tgz", + "integrity": "sha512-OdxhWvHgs2L4OJGIs7hnuTr5WjjMM6enhNEAMRqiekhF8+ITvA2LRwNftOZwcogaoCslGYq5S2VQTQwnm0GbCA==", + "license": "Apache-2.0", + "dependencies": { + "@internationalized/date": "^3.12.1", + "@internationalized/number": "^3.6.6", + "@internationalized/string": "^3.2.8", + "@react-types/shared": "^3.34.0", + "@swc/helpers": "^0.5.0", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1" + } + }, "node_modules/react-syntax-highlighter": { "version": "15.6.6", "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz", @@ -11308,12 +11377,16 @@ "license": "MIT" }, "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "version": "2.0.0-next.6", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.6.tgz", + "integrity": "sha512-3JmVl5hMGtJ3kMmB3zi3DL25KfkCEyy3Tw7Gmw7z5w8M9WlwoPFnIvwChzu1+cF3iaK3sp18hhPz8ANeimdJfA==", + "dev": true, "license": "MIT", "dependencies": { + "es-errors": "^1.3.0", "is-core-module": "^2.16.1", + "node-exports-info": "^1.6.0", + "object-keys": "^1.1.1", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, @@ -11358,9 +11431,9 @@ } }, "node_modules/rollup": { - "version": "4.59.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", - "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.3.tgz", + "integrity": "sha512-pAQK9HalE84QSm4Po3EmWIZPd3FnjkShVkiMlz1iligWYkWQ7wHYd1PF/T7QZ5TVSD6uSTon5gBVMSM4JfBV+A==", "dev": true, "license": "MIT", "dependencies": { @@ -11374,31 +11447,31 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.59.0", - "@rollup/rollup-android-arm64": "4.59.0", - "@rollup/rollup-darwin-arm64": "4.59.0", - "@rollup/rollup-darwin-x64": "4.59.0", - "@rollup/rollup-freebsd-arm64": "4.59.0", - "@rollup/rollup-freebsd-x64": "4.59.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", - "@rollup/rollup-linux-arm-musleabihf": "4.59.0", - "@rollup/rollup-linux-arm64-gnu": "4.59.0", - "@rollup/rollup-linux-arm64-musl": "4.59.0", - "@rollup/rollup-linux-loong64-gnu": "4.59.0", - "@rollup/rollup-linux-loong64-musl": "4.59.0", - "@rollup/rollup-linux-ppc64-gnu": "4.59.0", - "@rollup/rollup-linux-ppc64-musl": "4.59.0", - "@rollup/rollup-linux-riscv64-gnu": "4.59.0", - "@rollup/rollup-linux-riscv64-musl": "4.59.0", - "@rollup/rollup-linux-s390x-gnu": "4.59.0", - "@rollup/rollup-linux-x64-gnu": "4.59.0", - "@rollup/rollup-linux-x64-musl": "4.59.0", - "@rollup/rollup-openbsd-x64": "4.59.0", - "@rollup/rollup-openharmony-arm64": "4.59.0", - "@rollup/rollup-win32-arm64-msvc": "4.59.0", - "@rollup/rollup-win32-ia32-msvc": "4.59.0", - "@rollup/rollup-win32-x64-gnu": "4.59.0", - "@rollup/rollup-win32-x64-msvc": "4.59.0", + "@rollup/rollup-android-arm-eabi": "4.60.3", + "@rollup/rollup-android-arm64": "4.60.3", + "@rollup/rollup-darwin-arm64": "4.60.3", + "@rollup/rollup-darwin-x64": "4.60.3", + "@rollup/rollup-freebsd-arm64": "4.60.3", + "@rollup/rollup-freebsd-x64": "4.60.3", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.3", + "@rollup/rollup-linux-arm-musleabihf": "4.60.3", + "@rollup/rollup-linux-arm64-gnu": "4.60.3", + "@rollup/rollup-linux-arm64-musl": "4.60.3", + "@rollup/rollup-linux-loong64-gnu": "4.60.3", + "@rollup/rollup-linux-loong64-musl": "4.60.3", + "@rollup/rollup-linux-ppc64-gnu": "4.60.3", + "@rollup/rollup-linux-ppc64-musl": "4.60.3", + "@rollup/rollup-linux-riscv64-gnu": "4.60.3", + "@rollup/rollup-linux-riscv64-musl": "4.60.3", + "@rollup/rollup-linux-s390x-gnu": "4.60.3", + "@rollup/rollup-linux-x64-gnu": "4.60.3", + "@rollup/rollup-linux-x64-musl": "4.60.3", + "@rollup/rollup-openbsd-x64": "4.60.3", + "@rollup/rollup-openharmony-arm64": "4.60.3", + "@rollup/rollup-win32-arm64-msvc": "4.60.3", + "@rollup/rollup-win32-ia32-msvc": "4.60.3", + "@rollup/rollup-win32-x64-gnu": "4.60.3", + "@rollup/rollup-win32-x64-msvc": "4.60.3", "fsevents": "~2.3.2" } }, @@ -11426,15 +11499,15 @@ } }, "node_modules/safe-array-concat": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.3.tgz", - "integrity": "sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz", + "integrity": "sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==", "dev": true, "license": "MIT", "dependencies": { - "call-bind": "^1.0.8", - "call-bound": "^1.0.2", - "get-intrinsic": "^1.2.6", + "call-bind": "^1.0.9", + "call-bound": "^1.0.4", + "get-intrinsic": "^1.3.0", "has-symbols": "^1.1.0", "isarray": "^2.0.5" }, @@ -11512,9 +11585,9 @@ } }, "node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", "devOptional": true, "license": "ISC", "bin": { @@ -11662,14 +11735,14 @@ } }, "node_modules/side-channel-list": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", - "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", - "object-inspect": "^1.13.3" + "object-inspect": "^1.13.4" }, "engines": { "node": ">= 0.4" @@ -12037,9 +12110,9 @@ } }, "node_modules/stylis": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.6.tgz", - "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.4.0.tgz", + "integrity": "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA==", "license": "MIT" }, "node_modules/sucrase": { @@ -12177,16 +12250,46 @@ "node": ">= 6" } }, + "node_modules/tailwindcss/node_modules/jiti": { + "version": "1.21.7", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", + "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", + "license": "MIT", + "bin": { + "jiti": "bin/jiti.js" + } + }, + "node_modules/tailwindcss/node_modules/resolve": { + "version": "1.22.12", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", + "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/test-exclude": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.1.tgz", - "integrity": "sha512-pFYqmTw68LXVjeWJMST4+borgQP2AyMNbg1BpZh9LbyhUeNkeaPF9gzfPGUAnSMV3qPYdWUwDIjjCLiSDOl7vg==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-7.0.2.tgz", + "integrity": "sha512-u9E6A+ZDYdp7a4WnarkXPZOx8Ilz46+kby6p1yZ8zsGTz9gYa6FIS7lj2oezzNKmtdyyJNNmmXDppga5GB7kSw==", "dev": true, "license": "ISC", "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^10.4.1", - "minimatch": "^9.0.4" + "minimatch": "^10.2.2" }, "engines": { "node": ">=18" @@ -12243,13 +12346,13 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", "license": "MIT", "dependencies": { "fdir": "^6.5.0", - "picomatch": "^4.0.3" + "picomatch": "^4.0.4" }, "engines": { "node": ">=12.0.0" @@ -12289,22 +12392,22 @@ } }, "node_modules/tldts": { - "version": "7.0.21", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.21.tgz", - "integrity": "sha512-Plu6V8fF/XU6d2k8jPtlQf5F4Xx2hAin4r2C2ca7wR8NK5MbRTo9huLUWRe28f3Uk8bYZfg74tit/dSjc18xnw==", + "version": "7.0.30", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.30.tgz", + "integrity": "sha512-ELrFxuqsDdHUwoh0XxDbxuLD3Wnz49Z57IFvTtvWy1hJdcMZjXLIuonjilCiWHlT2GbE4Wlv1wKVTzDFnXH1aw==", "dev": true, "license": "MIT", "dependencies": { - "tldts-core": "^7.0.21" + "tldts-core": "^7.0.30" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "7.0.21", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.21.tgz", - "integrity": "sha512-oVOMdHvgjqyzUZH1rOESgJP1uNe2bVrfK0jUHHmiM2rpEiRbf3j4BrsIc6JigJRbHGanQwuZv/R+LTcHsw+bLA==", + "version": "7.0.30", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.30.tgz", + "integrity": "sha512-uiHN8PIB1VmWyS98eZYja4xzlYqeFZVjb4OuYlJQnZAuJhMw4PbKQOKgHKhBdJR3FE/t5mUQ1Kd80++B+qhD1Q==", "dev": true, "license": "MIT" }, @@ -12337,9 +12440,9 @@ } }, "node_modules/tough-cookie": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.0.tgz", - "integrity": "sha512-kXuRi1mtaKMrsLUxz3sQYvVl37B0Ns6MzfrtV5DvJceE9bPyspOqk9xxv7XbZWcfLWbFmm997vl83qUWVJA64w==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", + "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -12389,9 +12492,9 @@ "license": "MIT" }, "node_modules/ts-api-utils": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.4.0.tgz", - "integrity": "sha512-3TaVTaAv2gTiMB35i3FiGJaRfwb3Pyn/j3m/bfAvGe8FB7CF6u+LMYqYlDh7reQf7UNvoTvdfAqHGmPGOSsPmA==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", "dev": true, "license": "MIT", "engines": { @@ -12719,6 +12822,15 @@ "punycode": "^2.1.0" } }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -13233,6 +13345,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "extraneous": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, "node_modules/zwitch": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", @@ -13242,6 +13364,21 @@ "type": "github", "url": "https://github.com/sponsors/wooorm" } + }, + "node_modules/@next/swc-win32-ia32-msvc": { + "version": "14.2.33", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.33.tgz", + "integrity": "sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } } } } diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index af5bba6dc3e..997e76982d4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -1102,10 +1102,6 @@ const MCPServerEdit: React.FC = ({ toolNameToDescription={toolNameToDescription} onToolNameToDisplayNameChange={setToolNameToDisplayName} onToolNameToDescriptionChange={setToolNameToDescription} - externalTools={tools} - externalIsLoading={isLoadingTools} - externalError={toolsError} - externalCanFetch={!!mcpServer.server_id} /> diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx index a3dd864b894..682ea62cbe9 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/RealtimePlayground.tsx @@ -134,10 +134,12 @@ const RealtimePlayground: React.FC = ({ const type = data.type; if (type === "session.created") { + // GA: session.type is required ("realtime" | "transcription") ws.send( JSON.stringify({ type: "session.update", session: { + type: "realtime", modalities: ["text", "audio"], voice: selectedVoice, input_audio_format: "pcm16", @@ -149,24 +151,36 @@ const RealtimePlayground: React.FC = ({ ); } else if (type === "session.updated") { // session configured - } else if (type === "response.audio.delta") { + } else if ( + // GA: response.output_audio.delta | beta: response.audio.delta + type === "response.output_audio.delta" || type === "response.audio.delta" + ) { if (data.delta) playAudioChunk(data.delta); - } else if (type === "response.audio_transcript.delta" || type === "response.text.delta") { + } else if ( + // GA: response.output_text.delta / response.output_audio_transcript.delta + // beta: response.text.delta / response.audio_transcript.delta + type === "response.output_text.delta" || + type === "response.output_audio_transcript.delta" || + type === "response.audio_transcript.delta" || + type === "response.text.delta" + ) { if (data.delta) appendAssistantText(data.delta); } else if ( type === "conversation.item.input_audio_transcription.completed" ) { if (data.transcript) addMessage("user", data.transcript); } else if (type === "response.done") { - // Ensure we have the full text if deltas were missed + // Ensure we have the full text if deltas were missed. + // Accept both beta (type=text/audio) and GA (type=output_text/output_audio) content. setMessages((prev) => { const last = prev[prev.length - 1]; if (last && last.role === "assistant" && last.content) return prev; - // No assistant message yet — extract from response.done const output = data.response?.output || []; const texts: string[] = []; for (const item of output) { for (const c of item.content || []) { + // beta: c.text (type=text), c.transcript (type=audio) + // GA: c.text (type=output_text), c.transcript (type=output_audio) const t = c.text || c.transcript; if (t) texts.push(t); } @@ -219,10 +233,12 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; // Switch to server VAD mode for voice input + // GA: session.type is required wsRef.current.send( JSON.stringify({ type: "session.update", session: { + type: "realtime", modalities: ["text", "audio"], voice: selectedVoice, input_audio_format: "pcm16", @@ -304,10 +320,12 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; if (configureSessionRef.current) return; configureSessionRef.current = true; + // GA: session.type is required wsRef.current.send( JSON.stringify({ type: "session.update", session: { + type: "realtime", modalities: ["text", "audio"], voice: selectedVoice, input_audio_format: "pcm16", diff --git a/uv.lock b/uv.lock index 8168b516f24..f8d78fe8794 100644 --- a/uv.lock +++ b/uv.lock @@ -339,6 +339,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "audioop-lts" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/53/946db57842a50b2da2e0c1e34bd37f36f5aadba1a929a3971c5d7841dbca/audioop_lts-0.2.2.tar.gz", hash = "sha256:64d0c62d88e67b98a1a5e71987b7aa7b5bcffc7dcee65b635823dbdd0a8dbbd0", size = 30686, upload-time = "2025-08-05T16:43:17.409Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/d4/94d277ca941de5a507b07f0b592f199c22454eeaec8f008a286b3fbbacd6/audioop_lts-0.2.2-cp313-abi3-macosx_10_13_universal2.whl", hash = "sha256:fd3d4602dc64914d462924a08c1a9816435a2155d74f325853c1f1ac3b2d9800", size = 46523, upload-time = "2025-08-05T16:42:20.836Z" }, + { url = "https://files.pythonhosted.org/packages/f8/5a/656d1c2da4b555920ce4177167bfeb8623d98765594af59702c8873f60ec/audioop_lts-0.2.2-cp313-abi3-macosx_10_13_x86_64.whl", hash = "sha256:550c114a8df0aafe9a05442a1162dfc8fec37e9af1d625ae6060fed6e756f303", size = 27455, upload-time = "2025-08-05T16:42:22.283Z" }, + { url = "https://files.pythonhosted.org/packages/1b/83/ea581e364ce7b0d41456fb79d6ee0ad482beda61faf0cab20cbd4c63a541/audioop_lts-0.2.2-cp313-abi3-macosx_11_0_arm64.whl", hash = "sha256:9a13dc409f2564de15dd68be65b462ba0dde01b19663720c68c1140c782d1d75", size = 26997, upload-time = "2025-08-05T16:42:23.849Z" }, + { url = "https://files.pythonhosted.org/packages/b8/3b/e8964210b5e216e5041593b7d33e97ee65967f17c282e8510d19c666dab4/audioop_lts-0.2.2-cp313-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:51c916108c56aa6e426ce611946f901badac950ee2ddaf302b7ed35d9958970d", size = 85844, upload-time = "2025-08-05T16:42:25.208Z" }, + { url = "https://files.pythonhosted.org/packages/c7/2e/0a1c52faf10d51def20531a59ce4c706cb7952323b11709e10de324d6493/audioop_lts-0.2.2-cp313-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47eba38322370347b1c47024defbd36374a211e8dd5b0dcbce7b34fdb6f8847b", size = 85056, upload-time = "2025-08-05T16:42:26.559Z" }, + { url = "https://files.pythonhosted.org/packages/75/e8/cd95eef479656cb75ab05dfece8c1f8c395d17a7c651d88f8e6e291a63ab/audioop_lts-0.2.2-cp313-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba7c3a7e5f23e215cb271516197030c32aef2e754252c4c70a50aaff7031a2c8", size = 93892, upload-time = "2025-08-05T16:42:27.902Z" }, + { url = "https://files.pythonhosted.org/packages/5c/1e/a0c42570b74f83efa5cca34905b3eef03f7ab09fe5637015df538a7f3345/audioop_lts-0.2.2-cp313-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def246fe9e180626731b26e89816e79aae2276f825420a07b4a647abaa84becc", size = 96660, upload-time = "2025-08-05T16:42:28.9Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/8a0ae607ca07dbb34027bac8db805498ee7bfecc05fd2c148cc1ed7646e7/audioop_lts-0.2.2-cp313-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e160bf9df356d841bb6c180eeeea1834085464626dc1b68fa4e1d59070affdc3", size = 79143, upload-time = "2025-08-05T16:42:29.929Z" }, + { url = "https://files.pythonhosted.org/packages/12/17/0d28c46179e7910bfb0bb62760ccb33edb5de973052cb2230b662c14ca2e/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4b4cd51a57b698b2d06cb9993b7ac8dfe89a3b2878e96bc7948e9f19ff51dba6", size = 84313, upload-time = "2025-08-05T16:42:30.949Z" }, + { url = "https://files.pythonhosted.org/packages/84/ba/bd5d3806641564f2024e97ca98ea8f8811d4e01d9b9f9831474bc9e14f9e/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4a53aa7c16a60a6857e6b0b165261436396ef7293f8b5c9c828a3a203147ed4a", size = 93044, upload-time = "2025-08-05T16:42:31.959Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5e/435ce8d5642f1f7679540d1e73c1c42d933331c0976eb397d1717d7f01a3/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:3fc38008969796f0f689f1453722a0f463da1b8a6fbee11987830bfbb664f623", size = 78766, upload-time = "2025-08-05T16:42:33.302Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3b/b909e76b606cbfd53875693ec8c156e93e15a1366a012f0b7e4fb52d3c34/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_s390x.whl", hash = "sha256:15ab25dd3e620790f40e9ead897f91e79c0d3ce65fe193c8ed6c26cffdd24be7", size = 87640, upload-time = "2025-08-05T16:42:34.854Z" }, + { url = "https://files.pythonhosted.org/packages/30/e7/8f1603b4572d79b775f2140d7952f200f5e6c62904585d08a01f0a70393a/audioop_lts-0.2.2-cp313-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:03f061a1915538fd96272bac9551841859dbb2e3bf73ebe4a23ef043766f5449", size = 86052, upload-time = "2025-08-05T16:42:35.839Z" }, + { url = "https://files.pythonhosted.org/packages/b5/96/c37846df657ccdda62ba1ae2b6534fa90e2e1b1742ca8dcf8ebd38c53801/audioop_lts-0.2.2-cp313-abi3-win32.whl", hash = "sha256:3bcddaaf6cc5935a300a8387c99f7a7fbbe212a11568ec6cf6e4bc458c048636", size = 26185, upload-time = "2025-08-05T16:42:37.04Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/9d78fdb5b844a83da8a71226c7bdae7cc638861085fff7a1d707cb4823fa/audioop_lts-0.2.2-cp313-abi3-win_amd64.whl", hash = "sha256:a2c2a947fae7d1062ef08c4e369e0ba2086049a5e598fda41122535557012e9e", size = 30503, upload-time = "2025-08-05T16:42:38.427Z" }, + { url = "https://files.pythonhosted.org/packages/34/25/20d8fde083123e90c61b51afb547bb0ea7e77bab50d98c0ab243d02a0e43/audioop_lts-0.2.2-cp313-abi3-win_arm64.whl", hash = "sha256:5f93a5db13927a37d2d09637ccca4b2b6b48c19cd9eda7b17a2e9f77edee6a6f", size = 24173, upload-time = "2025-08-05T16:42:39.704Z" }, + { url = "https://files.pythonhosted.org/packages/58/a7/0a764f77b5c4ac58dc13c01a580f5d32ae8c74c92020b961556a43e26d02/audioop_lts-0.2.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:73f80bf4cd5d2ca7814da30a120de1f9408ee0619cc75da87d0641273d202a09", size = 47096, upload-time = "2025-08-05T16:42:40.684Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ed/ebebedde1a18848b085ad0fa54b66ceb95f1f94a3fc04f1cd1b5ccb0ed42/audioop_lts-0.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:106753a83a25ee4d6f473f2be6b0966fc1c9af7e0017192f5531a3e7463dce58", size = 27748, upload-time = "2025-08-05T16:42:41.992Z" }, + { url = "https://files.pythonhosted.org/packages/cb/6e/11ca8c21af79f15dbb1c7f8017952ee8c810c438ce4e2b25638dfef2b02c/audioop_lts-0.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fbdd522624141e40948ab3e8cdae6e04c748d78710e9f0f8d4dae2750831de19", size = 27329, upload-time = "2025-08-05T16:42:42.987Z" }, + { url = "https://files.pythonhosted.org/packages/84/52/0022f93d56d85eec5da6b9da6a958a1ef09e80c39f2cc0a590c6af81dcbb/audioop_lts-0.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:143fad0311e8209ece30a8dbddab3b65ab419cbe8c0dde6e8828da25999be911", size = 92407, upload-time = "2025-08-05T16:42:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/87/1d/48a889855e67be8718adbc7a01f3c01d5743c325453a5e81cf3717664aad/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfbbc74ec68a0fd08cfec1f4b5e8cca3d3cd7de5501b01c4b5d209995033cde9", size = 91811, upload-time = "2025-08-05T16:42:45.325Z" }, + { url = "https://files.pythonhosted.org/packages/98/a6/94b7213190e8077547ffae75e13ed05edc488653c85aa5c41472c297d295/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfcac6aa6f42397471e4943e0feb2244549db5c5d01efcd02725b96af417f3fe", size = 100470, upload-time = "2025-08-05T16:42:46.468Z" }, + { url = "https://files.pythonhosted.org/packages/e9/e9/78450d7cb921ede0cfc33426d3a8023a3bda755883c95c868ee36db8d48d/audioop_lts-0.2.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:752d76472d9804ac60f0078c79cdae8b956f293177acd2316cd1e15149aee132", size = 103878, upload-time = "2025-08-05T16:42:47.576Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e2/cd5439aad4f3e34ae1ee852025dc6aa8f67a82b97641e390bf7bd9891d3e/audioop_lts-0.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:83c381767e2cc10e93e40281a04852facc4cd9334550e0f392f72d1c0a9c5753", size = 84867, upload-time = "2025-08-05T16:42:49.003Z" }, + { url = "https://files.pythonhosted.org/packages/68/4b/9d853e9076c43ebba0d411e8d2aa19061083349ac695a7d082540bad64d0/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c0022283e9556e0f3643b7c3c03f05063ca72b3063291834cca43234f20c60bb", size = 90001, upload-time = "2025-08-05T16:42:50.038Z" }, + { url = "https://files.pythonhosted.org/packages/58/26/4bae7f9d2f116ed5593989d0e521d679b0d583973d203384679323d8fa85/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:a2d4f1513d63c795e82948e1305f31a6d530626e5f9f2605408b300ae6095093", size = 99046, upload-time = "2025-08-05T16:42:51.111Z" }, + { url = "https://files.pythonhosted.org/packages/b2/67/a9f4fb3e250dda9e9046f8866e9fa7d52664f8985e445c6b4ad6dfb55641/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c9c8e68d8b4a56fda8c025e538e639f8c5953f5073886b596c93ec9b620055e7", size = 84788, upload-time = "2025-08-05T16:42:52.198Z" }, + { url = "https://files.pythonhosted.org/packages/70/f7/3de86562db0121956148bcb0fe5b506615e3bcf6e63c4357a612b910765a/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:96f19de485a2925314f5020e85911fb447ff5fbef56e8c7c6927851b95533a1c", size = 94472, upload-time = "2025-08-05T16:42:53.59Z" }, + { url = "https://files.pythonhosted.org/packages/f1/32/fd772bf9078ae1001207d2df1eef3da05bea611a87dd0e8217989b2848fa/audioop_lts-0.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e541c3ef484852ef36545f66209444c48b28661e864ccadb29daddb6a4b8e5f5", size = 92279, upload-time = "2025-08-05T16:42:54.632Z" }, + { url = "https://files.pythonhosted.org/packages/4f/41/affea7181592ab0ab560044632571a38edaf9130b84928177823fbf3176a/audioop_lts-0.2.2-cp313-cp313t-win32.whl", hash = "sha256:d5e73fa573e273e4f2e5ff96f9043858a5e9311e94ffefd88a3186a910c70917", size = 26568, upload-time = "2025-08-05T16:42:55.627Z" }, + { url = "https://files.pythonhosted.org/packages/28/2b/0372842877016641db8fc54d5c88596b542eec2f8f6c20a36fb6612bf9ee/audioop_lts-0.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9191d68659eda01e448188f60364c7763a7ca6653ed3f87ebb165822153a8547", size = 30942, upload-time = "2025-08-05T16:42:56.674Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/baf2b9cc7e96c179bb4a54f30fcd83e6ecb340031bde68f486403f943768/audioop_lts-0.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c174e322bb5783c099aaf87faeb240c8d210686b04bd61dfd05a8e5a83d88969", size = 24603, upload-time = "2025-08-05T16:42:57.571Z" }, +] + +[[package]] +name = "audioread" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "standard-aifc", marker = "python_full_version >= '3.13'" }, + { name = "standard-sunau", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/4a/874ecf9b472f998130c2b5e145dcdb9f6131e84786111489103b66772143/audioread-3.1.0.tar.gz", hash = "sha256:1c4ab2f2972764c896a8ac61ac53e261c8d29f0c6ccd652f84e18f08a4cab190", size = 20082, upload-time = "2025-10-26T19:44:13.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, +] + [[package]] name = "aurelio-sdk" version = "0.0.19" @@ -2176,6 +2229,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/58/317b0134129b556a93a3b0afe00ee675b5657f0155509e22fcb853bafe2d/grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3", size = 14424, upload-time = "2025-06-28T04:23:42.136Z" }, ] +[[package]] +name = "grpcio-tools" +version = "1.71.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, + { name = "setuptools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/9a/edfefb47f11ef6b0f39eea4d8f022c5bb05ac1d14fcc7058e84a51305b73/grpcio_tools-1.71.2.tar.gz", hash = "sha256:b5304d65c7569b21270b568e404a5a843cf027c66552a6a0978b23f137679c09", size = 5330655, upload-time = "2025-06-28T04:22:00.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/ad/e74a4d1cffff628c2ef1ec5b9944fb098207cc4af6eb8db4bc52e6d99236/grpcio_tools-1.71.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:ab8a28c2e795520d6dc6ffd7efaef4565026dbf9b4f5270de2f3dd1ce61d2318", size = 2385557, upload-time = "2025-06-28T04:20:38.833Z" }, + { url = "https://files.pythonhosted.org/packages/63/bf/30b63418279d6fdc4fd4a3781a7976c40c7e8ee052333b9ce6bd4ce63f30/grpcio_tools-1.71.2-cp310-cp310-macosx_10_14_universal2.whl", hash = "sha256:654ecb284a592d39a85556098b8c5125163435472a20ead79b805cf91814b99e", size = 5446915, upload-time = "2025-06-28T04:20:40.947Z" }, + { url = "https://files.pythonhosted.org/packages/83/cd/2994e0a0a67714fdb00c207c4bec60b9b356fbd6b0b7a162ecaabe925155/grpcio_tools-1.71.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b49aded2b6c890ff690d960e4399a336c652315c6342232c27bd601b3705739e", size = 2348301, upload-time = "2025-06-28T04:20:42.766Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8b/4f2315927af306af1b35793b332b9ca9dc5b5a2cde2d55811c9577b5f03f/grpcio_tools-1.71.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7811a6fc1c4b4e5438e5eb98dbd52c2dc4a69d1009001c13356e6636322d41a", size = 2742159, upload-time = "2025-06-28T04:20:44.206Z" }, + { url = "https://files.pythonhosted.org/packages/8d/98/d513f6c09df405c82583e7083c20718ea615ed0da69ec42c80ceae7ebdc5/grpcio_tools-1.71.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:393a9c80596aa2b3f05af854e23336ea8c295593bbb35d9adae3d8d7943672bd", size = 2473444, upload-time = "2025-06-28T04:20:45.5Z" }, + { url = "https://files.pythonhosted.org/packages/fa/fe/00af17cc841916d5e4227f11036bf443ce006629212c876937c7904b0ba3/grpcio_tools-1.71.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:823e1f23c12da00f318404c4a834bb77cd150d14387dee9789ec21b335249e46", size = 2850339, upload-time = "2025-06-28T04:20:46.758Z" }, + { url = "https://files.pythonhosted.org/packages/7d/59/745fc50dfdbed875fcfd6433883270d39d23fb1aa4ecc9587786f772dce3/grpcio_tools-1.71.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9bfbea79d6aec60f2587133ba766ede3dc3e229641d1a1e61d790d742a3d19eb", size = 3300795, upload-time = "2025-06-28T04:20:48.327Z" }, + { url = "https://files.pythonhosted.org/packages/62/3e/d9d0fb2df78e601c28d02ef0cd5d007f113c1b04fc21e72bf56e8c3df66b/grpcio_tools-1.71.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:32f3a67b10728835b5ffb63fbdbe696d00e19a27561b9cf5153e72dbb93021ba", size = 2913729, upload-time = "2025-06-28T04:20:49.641Z" }, + { url = "https://files.pythonhosted.org/packages/09/ae/ddb264b4a10c6c10336a7c177f8738b230c2c473d0c91dd5d8ce8ea1b857/grpcio_tools-1.71.2-cp310-cp310-win32.whl", hash = "sha256:7fcf9d92c710bfc93a1c0115f25e7d49a65032ff662b38b2f704668ce0a938df", size = 945997, upload-time = "2025-06-28T04:20:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/ad/8d/5efd93698fe359f63719d934ebb2d9337e82d396e13d6bf00f4b06793e37/grpcio_tools-1.71.2-cp310-cp310-win_amd64.whl", hash = "sha256:914b4275be810290266e62349f2d020bb7cc6ecf9edb81da3c5cddb61a95721b", size = 1117474, upload-time = "2025-06-28T04:20:52.54Z" }, + { url = "https://files.pythonhosted.org/packages/17/e4/0568d38b8da6237ea8ea15abb960fb7ab83eb7bb51e0ea5926dab3d865b1/grpcio_tools-1.71.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:0acb8151ea866be5b35233877fbee6445c36644c0aa77e230c9d1b46bf34b18b", size = 2385557, upload-time = "2025-06-28T04:20:54.323Z" }, + { url = "https://files.pythonhosted.org/packages/76/fb/700d46f72b0f636cf0e625f3c18a4f74543ff127471377e49a071f64f1e7/grpcio_tools-1.71.2-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:b28f8606f4123edb4e6da281547465d6e449e89f0c943c376d1732dc65e6d8b3", size = 5447590, upload-time = "2025-06-28T04:20:55.836Z" }, + { url = "https://files.pythonhosted.org/packages/12/69/d9bb2aec3de305162b23c5c884b9f79b1a195d42b1e6dabcc084cc9d0804/grpcio_tools-1.71.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:cbae6f849ad2d1f5e26cd55448b9828e678cb947fa32c8729d01998238266a6a", size = 2348495, upload-time = "2025-06-28T04:20:57.33Z" }, + { url = "https://files.pythonhosted.org/packages/d5/83/f840aba1690461b65330efbca96170893ee02fae66651bcc75f28b33a46c/grpcio_tools-1.71.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4d1027615cfb1e9b1f31f2f384251c847d68c2f3e025697e5f5c72e26ed1316", size = 2742333, upload-time = "2025-06-28T04:20:59.051Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/c02cd9b37de26045190ba665ee6ab8597d47f033d098968f812d253bbf8c/grpcio_tools-1.71.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9bac95662dc69338edb9eb727cc3dd92342131b84b12b3e8ec6abe973d4cbf1b", size = 2473490, upload-time = "2025-06-28T04:21:00.614Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c7/375718ae091c8f5776828ce97bdcb014ca26244296f8b7f70af1a803ed2f/grpcio_tools-1.71.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c50250c7248055040f89eb29ecad39d3a260a4b6d3696af1575945f7a8d5dcdc", size = 2850333, upload-time = "2025-06-28T04:21:01.95Z" }, + { url = "https://files.pythonhosted.org/packages/19/37/efc69345bd92a73b2bc80f4f9e53d42dfdc234b2491ae58c87da20ca0ea5/grpcio_tools-1.71.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6ab1ad955e69027ef12ace4d700c5fc36341bdc2f420e87881e9d6d02af3d7b8", size = 3300748, upload-time = "2025-06-28T04:21:03.451Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1f/15f787eb25ae42086f55ed3e4260e85f385921c788debf0f7583b34446e3/grpcio_tools-1.71.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dd75dde575781262b6b96cc6d0b2ac6002b2f50882bf5e06713f1bf364ee6e09", size = 2913178, upload-time = "2025-06-28T04:21:04.879Z" }, + { url = "https://files.pythonhosted.org/packages/12/aa/69cb3a9dff7d143a05e4021c3c9b5cde07aacb8eb1c892b7c5b9fb4973e3/grpcio_tools-1.71.2-cp311-cp311-win32.whl", hash = "sha256:9a3cb244d2bfe0d187f858c5408d17cb0e76ca60ec9a274c8fd94cc81457c7fc", size = 946256, upload-time = "2025-06-28T04:21:06.518Z" }, + { url = "https://files.pythonhosted.org/packages/1e/df/fb951c5c87eadb507a832243942e56e67d50d7667b0e5324616ffd51b845/grpcio_tools-1.71.2-cp311-cp311-win_amd64.whl", hash = "sha256:00eb909997fd359a39b789342b476cbe291f4dd9c01ae9887a474f35972a257e", size = 1117661, upload-time = "2025-06-28T04:21:08.18Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d3/3ed30a9c5b2424627b4b8411e2cd6a1a3f997d3812dbc6a8630a78bcfe26/grpcio_tools-1.71.2-cp312-cp312-linux_armv7l.whl", hash = "sha256:bfc0b5d289e383bc7d317f0e64c9dfb59dc4bef078ecd23afa1a816358fb1473", size = 2385479, upload-time = "2025-06-28T04:21:10.413Z" }, + { url = "https://files.pythonhosted.org/packages/54/61/e0b7295456c7e21ef777eae60403c06835160c8d0e1e58ebfc7d024c51d3/grpcio_tools-1.71.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b4669827716355fa913b1376b1b985855d5cfdb63443f8d18faf210180199006", size = 5431521, upload-time = "2025-06-28T04:21:12.261Z" }, + { url = "https://files.pythonhosted.org/packages/75/d7/7bcad6bcc5f5b7fab53e6bce5db87041f38ef3e740b1ec2d8c49534fa286/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:d4071f9b44564e3f75cdf0f05b10b3e8c7ea0ca5220acbf4dc50b148552eef2f", size = 2350289, upload-time = "2025-06-28T04:21:13.625Z" }, + { url = "https://files.pythonhosted.org/packages/b2/8a/e4c1c4cb8c9ff7f50b7b2bba94abe8d1e98ea05f52a5db476e7f1c1a3c70/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a28eda8137d587eb30081384c256f5e5de7feda34776f89848b846da64e4be35", size = 2743321, upload-time = "2025-06-28T04:21:15.007Z" }, + { url = "https://files.pythonhosted.org/packages/fd/aa/95bc77fda5c2d56fb4a318c1b22bdba8914d5d84602525c99047114de531/grpcio_tools-1.71.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b19c083198f5eb15cc69c0a2f2c415540cbc636bfe76cea268e5894f34023b40", size = 2474005, upload-time = "2025-06-28T04:21:16.443Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ff/ca11f930fe1daa799ee0ce1ac9630d58a3a3deed3dd2f465edb9a32f299d/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:784c284acda0d925052be19053d35afbf78300f4d025836d424cf632404f676a", size = 2851559, upload-time = "2025-06-28T04:21:18.139Z" }, + { url = "https://files.pythonhosted.org/packages/64/10/c6fc97914c7e19c9bb061722e55052fa3f575165da9f6510e2038d6e8643/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:381e684d29a5d052194e095546eef067201f5af30fd99b07b5d94766f44bf1ae", size = 3300622, upload-time = "2025-06-28T04:21:20.291Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d6/965f36cfc367c276799b730d5dd1311b90a54a33726e561393b808339b04/grpcio_tools-1.71.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3e4b4801fabd0427fc61d50d09588a01b1cfab0ec5e8a5f5d515fbdd0891fd11", size = 2913863, upload-time = "2025-06-28T04:21:22.196Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f0/c05d5c3d0c1d79ac87df964e9d36f1e3a77b60d948af65bec35d3e5c75a3/grpcio_tools-1.71.2-cp312-cp312-win32.whl", hash = "sha256:84ad86332c44572305138eafa4cc30040c9a5e81826993eae8227863b700b490", size = 945744, upload-time = "2025-06-28T04:21:23.463Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e9/c84c1078f0b7af7d8a40f5214a9bdd8d2a567ad6c09975e6e2613a08d29d/grpcio_tools-1.71.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e1108d37eecc73b1c4a27350a6ed921b5dda25091700c1da17cfe30761cd462", size = 1117695, upload-time = "2025-06-28T04:21:25.22Z" }, + { url = "https://files.pythonhosted.org/packages/60/9c/bdf9c5055a1ad0a09123402d73ecad3629f75b9cf97828d547173b328891/grpcio_tools-1.71.2-cp313-cp313-linux_armv7l.whl", hash = "sha256:b0f0a8611614949c906e25c225e3360551b488d10a366c96d89856bcef09f729", size = 2384758, upload-time = "2025-06-28T04:21:26.712Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/6aaee4940a8fb8269c13719f56d69c8d39569bee272924086aef81616d4a/grpcio_tools-1.71.2-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:7931783ea7ac42ac57f94c5047d00a504f72fbd96118bf7df911bb0e0435fc0f", size = 5443127, upload-time = "2025-06-28T04:21:28.383Z" }, + { url = "https://files.pythonhosted.org/packages/d9/11/50a471dcf301b89c0ed5ab92c533baced5bd8f796abfd133bbfadf6b60e5/grpcio_tools-1.71.2-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:d188dc28e069aa96bb48cb11b1338e47ebdf2e2306afa58a8162cc210172d7a8", size = 2349627, upload-time = "2025-06-28T04:21:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/bb/66/e3dc58362a9c4c2fbe98a7ceb7e252385777ebb2bbc7f42d5ab138d07ace/grpcio_tools-1.71.2-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f36c4b3cc42ad6ef67430639174aaf4a862d236c03c4552c4521501422bfaa26", size = 2742932, upload-time = "2025-06-28T04:21:32.325Z" }, + { url = "https://files.pythonhosted.org/packages/b7/1e/1e07a07ed8651a2aa9f56095411198385a04a628beba796f36d98a5a03ec/grpcio_tools-1.71.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bd9ed12ce93b310f0cef304176049d0bc3b9f825e9c8c6a23e35867fed6affd", size = 2473627, upload-time = "2025-06-28T04:21:33.752Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f9/3b7b32e4acb419f3a0b4d381bc114fe6cd48e3b778e81273fc9e4748caad/grpcio_tools-1.71.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7ce27e76dd61011182d39abca38bae55d8a277e9b7fe30f6d5466255baccb579", size = 2850879, upload-time = "2025-06-28T04:21:35.241Z" }, + { url = "https://files.pythonhosted.org/packages/1e/99/cd9e1acd84315ce05ad1fcdfabf73b7df43807cf00c3b781db372d92b899/grpcio_tools-1.71.2-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:dcc17bf59b85c3676818f2219deacac0156492f32ca165e048427d2d3e6e1157", size = 3300216, upload-time = "2025-06-28T04:21:36.826Z" }, + { url = "https://files.pythonhosted.org/packages/9f/c0/66eab57b14550c5b22404dbf60635c9e33efa003bd747211981a9859b94b/grpcio_tools-1.71.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:706360c71bdd722682927a1fb517c276ccb816f1e30cb71f33553e5817dc4031", size = 2913521, upload-time = "2025-06-28T04:21:38.347Z" }, + { url = "https://files.pythonhosted.org/packages/05/9b/7c90af8f937d77005625d705ab1160bc42a7e7b021ee5c788192763bccd6/grpcio_tools-1.71.2-cp313-cp313-win32.whl", hash = "sha256:bcf751d5a81c918c26adb2d6abcef71035c77d6eb9dd16afaf176ee096e22c1d", size = 945322, upload-time = "2025-06-28T04:21:39.864Z" }, + { url = "https://files.pythonhosted.org/packages/5f/80/6db6247f767c94fe551761772f89ceea355ff295fd4574cb8efc8b2d1199/grpcio_tools-1.71.2-cp313-cp313-win_amd64.whl", hash = "sha256:b1581a1133552aba96a730178bc44f6f1a071f0eb81c5b6bc4c0f89f5314e2b8", size = 1117234, upload-time = "2025-06-28T04:21:41.893Z" }, +] + [[package]] name = "gunicorn" version = "23.0.0" @@ -3083,7 +3189,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.84.0" +version = "1.85.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -3174,6 +3280,13 @@ semantic-router = [ { name = "aurelio-sdk" }, { name = "semantic-router" }, ] +stt-nvidia-riva = [ + { name = "audioread" }, + { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "nvidia-riva-client" }, + { name = "soundfile" }, +] utils = [ { name = "numpydoc" }, ] @@ -3261,9 +3374,10 @@ proxy-dev = [ [package.metadata] requires-dist = [ { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = "==0.3.24" }, - { name = "aiohttp", specifier = "==3.13.4" }, + { name = "aiohttp", specifier = ">=3.10,<4.0" }, { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = "==0.84.0" }, { name = "apscheduler", marker = "extra == 'proxy'", specifier = "==3.11.2" }, + { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, { name = "aurelio-sdk", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = "==0.0.19" }, { name = "azure-ai-contentsafety", marker = "extra == 'proxy-runtime'", specifier = "==1.0.0" }, { name = "azure-identity", marker = "extra == 'extra-proxy'", specifier = "==1.25.2" }, @@ -3273,14 +3387,14 @@ requires-dist = [ { name = "azure-storage-file-datalake", marker = "extra == 'proxy-runtime'", specifier = "==12.20.0" }, { name = "backoff", marker = "extra == 'proxy'", specifier = "==2.2.1" }, { name = "boto3", marker = "extra == 'proxy'", specifier = "==1.43.1" }, - { name = "click", specifier = "==8.1.8" }, + { name = "click", specifier = ">=8.0.0,<9.0" }, { name = "cryptography", marker = "extra == 'proxy'", specifier = "==46.0.7" }, { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = "==2.19.0" }, { name = "detect-secrets", marker = "extra == 'proxy-runtime'", specifier = "==1.5.0" }, { name = "diskcache", marker = "extra == 'caching'", specifier = "==5.6.3" }, { name = "fastapi", marker = "extra == 'proxy'", specifier = "==0.124.4" }, { name = "fastapi-sso", marker = "extra == 'proxy'", specifier = "==0.19.0" }, - { name = "fastuuid", specifier = "==0.14.0" }, + { name = "fastuuid", specifier = ">=0.14.0,<1.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = "==1.133.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'proxy-runtime'", specifier = "==1.133.0" }, { name = "google-cloud-iam", marker = "extra == 'extra-proxy'", specifier = "==2.19.1" }, @@ -3289,10 +3403,10 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = "==23.0.0" }, - { name = "httpx", specifier = "==0.28.1" }, - { name = "importlib-metadata", specifier = "==8.5.0" }, - { name = "jinja2", specifier = "==3.1.6" }, - { name = "jsonschema", specifier = "==4.23.0" }, + { name = "httpx", specifier = ">=0.28.0,<1.0" }, + { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, + { name = "jinja2", specifier = ">=3.1.0,<4.0" }, + { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = "==2.59.7" }, { name = "litellm-enterprise", marker = "extra == 'proxy'", editable = "enterprise" }, { name = "litellm-proxy-extras", marker = "extra == 'proxy'", editable = "litellm-proxy-extras" }, @@ -3300,8 +3414,10 @@ requires-dist = [ { name = "mangum", marker = "extra == 'proxy-runtime'", specifier = "==0.17.0" }, { name = "mcp", marker = "extra == 'proxy'", specifier = "==1.26.0" }, { name = "mlflow", marker = "extra == 'mlflow'", specifier = "==3.11.1" }, + { name = "numpy", marker = "extra == 'stt-nvidia-riva'", specifier = ">=1.26.0" }, { name = "numpydoc", marker = "extra == 'utils'", specifier = "==1.8.0" }, - { name = "openai", specifier = "==2.33.0" }, + { name = "nvidia-riva-client", marker = "extra == 'stt-nvidia-riva'", specifier = ">=2.15.0" }, + { name = "openai", specifier = ">=2.20.0,<3.0.0" }, { name = "opentelemetry-api", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, { name = "opentelemetry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==1.28.0" }, @@ -3309,12 +3425,12 @@ requires-dist = [ { name = "polars", marker = "extra == 'proxy'", specifier = "==1.38.1" }, { name = "prisma", marker = "extra == 'extra-proxy'", specifier = "==0.11.0" }, { name = "prometheus-client", marker = "extra == 'proxy-runtime'", specifier = "==0.20.0" }, - { name = "pydantic", specifier = "==2.12.5" }, + { name = "pydantic", specifier = ">=2.10.0,<3.0.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = "==2.12.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = "==1.6.2" }, { name = "pypdf", marker = "python_full_version < '3.14' and extra == 'proxy-runtime'", specifier = "==6.10.2" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = "==0.8.16" }, - { name = "python-dotenv", specifier = "==1.2.2" }, + { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = "==0.0.27" }, { name = "pyyaml", marker = "extra == 'proxy'", specifier = "==6.0.3" }, { name = "redisvl", marker = "python_full_version < '3.14' and extra == 'extra-proxy'", specifier = "==0.4.1" }, @@ -3325,13 +3441,14 @@ requires-dist = [ { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = "==0.1.12" }, { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = "==2.21.0" }, { name = "soundfile", marker = "extra == 'proxy'", specifier = "==0.12.1" }, - { name = "tiktoken", specifier = "==0.12.0" }, - { name = "tokenizers", specifier = "==0.23.1" }, + { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, + { name = "tiktoken", specifier = ">=0.8.0,<1.0" }, + { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, { name = "uvicorn", marker = "extra == 'proxy'", specifier = "==0.33.0" }, { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = "==0.21.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = "==15.0.1" }, ] -provides-extras = ["proxy", "extra-proxy", "utils", "caching", "semantic-router", "mlflow", "grpc", "google", "proxy-runtime"] +provides-extras = ["proxy", "extra-proxy", "utils", "caching", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "proxy-runtime"] [package.metadata.requires-dev] ci = [ @@ -4156,6 +4273,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/45/56d99ba9366476cd8548527667f01869279cedb9e66b28eb4dfb27701679/numpydoc-1.8.0-py3-none-any.whl", hash = "sha256:72024c7fd5e17375dec3608a27c03303e8ad00c81292667955c6fea7a3ccf541", size = 64003, upload-time = "2024-08-09T15:52:37.276Z" }, ] +[[package]] +name = "nvidia-riva-client" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio-tools" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/82/0484c225bebe7ed37334474fba5c6ac7228638e692b84da0a0e7f2395672/nvidia_riva_client-2.16.0-py3-none-any.whl", hash = "sha256:99ef37b8f487d75a70c053736848221e09b728e5c910fb476333d375bd4347a3", size = 45491, upload-time = "2024-07-02T14:54:22.63Z" }, +] + [[package]] name = "oauthlib" version = "3.3.1" @@ -7068,6 +7197,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ff/07/45c21ed03d708c477367305726b89919b020a3a2a01f72aaf5ad941caf35/sse_starlette-3.4.1-py3-none-any.whl", hash = "sha256:6b43cf21f1d574d582a6e1b0cfbde1c94dc86a32a701a7168c99c4475c6bd1d0", size = 16487, upload-time = "2026-04-26T13:32:30.819Z" }, ] +[[package]] +name = "standard-aifc" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/52/5fbb203394cc852334d1575cc020f6bcec768d2265355984dfd361968f36/standard_aifc-3.13.0-py3-none-any.whl", hash = "sha256:f7ae09cc57de1224a0dd8e3eb8f73830be7c3d0bc485de4c1f82b4a7f645ac66", size = 10492, upload-time = "2024-10-30T16:01:07.071Z" }, +] + +[[package]] +name = "standard-chunk" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/06/ce1bb165c1f111c7d23a1ad17204d67224baa69725bb6857a264db61beaf/standard_chunk-3.13.0.tar.gz", hash = "sha256:4ac345d37d7e686d2755e01836b8d98eda0d1a3ee90375e597ae43aaf064d654", size = 4672, upload-time = "2024-10-30T16:18:28.326Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/90/a5c1084d87767d787a6caba615aa50dc587229646308d9420c960cb5e4c0/standard_chunk-3.13.0-py3-none-any.whl", hash = "sha256:17880a26c285189c644bd5bd8f8ed2bdb795d216e3293e6dbe55bbd848e2982c", size = 4944, upload-time = "2024-10-30T16:18:26.694Z" }, +] + +[[package]] +name = "standard-sunau" +version = "3.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/e3/ce8d38cb2d70e05ffeddc28bb09bad77cfef979eb0a299c9117f7ed4e6a9/standard_sunau-3.13.0.tar.gz", hash = "sha256:b319a1ac95a09a2378a8442f403c66f4fd4b36616d6df6ae82b8e536ee790908", size = 9368, upload-time = "2024-10-30T16:01:41.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/ae/e3707f6c1bc6f7aa0df600ba8075bfb8a19252140cd595335be60e25f9ee/standard_sunau-3.13.0-py3-none-any.whl", hash = "sha256:53af624a9529c41062f4c2fd33837f297f3baa196b0cfceffea6555654602622", size = 7364, upload-time = "2024-10-30T16:01:28.003Z" }, +] + [[package]] name = "starlette" version = "0.50.0"