mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
Merge branch 'litellm_internal_staging' into feature/improve-gigachat-provider
This commit is contained in:
KnyazSh
commit
c5e8a498b2
1785 changed files with 166763 additions and 28970 deletions
|
|
@ -158,6 +158,8 @@ jobs:
|
|||
CHOCOLATEY_CONFIRM_ALL: "true"
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
$installer = Join-Path $env:TEMP "uv-install.ps1"
|
||||
Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer
|
||||
|
|
@ -180,7 +182,14 @@ jobs:
|
|||
- run:
|
||||
name: Run Windows-specific test
|
||||
command: |
|
||||
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
|
||||
uv run --no-sync python -m pytest tests/windows_tests/ -v
|
||||
- run:
|
||||
name: Guard against MAX_PATH-busting packaged wheel paths
|
||||
environment:
|
||||
UV_HTTP_TIMEOUT: "300"
|
||||
command: |
|
||||
uv build --wheel --out-dir dist
|
||||
uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
|
||||
|
||||
local_testing_part1:
|
||||
docker:
|
||||
|
|
@ -226,7 +235,7 @@ jobs:
|
|||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv \
|
||||
--cov=./litellm \
|
||||
--cov-report=xml \
|
||||
|
|
@ -240,8 +249,15 @@ jobs:
|
|||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part1_coverage.xml
|
||||
mv .coverage local_testing_part1_coverage
|
||||
# When CI reruns only the failed tests, a parallel node can receive
|
||||
# zero tests and pytest never writes coverage. Emit empty placeholders
|
||||
# so persist_to_workspace and the downstream coverage combine stay green.
|
||||
if [ -f coverage.xml ]; then
|
||||
mv coverage.xml local_testing_part1_coverage.xml
|
||||
mv .coverage local_testing_part1_coverage
|
||||
else
|
||||
touch local_testing_part1_coverage.xml local_testing_part1_coverage
|
||||
fi
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
|
|
@ -291,7 +307,7 @@ jobs:
|
|||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv \
|
||||
--cov=./litellm \
|
||||
--cov-report=xml \
|
||||
|
|
@ -305,8 +321,15 @@ jobs:
|
|||
- run:
|
||||
name: Rename the coverage files
|
||||
command: |
|
||||
mv coverage.xml local_testing_part2_coverage.xml
|
||||
mv .coverage local_testing_part2_coverage
|
||||
# When CI reruns only the failed tests, a parallel node can receive
|
||||
# zero tests and pytest never writes coverage. Emit empty placeholders
|
||||
# so persist_to_workspace and the downstream coverage combine stay green.
|
||||
if [ -f coverage.xml ]; then
|
||||
mv coverage.xml local_testing_part2_coverage.xml
|
||||
mv .coverage local_testing_part2_coverage
|
||||
else
|
||||
touch local_testing_part2_coverage.xml local_testing_part2_coverage
|
||||
fi
|
||||
|
||||
# Store test results
|
||||
- store_test_results:
|
||||
|
|
@ -354,7 +377,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5 \
|
||||
|
|
@ -407,7 +430,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -455,12 +478,17 @@ jobs:
|
|||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
# On a "rerun failed tests" build a parallel node can receive no
|
||||
# tests, so the test command never creates test-results. Pre-create it
|
||||
# so store_test_results doesn't fail the node on a missing path.
|
||||
mkdir -p test-results
|
||||
|
||||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--split-by=timings \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v \
|
||||
-k 'router' \
|
||||
-n 4 \
|
||||
|
|
@ -502,7 +530,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -545,7 +573,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5 \
|
||||
|
|
@ -587,7 +615,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=20 \
|
||||
|
|
@ -623,7 +651,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -666,7 +694,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -708,7 +736,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -752,7 +780,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -803,7 +831,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5 \
|
||||
|
|
@ -834,7 +862,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -876,7 +904,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -920,7 +948,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit-enterprise.xml \
|
||||
--durations=10 \
|
||||
|
|
@ -950,7 +978,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -992,7 +1020,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -1035,7 +1063,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -1078,7 +1106,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5 \
|
||||
|
|
@ -1110,7 +1138,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
-n 4 \
|
||||
|
|
@ -1153,7 +1181,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -1204,7 +1232,7 @@ jobs:
|
|||
tests/local_testing/test_router_utils.py)
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--cov=./litellm --cov-report=xml \
|
||||
--junitxml=test-results/junit.xml \
|
||||
|
|
@ -1454,7 +1482,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit-2.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1537,7 +1565,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-s -v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
-n 4 \
|
||||
|
|
@ -1620,7 +1648,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-s -vv \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1696,7 +1724,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1746,7 +1774,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit-2.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1822,7 +1850,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1920,7 +1948,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -1983,7 +2011,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -2063,7 +2091,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x \
|
||||
--junitxml=test-results/junit-2.xml \
|
||||
--durations=5"
|
||||
|
|
@ -2207,7 +2235,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-v -x \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -2273,7 +2301,7 @@ jobs:
|
|||
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
|
||||
echo "$TEST_FILES" | circleci tests run \
|
||||
--verbose \
|
||||
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
|
||||
-vv -x -s \
|
||||
--junitxml=test-results/junit.xml \
|
||||
--durations=5"
|
||||
|
|
@ -2398,6 +2426,11 @@ jobs:
|
|||
environment:
|
||||
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
|
||||
CI: "true"
|
||||
# Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can
|
||||
# assert the redirect. Set at job level so both the proxy boot step and the
|
||||
# Playwright step (whose skip guard reads this) see the same value. Safe for
|
||||
# the rest of the suite: nothing else performs a logout.
|
||||
PROXY_LOGOUT_URL: "https://www.example.com"
|
||||
steps:
|
||||
- checkout
|
||||
- setup_google_dns
|
||||
|
|
@ -2474,11 +2507,17 @@ jobs:
|
|||
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
||||
DISABLE_SCHEMA_UPDATE: "true"
|
||||
SERVER_ROOT_PATH: ""
|
||||
PROXY_LOGOUT_URL: ""
|
||||
# PROXY_LOGOUT_URL is inherited from the job-level environment so the
|
||||
# proxy and proxyLogoutUrl.spec.ts agree on the logout target.
|
||||
# LITELLM_LICENSE is forwarded from the project env so premium-gated
|
||||
# UI flows can be exercised. license.spec.ts asserts the resulting
|
||||
# JWT carries premium_user=true; if it ever stops being passed, that
|
||||
# test fails loudly rather than silently regressing premium coverage.
|
||||
command: |
|
||||
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
||||
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
||||
--port 4000
|
||||
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
||||
uv run --no-sync python -m litellm.proxy.proxy_cli \
|
||||
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
|
||||
--port 4000
|
||||
background: true
|
||||
- run:
|
||||
name: Wait for proxy to be ready
|
||||
|
|
@ -2495,9 +2534,12 @@ jobs:
|
|||
exit 1
|
||||
- run:
|
||||
name: Run Playwright E2E tests
|
||||
# Forward LITELLM_LICENSE so license.spec.ts can detect that the
|
||||
# proxy was launched with a license and assert premium_user=true.
|
||||
command: |
|
||||
cd ui/litellm-dashboard
|
||||
npx playwright test --config e2e_tests/playwright.config.ts
|
||||
LITELLM_LICENSE="$LITELLM_LICENSE" \
|
||||
npx playwright test --config e2e_tests/playwright.config.ts
|
||||
no_output_timeout: 10m
|
||||
- store_artifacts:
|
||||
path: ui/litellm-dashboard/test-results
|
||||
|
|
@ -2531,7 +2573,6 @@ jobs:
|
|||
paths:
|
||||
- litellm-docker-database.tar.zst
|
||||
|
||||
|
||||
test_bad_database_url:
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
|
|
|
|||
4
.github/pull_request_template.md
vendored
4
.github/pull_request_template.md
vendored
|
|
@ -10,9 +10,9 @@
|
|||
|
||||
**Please complete all items before asking a LiteLLM maintainer to review your PR**
|
||||
|
||||
- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
|
||||
- [ ] I have added meaningful tests
|
||||
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
|
||||
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
|
||||
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
|
||||
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
|
||||
|
||||
## Delays in PR merge?
|
||||
|
|
|
|||
28
.github/workflows/codeql.yml
vendored
28
.github/workflows/codeql.yml
vendored
|
|
@ -53,3 +53,31 @@ jobs:
|
|||
uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
output: sarif-results
|
||||
upload: failure-only
|
||||
|
||||
# py/weak-sensitive-data-hashing (CWE-328) fires on the OCI signing call at
|
||||
# litellm/llms/oci/common_utils.py, which hashes the HTTP request body to
|
||||
# produce the x-content-sha256 header required by the OCI HTTP signing spec —
|
||||
# a content-integrity hash, not a password or secret hash. SHA-256 is mandated
|
||||
# by Oracle for this header; see
|
||||
# https://docs.oracle.com/en-us/iaas/Content/API/Concepts/signingrequests.htm
|
||||
# The `usedforsecurity=False` flag on the hashlib.sha256 call already declares
|
||||
# non-security intent, but CodeQL's taint flow still re-fires when callers
|
||||
# further up the stack are modified. The suppression is scoped to this one
|
||||
# file/rule pair via SARIF post-filtering so every other callsite of
|
||||
# py/weak-sensitive-data-hashing in the repository continues to be analyzed.
|
||||
- name: Filter SARIF (OCI sha256)
|
||||
if: matrix.language == 'python'
|
||||
uses: advanced-security/filter-sarif@2da736ff05ef065cb2894ac6892e47b5eac2c3c0 # v1.1
|
||||
with:
|
||||
patterns: |
|
||||
-litellm/llms/oci/common_utils.py:py/weak-sensitive-data-hashing
|
||||
input: sarif-results/python.sarif
|
||||
output: sarif-results/python.sarif
|
||||
|
||||
- name: Upload SARIF
|
||||
uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
|
||||
with:
|
||||
sarif_file: sarif-results
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
|
|
|||
25
.github/workflows/create-release-branch.yml
vendored
25
.github/workflows/create-release-branch.yml
vendored
|
|
@ -63,3 +63,28 @@ jobs:
|
|||
sha: commitHash,
|
||||
});
|
||||
core.info(`Created branch ${branchName} at ${commitHash}`);
|
||||
|
||||
- name: Create stable line branch
|
||||
env:
|
||||
TAG: ${{ inputs.tag }}
|
||||
COMMIT_HASH: ${{ inputs.commit_hash }}
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
script: |
|
||||
const tag = process.env.TAG;
|
||||
const commitHash = process.env.COMMIT_HASH;
|
||||
|
||||
const match = tag.match(/^v?(\d+)\.(\d+)\.0$/);
|
||||
if (!match) {
|
||||
core.info(`Tag ${tag} is not the X.Y.0 stable opener; skipping stable line branch`);
|
||||
return;
|
||||
}
|
||||
const lineBranch = `stable/${match[1]}.${match[2]}.x`;
|
||||
|
||||
await github.rest.git.createRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: `refs/heads/${lineBranch}`,
|
||||
sha: commitHash,
|
||||
});
|
||||
core.info(`Created branch ${lineBranch} at ${commitHash}`);
|
||||
|
|
|
|||
47
.github/workflows/create_daily_oss_agent_shin_branch.yml
vendored
Normal file
47
.github/workflows/create_daily_oss_agent_shin_branch.yml
vendored
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
name: Create Daily oss-agent-shin Branch
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # Runs every day at midnight UTC
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
jobs:
|
||||
create-oss-agent-shin-branch:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Create daily oss-agent-shin branch
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Configure Git user
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
# Generate branch name with MM_DD_YYYY format
|
||||
BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')"
|
||||
echo "Creating branch: $BRANCH_NAME"
|
||||
|
||||
# Fetch all branches
|
||||
git fetch --all
|
||||
|
||||
# Check if the branch already exists
|
||||
if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
|
||||
echo "Branch $BRANCH_NAME already exists. Skipping creation."
|
||||
else
|
||||
echo "Creating new branch: $BRANCH_NAME"
|
||||
# Create the new branch from main
|
||||
git checkout -b $BRANCH_NAME origin/main
|
||||
# Push the new branch
|
||||
git push origin $BRANCH_NAME
|
||||
echo "Successfully created and pushed branch: $BRANCH_NAME"
|
||||
fi
|
||||
76
.github/workflows/test-litellm-ui-build.yml
vendored
76
.github/workflows/test-litellm-ui-build.yml
vendored
|
|
@ -36,3 +36,79 @@ jobs:
|
|||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
frontend-lint:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui/litellm-dashboard
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Collect changed files
|
||||
id: changed
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
: > "$RUNNER_TEMP/eslint_files.txt"
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
case "$f" in
|
||||
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
|
||||
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
|
||||
esac
|
||||
done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
|
||||
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
|
||||
echo "has_files=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_files=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No lintable UI files changed in this PR; nothing to check."
|
||||
fi
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
|
||||
with:
|
||||
node-version: "20"
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Lint changed files (prettier + eslint)
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: |
|
||||
prettier_files=()
|
||||
eslint_files=()
|
||||
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
|
||||
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
|
||||
status=0
|
||||
if [ ${#prettier_files[@]} -gt 0 ]; then
|
||||
echo "::group::Prettier (${#prettier_files[@]} files)"
|
||||
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
if [ ${#eslint_files[@]} -gt 0 ]; then
|
||||
echo "::group::ESLint (${#eslint_files[@]} files)"
|
||||
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
exit $status
|
||||
|
||||
- name: Check lint budgets
|
||||
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
|
||||
run: |
|
||||
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
|
||||
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
|
||||
|
|
|
|||
2
.github/workflows/test-unit-proxy-db.yml
vendored
2
.github/workflows/test-unit-proxy-db.yml
vendored
|
|
@ -215,8 +215,10 @@ jobs:
|
|||
tests/proxy_unit_tests/test_models_fallback_endpoint.py
|
||||
tests/proxy_unit_tests/test_google_endpoint_routing.py
|
||||
tests/proxy_unit_tests/test_google_gemini_proxy_request.py
|
||||
tests/proxy_unit_tests/test_gemini_agents_endpoints.py
|
||||
tests/proxy_unit_tests/test_get_favicon.py
|
||||
tests/proxy_unit_tests/test_get_image.py
|
||||
tests/proxy_unit_tests/test_reducto_ocr_route.py
|
||||
tests/proxy_unit_tests/test_ui_path_detection.py
|
||||
tests/proxy_unit_tests/test_prompt_test_endpoint.py
|
||||
tests/proxy_unit_tests/test_check_batch_cost.py
|
||||
|
|
|
|||
17
.github/workflows/test-unit-proxy-endpoints.yml
vendored
17
.github/workflows/test-unit-proxy-endpoints.yml
vendored
|
|
@ -7,6 +7,7 @@ on:
|
|||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -32,13 +33,29 @@ jobs:
|
|||
tests/test_litellm/proxy/image_endpoints
|
||||
tests/test_litellm/proxy/vector_store_endpoints
|
||||
tests/test_litellm/proxy/agent_endpoints
|
||||
tests/test_litellm/proxy/a2a
|
||||
tests/test_litellm/proxy/discovery_endpoints
|
||||
tests/test_litellm/proxy/health_endpoints
|
||||
tests/test_litellm/proxy/shutdown
|
||||
tests/test_litellm/proxy/public_endpoints
|
||||
tests/test_litellm/proxy/prompts
|
||||
tests/test_litellm/proxy/rag_endpoints
|
||||
tests/test_litellm/proxy/realtime_endpoints
|
||||
tests/test_litellm/proxy/ui_crud_endpoints
|
||||
tests/test_litellm/proxy/utils
|
||||
workers: 2
|
||||
reruns: 2
|
||||
artifact-name: proxy-endpoints
|
||||
|
||||
# Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its
|
||||
# own job (not a path on the proxy-endpoints job above) so its budget
|
||||
# is independent and its coverage artifact is uploaded separately.
|
||||
# See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc
|
||||
proxy-server:
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: tests/test_litellm/proxy/proxy_server
|
||||
workers: 4
|
||||
reruns: 2
|
||||
timeout-minutes: 60
|
||||
artifact-name: proxy-server
|
||||
|
|
|
|||
34
.github/workflows/test-unit-proxy-mgmt-behavior.yml
vendored
Normal file
34
.github/workflows/test-unit-proxy-mgmt-behavior.yml
vendored
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
name: "Unit Tests: Proxy Management-Endpoint Behavior Pinning"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_branch
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
proxy-mgmt-behavior:
|
||||
uses: ./.github/workflows/_test-unit-services-base.yml
|
||||
with:
|
||||
test-path: tests/proxy_behavior
|
||||
# workers=0 (no xdist): the world seed is a single shared Postgres
|
||||
# state — two xdist workers both call seed_world() and race on the
|
||||
# ``behavior-pin-budget`` row, producing UniqueViolation + cascading
|
||||
# missing-membership FK failures. The whole suite is ~7s sequentially,
|
||||
# so the cost of disabling parallelism here is negligible.
|
||||
workers: 0
|
||||
reruns: 0
|
||||
enable-postgres: true
|
||||
artifact-name: proxy-mgmt-behavior
|
||||
timeout-minutes: 15
|
||||
25
.github/workflows/test_server_root_path.yml
vendored
25
.github/workflows/test_server_root_path.yml
vendored
|
|
@ -101,6 +101,31 @@ jobs:
|
|||
docker logs litellm-test
|
||||
exit 1
|
||||
|
||||
- name: Setup Node for Playwright
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install UI deps and Chromium
|
||||
working-directory: ui/litellm-dashboard
|
||||
run: |
|
||||
npm ci
|
||||
npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run SERVER_ROOT_PATH redirect e2e
|
||||
working-directory: ui/litellm-dashboard
|
||||
env:
|
||||
SERVER_ROOT_PATH: ${{ matrix.root_path }}
|
||||
run: npx playwright test --config=e2e_tests/serverRootPath.config.ts
|
||||
|
||||
- name: Upload Playwright artifacts on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: playwright-trace-${{ strategy.job-index }}
|
||||
path: ui/litellm-dashboard/test-results/
|
||||
retention-days: 7
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
|
|
|
|||
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -28,6 +28,8 @@ litellm/tests/config_*.yaml
|
|||
litellm/tests/langfuse.log
|
||||
langfuse.log
|
||||
.langfuse.log
|
||||
.pin_list.txt
|
||||
.cov_new.xml
|
||||
litellm/tests/test_custom_logger.py
|
||||
litellm/tests/langfuse.log
|
||||
litellm/tests/dynamo*.log
|
||||
|
|
@ -120,4 +122,5 @@ crash.log
|
|||
crash.*.log
|
||||
# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
|
||||
# and should be committed.
|
||||
.vscode
|
||||
.vscode
|
||||
.pin_list.txt
|
||||
|
|
|
|||
294
AGENTS.md
294
AGENTS.md
|
|
@ -1,293 +1 @@
|
|||
# INSTRUCTIONS FOR LITELLM
|
||||
|
||||
This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
|
||||
|
||||
## OVERVIEW
|
||||
|
||||
LiteLLM is a unified interface for 100+ LLMs that:
|
||||
- Translates inputs to provider-specific completion, embedding, and image generation endpoints
|
||||
- Provides consistent OpenAI-format output across all providers
|
||||
- Includes retry/fallback logic across multiple deployments (Router)
|
||||
- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
|
||||
- Supports advanced features like function calling, streaming, caching, and observability
|
||||
|
||||
## REPOSITORY STRUCTURE
|
||||
|
||||
### Core Components
|
||||
- `litellm/` - Main library code
|
||||
- `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
|
||||
- `proxy/` - Proxy server implementation (LLM Gateway)
|
||||
- `router_utils/` - Load balancing and fallback logic
|
||||
- `types/` - Type definitions and schemas
|
||||
- `integrations/` - Third-party integrations (observability, caching, etc.)
|
||||
|
||||
### Key Directories
|
||||
- `tests/` - Comprehensive test suites
|
||||
- `ui/litellm-dashboard/` - Admin dashboard UI
|
||||
- `enterprise/` - Enterprise-specific features
|
||||
|
||||
Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai).
|
||||
|
||||
## DEVELOPMENT GUIDELINES
|
||||
|
||||
### MAKING CODE CHANGES
|
||||
|
||||
1. **Provider Implementations**: When adding/modifying LLM providers:
|
||||
- Follow existing patterns in `litellm/llms/{provider}/`
|
||||
- Implement proper transformation classes that inherit from `BaseConfig`
|
||||
- Support both sync and async operations
|
||||
- Handle streaming responses appropriately
|
||||
- Include proper error handling with provider-specific exceptions
|
||||
|
||||
2. **Type Safety**:
|
||||
- Use proper type hints throughout
|
||||
- Update type definitions in `litellm/types/`
|
||||
- Ensure compatibility with both Pydantic v1 and v2
|
||||
|
||||
3. **Testing**:
|
||||
- Add tests in appropriate `tests/` subdirectories
|
||||
- Include both unit tests and integration tests
|
||||
- Test provider-specific functionality thoroughly
|
||||
- Consider adding load tests for performance-critical changes
|
||||
|
||||
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
|
||||
|
||||
1. **Always use `antd` for new UI components — Tremor is DEPRECATED**
|
||||
- We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file.
|
||||
- Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
|
||||
- The only exception is the Tremor Table component and its required Tremor Table sub components.
|
||||
|
||||
2. **Use Common Components as much as possible**:
|
||||
- These are usually defined in the `common_components` directory
|
||||
- Use these components as much as possible and avoid building new components unless needed
|
||||
|
||||
3. **Testing**:
|
||||
- The codebase uses **Vitest** and **React Testing Library**
|
||||
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
|
||||
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
|
||||
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
|
||||
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
|
||||
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
|
||||
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
|
||||
- **Structure tests properly**:
|
||||
- First test should verify the component renders successfully
|
||||
- Subsequent tests should focus on functionality and user interactions
|
||||
- Use `waitFor` for async operations that aren't already awaited
|
||||
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
|
||||
|
||||
### IMPORTANT PATTERNS
|
||||
|
||||
1. **Function/Tool Calling**:
|
||||
- LiteLLM standardizes tool calling across providers
|
||||
- OpenAI format is the standard, with transformations for other providers
|
||||
- See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
|
||||
|
||||
2. **Streaming**:
|
||||
- All providers should support streaming where possible
|
||||
- Use consistent chunk formatting across providers
|
||||
- Handle both sync and async streaming
|
||||
|
||||
3. **Error Handling**:
|
||||
- Use provider-specific exception classes
|
||||
- Maintain consistent error formats across providers
|
||||
- Include proper retry logic and fallback mechanisms
|
||||
|
||||
4. **Configuration**:
|
||||
- Support both environment variables and programmatic configuration
|
||||
- Use `BaseConfig` classes for provider configurations
|
||||
- Allow dynamic parameter passing
|
||||
|
||||
## PROXY SERVER (LLM GATEWAY)
|
||||
|
||||
The proxy server is a critical component that provides:
|
||||
- Authentication and authorization
|
||||
- Rate limiting and budget management
|
||||
- Load balancing across multiple models/deployments
|
||||
- Observability and logging
|
||||
- Admin dashboard UI
|
||||
- Enterprise features
|
||||
|
||||
Key files:
|
||||
- `litellm/proxy/proxy_server.py` - Main server implementation
|
||||
- `litellm/proxy/auth/` - Authentication logic
|
||||
- `litellm/proxy/management_endpoints/` - Admin API endpoints
|
||||
|
||||
**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
|
||||
|
||||
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
|
||||
|
||||
LiteLLM supports MCP for agent workflows:
|
||||
- MCP server integration for tool calling
|
||||
- Transformation between OpenAI and MCP tool formats
|
||||
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
|
||||
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
|
||||
|
||||
## RUNNING SCRIPTS
|
||||
|
||||
Use `uv run python script.py` to run Python scripts in the project environment (for non-test files).
|
||||
|
||||
## GITHUB TEMPLATES
|
||||
|
||||
When opening issues or pull requests, follow these templates:
|
||||
|
||||
### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
|
||||
- Describe what happened vs. expected behavior
|
||||
- Include relevant log output
|
||||
- Specify LiteLLM version
|
||||
- Indicate if you're part of an ML Ops team (helps with prioritization)
|
||||
|
||||
### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
|
||||
- Clearly describe the feature
|
||||
- Explain motivation and use case with concrete examples
|
||||
|
||||
### Pull Requests (`.github/pull_request_template.md`)
|
||||
- Add at least 1 test in `tests/litellm/`
|
||||
- Ensure `make test-unit` passes
|
||||
|
||||
|
||||
## TESTING CONSIDERATIONS
|
||||
|
||||
1. **Provider Tests**: Test against real provider APIs when possible
|
||||
2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
|
||||
3. **Performance Tests**: Load testing for high-throughput scenarios
|
||||
4. **Integration Tests**: End-to-end workflows including tool calling
|
||||
|
||||
## DOCUMENTATION
|
||||
|
||||
- Keep documentation in sync with code changes
|
||||
- Update provider documentation when adding new providers
|
||||
- Include code examples for new features
|
||||
- Update changelog and release notes
|
||||
|
||||
## SECURITY CONSIDERATIONS
|
||||
|
||||
- Handle API keys securely
|
||||
- Validate all inputs, especially for proxy endpoints
|
||||
- Consider rate limiting and abuse prevention
|
||||
- Follow security best practices for authentication
|
||||
|
||||
## ENTERPRISE FEATURES
|
||||
|
||||
- Some features are enterprise-only
|
||||
- Check `enterprise/` directory for enterprise-specific code
|
||||
- Maintain compatibility between open-source and enterprise versions
|
||||
|
||||
## COMMON PITFALLS TO AVOID
|
||||
|
||||
1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
|
||||
2. **Provider Specifics**: Each provider has unique quirks - handle them properly
|
||||
3. **Rate Limits**: Respect provider rate limits in tests
|
||||
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
|
||||
5. **Dependencies**: Keep dependencies minimal and well-justified
|
||||
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
|
||||
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
|
||||
8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
|
||||
|
||||
8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
|
||||
|
||||
**Example of BAD** (hardcoded model checks):
|
||||
|
||||
```python
|
||||
@staticmethod
|
||||
def _is_effort_supported_model(model: str) -> bool:
|
||||
"""Check if the model supports the output_config.effort parameter..."""
|
||||
model_lower = model.lower()
|
||||
if AnthropicConfig._is_claude_4_6_model(model):
|
||||
return True
|
||||
return any(
|
||||
v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
|
||||
)
|
||||
```
|
||||
|
||||
**Example of GOOD** (config-driven or helper that reads from config):
|
||||
|
||||
```python
|
||||
if (
|
||||
"claude-3-7-sonnet" in model
|
||||
or AnthropicConfig._is_claude_4_6_model(model)
|
||||
or supports_reasoning(
|
||||
model=model,
|
||||
custom_llm_provider=self.custom_llm_provider,
|
||||
)
|
||||
):
|
||||
...
|
||||
```
|
||||
|
||||
Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
|
||||
|
||||
9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
|
||||
|
||||
## HELPFUL RESOURCES
|
||||
|
||||
- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs))
|
||||
- Provider-specific docs: https://docs.litellm.ai/docs/providers/
|
||||
- Admin UI for testing proxy features
|
||||
|
||||
## WHEN IN DOUBT
|
||||
|
||||
- Follow existing patterns in the codebase
|
||||
- Check similar provider implementations
|
||||
- Ensure comprehensive test coverage
|
||||
- Update documentation appropriately
|
||||
- Consider backward compatibility impact
|
||||
|
||||
## Cursor Cloud specific instructions
|
||||
|
||||
### Environment
|
||||
|
||||
- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
|
||||
- Python 3.12, Node 22 are pre-installed.
|
||||
- The project virtual environment lives under `.venv/`.
|
||||
|
||||
### Running the proxy server
|
||||
|
||||
Create a minimal config file and start the proxy:
|
||||
|
||||
```yaml
|
||||
# config.yaml
|
||||
model_list:
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/fake-model
|
||||
api_key: fake-key
|
||||
api_base: https://fake-api.example.com
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
|
||||
litellm_settings:
|
||||
drop_params: True
|
||||
telemetry: False
|
||||
```
|
||||
|
||||
```bash
|
||||
uv run litellm --config config.yaml --port 4000
|
||||
```
|
||||
|
||||
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
|
||||
|
||||
### Running tests
|
||||
|
||||
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
|
||||
|
||||
- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow.
|
||||
- The `--timeout` pytest flag is NOT available; don't pass it.
|
||||
- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4`
|
||||
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
|
||||
- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry.
|
||||
|
||||
### Lint
|
||||
|
||||
```bash
|
||||
cd litellm && uv run ruff check .
|
||||
```
|
||||
|
||||
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
|
||||
|
||||
### UI Dashboard development
|
||||
|
||||
- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
|
||||
- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
|
||||
- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
|
||||
- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
|
||||
- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`
|
||||
Read @CLAUDE.md for coding guidelines
|
||||
|
|
|
|||
202
CLAUDE.md
202
CLAUDE.md
|
|
@ -1,181 +1,75 @@
|
|||
# CLAUDE.md
|
||||
Do not write comments unless they are absolutely necessary to explain some very complex business logic. Please clean up if there are comments that are not absolutely necessary. Do not remove comments that are unrelated to the addition of the code of this PR
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
Explanation: code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive to the reader, while being both easy to maintain and high performance
|
||||
|
||||
## Documentation
|
||||
Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in:
|
||||
- correct
|
||||
- secure
|
||||
- performant
|
||||
- readable
|
||||
- easy to maintain/change
|
||||
- modern
|
||||
|
||||
Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead.
|
||||
In that order of importance
|
||||
|
||||
## Development Commands
|
||||
When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate
|
||||
|
||||
### Installation
|
||||
- `make install-dev` - Install core development dependencies
|
||||
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
|
||||
- `make install-test-deps` - Install the full local test environment and generate the Prisma client
|
||||
Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
|
||||
|
||||
### Testing
|
||||
- `make test` - Run all tests
|
||||
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
|
||||
- `make test-integration` - Run integration tests (excludes unit tests)
|
||||
- `pytest tests/` - Direct pytest execution
|
||||
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
### Code Quality
|
||||
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
|
||||
- `make format` - Apply Black code formatting
|
||||
- `make lint-ruff` - Run Ruff linting only
|
||||
- `make lint-mypy` - Run MyPy type checking only
|
||||
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
|
||||
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
|
||||
|
||||
### Single Test Files
|
||||
- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
|
||||
- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
|
||||
Always use @.github/pull_request_template.md as a guide for your PR body
|
||||
|
||||
### Running Scripts
|
||||
- `uv run python script.py` - Run Python scripts (use for non-test files)
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
|
||||
|
||||
### GitHub Issue & PR Templates
|
||||
When contributing to the project, use the appropriate templates:
|
||||
If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
- don't use "—". Instead, reach for ";", ".", etc.
|
||||
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
|
||||
- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
|
||||
- don't add a trailing "." at the end of paragraphs (just like this file)
|
||||
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
|
||||
|
||||
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
|
||||
- Describe what happened vs. what you expected
|
||||
- Include relevant log output
|
||||
- Specify your LiteLLM version
|
||||
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
|
||||
|
||||
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
|
||||
- Describe the feature clearly
|
||||
- Explain the motivation and use case
|
||||
Run tests, format your code, and lint your code before each commit
|
||||
|
||||
**Pull Requests** (`.github/pull_request_template.md`):
|
||||
- Add at least 1 test in `tests/litellm/`
|
||||
- Ensure `make test-unit` passes
|
||||
Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it)
|
||||
|
||||
## Architecture Overview
|
||||
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
|
||||
|
||||
LiteLLM is a unified interface for 100+ LLM providers with two main components:
|
||||
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
|
||||
|
||||
### Core Library (`litellm/`)
|
||||
- **Main entry point**: `litellm/main.py` - Contains core completion() function
|
||||
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
|
||||
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
|
||||
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
|
||||
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
|
||||
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
|
||||
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
|
||||
|
||||
### Proxy Server (`litellm/proxy/`)
|
||||
- **Main server**: `proxy_server.py` - FastAPI application
|
||||
- **Authentication**: `auth/` - API key management, JWT, OAuth2
|
||||
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
|
||||
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
|
||||
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
|
||||
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
|
||||
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
|
||||
When working on a PR, keep the PR description in sync with new commits being made
|
||||
|
||||
## Key Patterns
|
||||
Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in
|
||||
|
||||
### Provider Implementation
|
||||
- Providers inherit from base classes in `litellm/llms/base.py`
|
||||
- Each provider has transformation functions for input/output formatting
|
||||
- Support both sync and async operations
|
||||
- Handle streaming responses and function calling
|
||||
Do not put names of customers or customer company names in code, PRs, and issues. The codebase is public
|
||||
|
||||
### Error Handling
|
||||
- Provider-specific exceptions mapped to OpenAI-compatible errors
|
||||
- Fallback logic handled by Router system
|
||||
- Comprehensive logging through `litellm/_logging.py`
|
||||
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
|
||||
|
||||
### Configuration
|
||||
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
|
||||
- Environment variables for API keys and settings
|
||||
- Database schema managed via Prisma (`proxy/schema.prisma`)
|
||||
## Think Before Coding
|
||||
|
||||
## Development Notes
|
||||
**Don't assume. Don't hide confusion. Surface tradeoffs**
|
||||
|
||||
### Code Style
|
||||
- Uses Black formatter, Ruff linter, MyPy type checker
|
||||
- Pydantic v2 for data validation
|
||||
- Async/await patterns throughout
|
||||
- Type hints required for all public APIs
|
||||
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
|
||||
- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
|
||||
- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
|
||||
- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
|
||||
- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
|
||||
Before implementing:
|
||||
- State your assumptions explicitly. If uncertain, ask
|
||||
- If multiple interpretations exist, present them. Don't pick silently
|
||||
- If a simpler approach exists, say so. Push back when warranted
|
||||
- If something is unclear, stop. Name what's confusing. Ask
|
||||
|
||||
### Testing Strategy
|
||||
- Unit tests in `tests/test_litellm/`
|
||||
- Integration tests for each provider in `tests/llm_translation/`
|
||||
- Proxy tests in `tests/proxy_unit_tests/`
|
||||
- Load tests in `tests/load_tests/`
|
||||
- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
|
||||
- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
|
||||
- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
|
||||
## Simplicity First
|
||||
|
||||
### UI / Backend Consistency
|
||||
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
|
||||
**Minimum code that solves the problem. Nothing speculative**
|
||||
|
||||
### UI Component Library
|
||||
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
|
||||
- No features beyond what was asked
|
||||
- No abstractions for single-use code
|
||||
- No "flexibility" or "configurability" that wasn't requested
|
||||
- No error handling for impossible scenarios
|
||||
- If you write 200 lines and it could be 50, rewrite it
|
||||
|
||||
### MCP OAuth / OpenAPI Transport Mapping
|
||||
- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
|
||||
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
|
||||
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
|
||||
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
|
||||
- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
|
||||
|
||||
### MCP Credential Storage
|
||||
- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
|
||||
- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
|
||||
- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
|
||||
- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
|
||||
|
||||
### Browser Storage Safety (UI)
|
||||
- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
|
||||
- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
|
||||
|
||||
### Database Migrations
|
||||
- Prisma handles schema migrations
|
||||
- Migration files auto-generated with `prisma migrate dev`
|
||||
- Always test migrations against both PostgreSQL and SQLite
|
||||
|
||||
### Proxy database access
|
||||
- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
|
||||
- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
|
||||
- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
|
||||
- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
|
||||
- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
|
||||
- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
|
||||
- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
|
||||
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
|
||||
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
|
||||
|
||||
### Setup Wizard (`litellm/setup_wizard.py`)
|
||||
- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
|
||||
- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
|
||||
- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
|
||||
|
||||
### Enterprise Features
|
||||
- Enterprise-specific code in `enterprise/` directory
|
||||
- Optional features enabled via environment variables
|
||||
- Separate licensing and authentication for enterprise features
|
||||
|
||||
### CI Supply-Chain Safety
|
||||
- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install.
|
||||
- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you.
|
||||
- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest.
|
||||
- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_<tool>` or `- wait_for_service`.
|
||||
- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it.
|
||||
- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions.
|
||||
|
||||
### HTTP Client Cache Safety
|
||||
- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
|
||||
|
||||
### Troubleshooting: DB schema out of sync after proxy restart
|
||||
`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
|
||||
|
||||
**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
|
||||
|
||||
**Fix options:**
|
||||
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
|
||||
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
|
||||
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
|
||||
|
|
|
|||
109
GEMINI.md
109
GEMINI.md
|
|
@ -1,108 +1 @@
|
|||
# GEMINI.md
|
||||
|
||||
This file provides guidance to Gemini when working with code in this repository.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Installation
|
||||
- `make install-dev` - Install core development dependencies
|
||||
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
|
||||
- `make install-test-deps` - Install all test dependencies
|
||||
|
||||
### Testing
|
||||
- `make test` - Run all tests
|
||||
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
|
||||
- `make test-integration` - Run integration tests (excludes unit tests)
|
||||
- `pytest tests/` - Direct pytest execution
|
||||
|
||||
### Code Quality
|
||||
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
|
||||
- `make format` - Apply Black code formatting
|
||||
- `make lint-ruff` - Run Ruff linting only
|
||||
- `make lint-mypy` - Run MyPy type checking only
|
||||
|
||||
### Single Test Files
|
||||
- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
|
||||
- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
|
||||
|
||||
### Running Scripts
|
||||
- `uv run python script.py` - Run Python scripts (use for non-test files)
|
||||
|
||||
### GitHub Issue & PR Templates
|
||||
When contributing to the project, use the appropriate templates:
|
||||
|
||||
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
|
||||
- Describe what happened vs. what you expected
|
||||
- Include relevant log output
|
||||
- Specify your LiteLLM version
|
||||
|
||||
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
|
||||
- Describe the feature clearly
|
||||
- Explain the motivation and use case
|
||||
|
||||
**Pull Requests** (`.github/pull_request_template.md`):
|
||||
- Add at least 1 test in `tests/litellm/`
|
||||
- Ensure `make test-unit` passes
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
LiteLLM is a unified interface for 100+ LLM providers with two main components:
|
||||
|
||||
### Core Library (`litellm/`)
|
||||
- **Main entry point**: `litellm/main.py` - Contains core completion() function
|
||||
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
|
||||
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
|
||||
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
|
||||
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
|
||||
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
|
||||
|
||||
### Proxy Server (`litellm/proxy/`)
|
||||
- **Main server**: `proxy_server.py` - FastAPI application
|
||||
- **Authentication**: `auth/` - API key management, JWT, OAuth2
|
||||
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
|
||||
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
|
||||
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
|
||||
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
|
||||
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
|
||||
|
||||
## Key Patterns
|
||||
|
||||
### Provider Implementation
|
||||
- Providers inherit from base classes in `litellm/llms/base.py`
|
||||
- Each provider has transformation functions for input/output formatting
|
||||
- Support both sync and async operations
|
||||
- Handle streaming responses and function calling
|
||||
|
||||
### Error Handling
|
||||
- Provider-specific exceptions mapped to OpenAI-compatible errors
|
||||
- Fallback logic handled by Router system
|
||||
- Comprehensive logging through `litellm/_logging.py`
|
||||
|
||||
### Configuration
|
||||
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
|
||||
- Environment variables for API keys and settings
|
||||
- Database schema managed via Prisma (`proxy/schema.prisma`)
|
||||
|
||||
## Development Notes
|
||||
|
||||
### Code Style
|
||||
- Uses Black formatter, Ruff linter, MyPy type checker
|
||||
- Pydantic v2 for data validation
|
||||
- Async/await patterns throughout
|
||||
- Type hints required for all public APIs
|
||||
|
||||
### Testing Strategy
|
||||
- Unit tests in `tests/test_litellm/`
|
||||
- Integration tests for each provider in `tests/llm_translation/`
|
||||
- Proxy tests in `tests/proxy_unit_tests/`
|
||||
- Load tests in `tests/load_tests/`
|
||||
|
||||
### Database Migrations
|
||||
- Prisma handles schema migrations
|
||||
- Migration files auto-generated with `prisma migrate dev`
|
||||
- Always test migrations against both PostgreSQL and SQLite
|
||||
|
||||
### Enterprise Features
|
||||
- Enterprise-specific code in `enterprise/` directory
|
||||
- Optional features enabled via environment variables
|
||||
- Separate licensing and authentication for enterprise features
|
||||
Read @CLAUDE.md for coding guidelines
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -146,7 +146,7 @@ test-unit-proxy-core: install-test-deps
|
|||
$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-proxy-misc: install-test-deps
|
||||
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
|
||||
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
|
||||
|
||||
test-unit-integrations: install-test-deps
|
||||
$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@
|
|||
</a>
|
||||
</h4>
|
||||
|
||||
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
|
||||
<img alt="LiteLLM AI Gateway" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -12,17 +12,27 @@ USER root
|
|||
|
||||
COPY --from=uvbin /uv /uvx /usr/local/bin/
|
||||
|
||||
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
|
||||
# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE
|
||||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
||||
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
|
||||
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
|
||||
# BuildKit cache mount (different filesystem).
|
||||
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
|
||||
# silently pulling a managed interpreter.
|
||||
# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't
|
||||
# silently re-enable nodeenv's Node download.
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_PYTHON_DOWNLOADS=0 \
|
||||
PRISMA_USE_GLOBAL_NODE=true \
|
||||
PATH="/app/.venv/bin:${PATH}"
|
||||
|
||||
# Stage 1 — install dependencies only.
|
||||
|
|
@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
|
||||
USER root
|
||||
|
||||
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
||||
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
|
||||
# /home/nonroot. We run the backend as that user
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
|
||||
|
||||
go 1.25.1
|
||||
go 1.26.3
|
||||
|
||||
require github.com/fugue-labs/gollem v0.1.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ version: 1.1.0
|
|||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: v1.80.12
|
||||
appVersion: v1.85.1
|
||||
|
||||
annotations:
|
||||
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ spec:
|
|||
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
|
||||
{{- end }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- tpl (toYaml .) $ | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "litellm.labels" . | nindent 8 }}
|
||||
|
|
@ -53,7 +53,7 @@ spec:
|
|||
- name: {{ include "litellm.name" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
env:
|
||||
- name: HOST
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ spec:
|
|||
{{- end }}
|
||||
containers:
|
||||
- name: prisma-migrations
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 12 }}
|
||||
|
|
|
|||
|
|
@ -377,3 +377,28 @@ tests:
|
|||
content:
|
||||
name: sidecar-tpl
|
||||
image: "ghcr.io/berriai/litellm-database:test"
|
||||
- it: should support tpl in podAnnotations
|
||||
template: deployment.yaml
|
||||
set:
|
||||
image:
|
||||
repository: ghcr.io/berriai/litellm-database
|
||||
tag: test
|
||||
# Mirrors the real-world scenario this feature unblocks:
|
||||
# user disables the built-in ConfigMap (and its built-in checksum/config
|
||||
# annotation) and re-implements checksum/config themselves via tpl.
|
||||
proxyConfigMap:
|
||||
create: false
|
||||
podAnnotations:
|
||||
checksum/config: "{{ .Values.image.tag }}"
|
||||
example.com/some-key: "{{ .Values.image.repository }}"
|
||||
example.com/literal: "plain-string-value"
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.metadata.annotations["checksum/config"]
|
||||
value: "test"
|
||||
- equal:
|
||||
path: spec.template.metadata.annotations["example.com/some-key"]
|
||||
value: "ghcr.io/berriai/litellm-database"
|
||||
- equal:
|
||||
path: spec.template.metadata.annotations["example.com/literal"]
|
||||
value: "plain-string-value"
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ image:
|
|||
repository: ghcr.io/berriai/litellm-database
|
||||
pullPolicy: Always
|
||||
# Overrides the image tag whose default is the chart appVersion.
|
||||
# tag: "main-latest"
|
||||
# tag: "latest"
|
||||
tag: ""
|
||||
|
||||
imagePullSecrets: []
|
||||
|
|
@ -285,11 +285,31 @@ db:
|
|||
deployStandalone: true
|
||||
|
||||
# Lifecycle hooks for the LiteLLM container
|
||||
#
|
||||
# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks
|
||||
# the pod NotReady and blocks only until in-flight requests actually finish
|
||||
# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always
|
||||
# waiting the worst-case duration. The drain runs once (the preStop hook and
|
||||
# the SIGTERM handler share it), so set terminationGracePeriodSeconds a few
|
||||
# seconds above GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before
|
||||
# SIGKILL.
|
||||
#
|
||||
# /health/drain is off by default; enable it with
|
||||
# general_settings.enable_drain_endpoint: true. The kubelet calls preStop
|
||||
# hooks without proxy credentials, so when the health port is reachable from
|
||||
# other pods (the common case) also set
|
||||
# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env
|
||||
# var) and send the same value on the X-Drain-Token header from the hook.
|
||||
# Calls missing/wrong the token get a 401 and have no side effect.
|
||||
# Example:
|
||||
# lifecycle:
|
||||
# preStop:
|
||||
# exec:
|
||||
# command: ["/bin/sh", "-c", "sleep 10"]
|
||||
# httpGet:
|
||||
# path: /health/drain
|
||||
# port: 4000
|
||||
# httpHeaders:
|
||||
# - name: X-Drain-Token
|
||||
# value: <same value as drain_endpoint_token>
|
||||
lifecycle: {}
|
||||
|
||||
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ RUN for i in 1 2 3; do \
|
|||
curl \
|
||||
openssl \
|
||||
libsndfile \
|
||||
nodejs && break || sleep 5; \
|
||||
nodejs \
|
||||
npm && break || sleep 5; \
|
||||
done
|
||||
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
|
|
@ -54,22 +55,10 @@ COPY . .
|
|||
# Set non-root flag for build time consistency
|
||||
ENV LITELLM_NON_ROOT=true
|
||||
|
||||
# Stage the pre-built Admin UI from the checked-in Next.js static export.
|
||||
# _experimental/out/ is regenerated as part of the release runbook.
|
||||
# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
|
||||
# proxy_server.py expects, and drop a readiness marker.
|
||||
RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
|
||||
cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
|
||||
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
|
||||
( cd /var/lib/litellm/ui && \
|
||||
for html_file in *.html; do \
|
||||
if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
|
||||
folder_name="${html_file%.html}" && \
|
||||
mkdir -p "$folder_name" && \
|
||||
mv "$html_file" "$folder_name/index.html"; \
|
||||
fi; \
|
||||
done && \
|
||||
touch .litellm_ui_ready )
|
||||
touch /var/lib/litellm/ui/.litellm_ui_ready
|
||||
|
||||
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
|
||||
if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
|
||||
|
|
|
|||
|
|
@ -19,12 +19,26 @@ RESEND_API_ENDPOINT = "https://api.resend.com/emails"
|
|||
|
||||
|
||||
class ResendEmailLogger(BaseEmailLogger):
|
||||
"""
|
||||
Send emails using Resend's API.
|
||||
|
||||
Required env vars:
|
||||
- RESEND_API_KEY
|
||||
|
||||
Optional env vars:
|
||||
- RESEND_FROM_EMAIL: Override the default sender address. Must be on a
|
||||
domain verified in your Resend account. When unset, falls back to the
|
||||
`from_email` argument passed by the caller (which defaults to
|
||||
`notifications@alerts.litellm.ai` and only works on LiteLLM Cloud).
|
||||
"""
|
||||
|
||||
def __init__(self, internal_usage_cache=None, **kwargs):
|
||||
super().__init__(internal_usage_cache=internal_usage_cache, **kwargs)
|
||||
self.async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
self.resend_api_key = os.getenv("RESEND_API_KEY")
|
||||
self.resend_from_email = os.getenv("RESEND_FROM_EMAIL")
|
||||
|
||||
async def send_email(
|
||||
self,
|
||||
|
|
@ -33,13 +47,14 @@ class ResendEmailLogger(BaseEmailLogger):
|
|||
subject: str,
|
||||
html_body: str,
|
||||
):
|
||||
sender_email = self.resend_from_email or from_email
|
||||
verbose_logger.debug(
|
||||
f"Sending email from {from_email} to {to_email} with subject {subject}"
|
||||
f"Sending email from {sender_email} to {to_email} with subject {subject}"
|
||||
)
|
||||
response = await self.async_httpx_client.post(
|
||||
url=RESEND_API_ENDPOINT,
|
||||
json={
|
||||
"from": from_email,
|
||||
"from": sender_email,
|
||||
"to": to_email,
|
||||
"subject": subject,
|
||||
"html": html_body,
|
||||
|
|
|
|||
|
|
@ -658,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
if isinstance(content, str):
|
||||
continue
|
||||
for c in content:
|
||||
if c["type"] == "file":
|
||||
if c.get("type") == "file":
|
||||
file_object = cast(ChatCompletionFileObject, c)
|
||||
file_object_file_field = file_object["file"]
|
||||
file_id = file_object_file_field.get("file_id")
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -12,17 +12,27 @@ USER root
|
|||
|
||||
COPY --from=uvbin /uv /uvx /usr/local/bin/
|
||||
|
||||
RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
|
||||
# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE
|
||||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
||||
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
|
||||
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
|
||||
# BuildKit cache mount (different filesystem).
|
||||
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
|
||||
# silently pulling a managed interpreter.
|
||||
# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't
|
||||
# silently re-enable nodeenv's Node download.
|
||||
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_PYTHON_DOWNLOADS=0 \
|
||||
PRISMA_USE_GLOBAL_NODE=true \
|
||||
PATH="/app/.venv/bin:${PATH}"
|
||||
|
||||
# Stage 1 — install dependencies only.
|
||||
|
|
@ -58,7 +68,11 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
|
||||
USER root
|
||||
|
||||
RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
||||
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
|
||||
# /home/nonroot. We run the proxy as that user.
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
# Health & ops
|
||||
"/health",
|
||||
"/metrics",
|
||||
"/watsonx"
|
||||
)
|
||||
|
||||
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
||||
|
|
|
|||
|
|
@ -56,16 +56,34 @@ app.kubernetes.io/component: ui
|
|||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Shared ServiceAccount name used by all three component Deployments. When
|
||||
`serviceAccount.create` is true and `serviceAccount.name` is empty, default
|
||||
to the chart fullname. When `create` is false, fall back to the provided
|
||||
name or the namespace's `default` SA.
|
||||
Per-component ServiceAccount name helpers.
|
||||
|
||||
Each component (gateway, backend, ui) has its own SA config under
|
||||
.Values.serviceAccounts.<component>. When `create` is true and `name` is
|
||||
empty the chart defaults to "<release>-litellm-<component>". When `create`
|
||||
is false the chart uses the provided name, or the namespace `default` SA.
|
||||
*/}}
|
||||
{{- define "litellm.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- define "litellm.gateway.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccounts.gateway.create -}}
|
||||
{{ default (include "litellm.gateway.fullname" .) .Values.serviceAccounts.gateway.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccount.name }}
|
||||
{{ default "default" .Values.serviceAccounts.gateway.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.backend.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccounts.backend.create -}}
|
||||
{{ default (include "litellm.backend.fullname" .) .Values.serviceAccounts.backend.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccounts.backend.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.ui.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccounts.ui.create -}}
|
||||
{{ default (include "litellm.ui.fullname" .) .Values.serviceAccounts.ui.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccounts.ui.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@ spec:
|
|||
labels:
|
||||
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ spec:
|
|||
labels:
|
||||
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ spec:
|
|||
app.kubernetes.io/component: migrations
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,51 @@
|
|||
{{- if .Values.serviceAccount.create -}}
|
||||
{{- $prev := false -}}
|
||||
{{- if .Values.serviceAccounts.gateway.create -}}
|
||||
{{- $prev = true }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "litellm.serviceAccountName" . }}
|
||||
name: {{ include "litellm.gateway.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
app.kubernetes.io/component: gateway
|
||||
{{- with .Values.serviceAccounts.gateway.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
|
||||
{{- end }}
|
||||
{{- if .Values.serviceAccounts.backend.create }}
|
||||
{{- if $prev }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- $prev = true }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "litellm.backend.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: backend
|
||||
{{- with .Values.serviceAccounts.backend.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
|
||||
{{- end }}
|
||||
{{- if .Values.serviceAccounts.ui.create }}
|
||||
{{- if $prev }}
|
||||
---
|
||||
{{- end }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "litellm.ui.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: ui
|
||||
{{- with .Values.serviceAccounts.ui.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@ spec:
|
|||
labels:
|
||||
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
|
||||
serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }}
|
||||
automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
|
|
@ -14,16 +14,33 @@ ingress:
|
|||
host: "" # optional; if set, becomes the rule's host
|
||||
tls: []
|
||||
|
||||
# Shared ServiceAccount used by all three component Deployments. Set
|
||||
# `create: true` to have the chart provision it (e.g. when wiring an EKS
|
||||
# Pod Identity association by SA name). Set `name` to use an existing SA
|
||||
# (chart-created or out-of-band). When both are empty / false, pods run
|
||||
# with the namespace's `default` SA.
|
||||
serviceAccount:
|
||||
create: false
|
||||
automount: true
|
||||
annotations: {}
|
||||
name: ""
|
||||
# Per-component ServiceAccounts for gateway, backend, and ui.
|
||||
#
|
||||
# Each section mirrors the old shared serviceAccount shape. Set `create:
|
||||
# true` to have the chart provision the SA (useful for EKS Pod Identity /
|
||||
# GKE Workload Identity annotations). Set `name` to bind an existing SA.
|
||||
# When both are unset the component pod runs with the namespace `default` SA.
|
||||
#
|
||||
# The UI SA deliberately defaults to `automount: false` — the static nginx
|
||||
# container does not need the K8s API and should not carry a projected
|
||||
# ServiceAccount token that a compromised container could use to call the
|
||||
# cloud-provider metadata service or the K8s API.
|
||||
serviceAccounts:
|
||||
gateway:
|
||||
create: false
|
||||
automount: true
|
||||
annotations: {}
|
||||
name: ""
|
||||
backend:
|
||||
create: false
|
||||
automount: true
|
||||
annotations: {}
|
||||
name: ""
|
||||
ui:
|
||||
create: false
|
||||
automount: false
|
||||
annotations: {}
|
||||
name: ""
|
||||
|
||||
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
|
||||
# against the writer database, creating the LiteLLM schema (tables that
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,2 @@
|
|||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "oauth_passthrough" BOOLEAN NOT NULL DEFAULT false;
|
||||
|
|
@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
|
|||
allow_all_keys Boolean @default(false)
|
||||
available_on_public_internet Boolean @default(true)
|
||||
delegate_auth_to_upstream Boolean @default(false)
|
||||
oauth_passthrough Boolean @default(false)
|
||||
is_byok Boolean @default(false)
|
||||
byok_description String[] @default([])
|
||||
byok_api_key_help_url String?
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.72"
|
||||
version = "0.4.73"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.72"
|
||||
version = "0.4.73"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -225,12 +225,22 @@ use_chat_completions_url_for_anthropic_messages: bool = bool(
|
|||
route_all_chat_openai_to_responses: bool = (
|
||||
os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
|
||||
) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
|
||||
# When True, Gemini/Vertex Live setup is deferred until client `session.update`.
|
||||
# Default False preserves historical behavior (auto-send setup on connect).
|
||||
gemini_live_defer_setup: bool = (
|
||||
os.getenv("LITELLM_GEMINI_LIVE_DEFER_SETUP", "false").lower() == "true"
|
||||
)
|
||||
use_legacy_interactions_schema: bool = (
|
||||
os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true"
|
||||
) # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs`
|
||||
# schema instead of the new `steps` schema. Remove this flag after June 8, 2026.
|
||||
retry = True
|
||||
### AUTH ###
|
||||
api_key: Optional[str] = None
|
||||
openai_key: Optional[str] = None
|
||||
groq_key: Optional[str] = None
|
||||
gigachat_key: Optional[str] = None
|
||||
xai_key: Optional[str] = None
|
||||
databricks_key: Optional[str] = None
|
||||
openai_like_key: Optional[str] = None
|
||||
azure_key: Optional[str] = None
|
||||
|
|
@ -268,6 +278,7 @@ ovhcloud_key: Optional[str] = None
|
|||
lemonade_key: Optional[str] = None
|
||||
sap_service_key: Optional[str] = None
|
||||
amazon_nova_api_key: Optional[str] = None
|
||||
inception_key: Optional[str] = None
|
||||
common_cloud_provider_auth_params: dict = {
|
||||
"params": ["project", "region_name", "token"],
|
||||
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
|
||||
|
|
@ -409,6 +420,12 @@ internal_user_budget_duration: Optional[str] = None
|
|||
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
|
||||
max_end_user_budget: Optional[float] = None
|
||||
max_end_user_budget_id: Optional[str] = None
|
||||
# When True, end-user IDs extracted from requests are validated against
|
||||
# LiteLLM_EndUserTable / LiteLLM_UserTable. Values that do not resolve to a
|
||||
# known row are dropped before reaching spend logs. Defaults to False for
|
||||
# backwards compatibility — arbitrary client-supplied identifiers still
|
||||
# pass through unchanged.
|
||||
validate_end_user_id_in_db: bool = False
|
||||
disable_end_user_cost_tracking: Optional[bool] = None
|
||||
disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
||||
enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
||||
|
|
@ -427,6 +444,7 @@ disable_copilot_system_to_assistant: bool = (
|
|||
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
|
||||
)
|
||||
public_mcp_servers: Optional[List[str]] = None
|
||||
public_mcp_hub_strict_whitelist: bool = True
|
||||
public_model_groups: Optional[List[str]] = None
|
||||
public_agent_groups: Optional[List[str]] = None
|
||||
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
|
||||
|
|
@ -535,6 +553,7 @@ cohere_models: Set = set()
|
|||
cohere_chat_models: Set = set()
|
||||
mistral_chat_models: Set = set()
|
||||
text_completion_codestral_models: Set = set()
|
||||
text_completion_inception_models: Set = set()
|
||||
anthropic_models: Set = set()
|
||||
openrouter_models: Set = set()
|
||||
datarobot_models: Set = set()
|
||||
|
|
@ -612,6 +631,7 @@ publicai_models: Set = set()
|
|||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
lambda_ai_models: Set = set()
|
||||
inception_models: Set = set()
|
||||
hyperbolic_models: Set = set()
|
||||
black_forest_labs_models: Set = set()
|
||||
recraft_models: Set = set()
|
||||
|
|
@ -632,6 +652,7 @@ minimax_models: Set = set()
|
|||
aws_polly_models: Set = set()
|
||||
gigachat_models: Set = set()
|
||||
llamagate_models: Set = set()
|
||||
reducto_models: Set = set()
|
||||
bedrock_mantle_models: Set = set()
|
||||
|
||||
|
||||
|
|
@ -775,6 +796,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
|
|||
fireworks_ai_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-codestral":
|
||||
text_completion_codestral_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-inception":
|
||||
text_completion_inception_models.add(key)
|
||||
elif value.get("litellm_provider") == "xai":
|
||||
xai_models.add(key)
|
||||
elif value.get("litellm_provider") == "zai":
|
||||
|
|
@ -861,6 +884,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
|
|||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "inception":
|
||||
inception_models.add(key)
|
||||
elif value.get("litellm_provider") == "hyperbolic":
|
||||
hyperbolic_models.add(key)
|
||||
elif value.get("litellm_provider") == "black_forest_labs":
|
||||
|
|
@ -899,6 +924,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
|
|||
gigachat_models.add(key)
|
||||
elif value.get("litellm_provider") == "llamagate":
|
||||
llamagate_models.add(key)
|
||||
elif value.get("litellm_provider") == "reducto":
|
||||
reducto_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_mantle":
|
||||
bedrock_mantle_models.add(key)
|
||||
|
||||
|
|
@ -961,6 +988,7 @@ model_list = list(
|
|||
| watsonx_models
|
||||
| gemini_models
|
||||
| text_completion_codestral_models
|
||||
| text_completion_inception_models
|
||||
| xai_models
|
||||
| zai_models
|
||||
| fal_ai_models
|
||||
|
|
@ -999,6 +1027,7 @@ model_list = list(
|
|||
| v0_models
|
||||
| morph_models
|
||||
| lambda_ai_models
|
||||
| inception_models
|
||||
| black_forest_labs_models
|
||||
| recraft_models
|
||||
| cometapi_models
|
||||
|
|
@ -1010,6 +1039,7 @@ model_list = list(
|
|||
| ovhcloud_models
|
||||
| lemonade_models
|
||||
| docker_model_runner_models
|
||||
| reducto_models
|
||||
| bedrock_mantle_models
|
||||
| set(clarifai_models)
|
||||
)
|
||||
|
|
@ -1054,6 +1084,7 @@ models_by_provider: dict = {
|
|||
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
|
||||
"aleph_alpha": aleph_alpha_models,
|
||||
"text-completion-codestral": text_completion_codestral_models,
|
||||
"text-completion-inception": text_completion_inception_models,
|
||||
"xai": xai_models,
|
||||
"zai": zai_models,
|
||||
"fal_ai": fal_ai_models,
|
||||
|
|
@ -1098,6 +1129,7 @@ models_by_provider: dict = {
|
|||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
"inception": inception_models,
|
||||
"hyperbolic": hyperbolic_models,
|
||||
"black_forest_labs": black_forest_labs_models,
|
||||
"recraft": recraft_models,
|
||||
|
|
@ -1116,6 +1148,7 @@ models_by_provider: dict = {
|
|||
"aws_polly": aws_polly_models,
|
||||
"gigachat": gigachat_models,
|
||||
"llamagate": llamagate_models,
|
||||
"reducto": reducto_models,
|
||||
"bedrock_mantle": bedrock_mantle_models,
|
||||
}
|
||||
|
||||
|
|
@ -1288,6 +1321,18 @@ from .responses.main import *
|
|||
# Interactions API is available as litellm.interactions module
|
||||
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
||||
from . import interactions
|
||||
from .interactions.agents.main import (
|
||||
acreate as acreate_agent,
|
||||
create as create_agent,
|
||||
alist as alist_agents,
|
||||
list as list_agents,
|
||||
aget as aget_agent,
|
||||
get as get_agent,
|
||||
adelete as adelete_agent,
|
||||
delete as delete_agent,
|
||||
alist_versions as alist_agent_versions,
|
||||
list_versions as list_agent_versions,
|
||||
)
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
|
|
@ -1836,6 +1881,9 @@ if TYPE_CHECKING:
|
|||
from .llms.codestral.completion.transformation import (
|
||||
CodestralTextCompletionConfig as CodestralTextCompletionConfig,
|
||||
)
|
||||
from .llms.inception.completion.transformation import (
|
||||
InceptionTextCompletionConfig as InceptionTextCompletionConfig,
|
||||
)
|
||||
from .llms.azure.azure import (
|
||||
AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig,
|
||||
)
|
||||
|
|
@ -1850,6 +1898,9 @@ if TYPE_CHECKING:
|
|||
from .llms.azure.completion.transformation import (
|
||||
AzureOpenAITextConfig as AzureOpenAITextConfig,
|
||||
)
|
||||
from .llms.azure.audio_transcription.transformation import (
|
||||
AzureSpeechAudioTranscriptionConfig as AzureSpeechAudioTranscriptionConfig,
|
||||
)
|
||||
from .llms.hosted_vllm.chat.transformation import (
|
||||
HostedVLLMChatConfig as HostedVLLMChatConfig,
|
||||
)
|
||||
|
|
@ -1901,6 +1952,9 @@ if TYPE_CHECKING:
|
|||
from .llms.lambda_ai.chat.transformation import (
|
||||
LambdaAIChatConfig as LambdaAIChatConfig,
|
||||
)
|
||||
from .llms.inception.chat.transformation import (
|
||||
InceptionChatConfig as InceptionChatConfig,
|
||||
)
|
||||
from .llms.hyperbolic.chat.transformation import (
|
||||
HyperbolicChatConfig as HyperbolicChatConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -267,12 +267,14 @@ LLM_CONFIG_NAMES = (
|
|||
"AIMLChatConfig",
|
||||
"VolcEngineChatConfig",
|
||||
"CodestralTextCompletionConfig",
|
||||
"InceptionTextCompletionConfig",
|
||||
"AzureOpenAIAssistantsAPIConfig",
|
||||
"HerokuChatConfig",
|
||||
"CometAPIConfig",
|
||||
"AzureOpenAIConfig",
|
||||
"AzureOpenAIGPT5Config",
|
||||
"AzureOpenAITextConfig",
|
||||
"AzureSpeechAudioTranscriptionConfig",
|
||||
"HostedVLLMChatConfig",
|
||||
"HostedVLLMEmbeddingConfig",
|
||||
# Alias for backwards compatibility
|
||||
|
|
@ -309,6 +311,7 @@ LLM_CONFIG_NAMES = (
|
|||
"MorphChatConfig",
|
||||
"RAGFlowConfig",
|
||||
"LambdaAIChatConfig",
|
||||
"InceptionChatConfig",
|
||||
"HyperbolicChatConfig",
|
||||
"VercelAIGatewayConfig",
|
||||
"OVHCloudChatConfig",
|
||||
|
|
@ -1039,6 +1042,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
".llms.codestral.completion.transformation",
|
||||
"CodestralTextCompletionConfig",
|
||||
),
|
||||
"InceptionTextCompletionConfig": (
|
||||
".llms.inception.completion.transformation",
|
||||
"InceptionTextCompletionConfig",
|
||||
),
|
||||
"AzureOpenAIAssistantsAPIConfig": (
|
||||
".llms.azure.azure",
|
||||
"AzureOpenAIAssistantsAPIConfig",
|
||||
|
|
@ -1054,6 +1061,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
".llms.azure.completion.transformation",
|
||||
"AzureOpenAITextConfig",
|
||||
),
|
||||
"AzureSpeechAudioTranscriptionConfig": (
|
||||
".llms.azure.audio_transcription.transformation",
|
||||
"AzureSpeechAudioTranscriptionConfig",
|
||||
),
|
||||
"HostedVLLMChatConfig": (
|
||||
".llms.hosted_vllm.chat.transformation",
|
||||
"HostedVLLMChatConfig",
|
||||
|
|
@ -1149,6 +1160,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
"MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"),
|
||||
"RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"),
|
||||
"LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"),
|
||||
"InceptionChatConfig": (
|
||||
".llms.inception.chat.transformation",
|
||||
"InceptionChatConfig",
|
||||
),
|
||||
"HyperbolicChatConfig": (
|
||||
".llms.hyperbolic.chat.transformation",
|
||||
"HyperbolicChatConfig",
|
||||
|
|
|
|||
|
|
@ -24,6 +24,22 @@ else:
|
|||
UserAPIKeyAuth = Any
|
||||
|
||||
|
||||
def _get_otel_v2_class() -> Optional[type]:
|
||||
"""Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent.
|
||||
|
||||
Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry
|
||||
SDK at module scope, so importing it eagerly would break installs without the
|
||||
SDK. The V2 logger only exists when ``LITELLM_OTEL_V2`` is enabled (which
|
||||
requires the SDK), so a failed import simply means "no V2 logger in play".
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations.otel.logger import OpenTelemetryV2
|
||||
|
||||
return OpenTelemetryV2
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
class ServiceLogging(CustomLogger):
|
||||
"""
|
||||
Separate class used for monitoring health of litellm-adjacent services (redis/postgres).
|
||||
|
|
@ -38,6 +54,37 @@ class ServiceLogging(CustomLogger):
|
|||
if "prometheus_system" in litellm.service_callback:
|
||||
self.prometheusServicesLogger = PrometheusServicesLogger()
|
||||
|
||||
def _resolve_otel_service_logger(self, callback: Any) -> Optional[Any]:
|
||||
"""Resolve the OTel logger (legacy or V2) to emit a service span on.
|
||||
|
||||
Returns the logger instance whose ``async_service_*_hook`` should fire for
|
||||
this ``callback``, or ``None`` when ``callback`` is not an OTel callback.
|
||||
|
||||
The V2 ``OpenTelemetryV2`` logger is a plain ``CustomLogger`` and is NOT a
|
||||
subclass of the legacy ``OpenTelemetry``, so the legacy ``isinstance``
|
||||
check alone misses it — which is why redis/postgres service spans never
|
||||
showed up under ``LITELLM_OTEL_V2``. Match both the legacy and V2 types,
|
||||
whether the callback is the logger instance itself or the ``"otel"`` string
|
||||
(which routes to the proxy's registered ``open_telemetry_logger``).
|
||||
"""
|
||||
otel_v2_cls = _get_otel_v2_class()
|
||||
|
||||
def _is_otel_logger(obj: Any) -> bool:
|
||||
if isinstance(obj, OpenTelemetry):
|
||||
return True
|
||||
return otel_v2_cls is not None and isinstance(obj, otel_v2_cls)
|
||||
|
||||
if _is_otel_logger(callback):
|
||||
return callback
|
||||
if callback == "otel":
|
||||
from litellm.proxy.proxy_server import open_telemetry_logger
|
||||
|
||||
if open_telemetry_logger is not None and _is_otel_logger(
|
||||
open_telemetry_logger
|
||||
):
|
||||
return open_telemetry_logger
|
||||
return None
|
||||
|
||||
def service_success_hook(
|
||||
self,
|
||||
service: ServiceTypes,
|
||||
|
|
@ -129,6 +176,13 @@ class ServiceLogging(CustomLogger):
|
|||
event_metadata=event_metadata,
|
||||
)
|
||||
|
||||
# OTel loggers already fired this event. ``service_callback`` can hold more
|
||||
# than one reference that resolves to the *same* logger — the ``"otel"``
|
||||
# string AND the registered instance both map to ``open_telemetry_logger``
|
||||
# (the V2 logger self-registers its instance even when the string is
|
||||
# present, unlike V1). Without this guard each such reference emits its own
|
||||
# span, so a single DB call shows up as duplicate ``postgres ...`` spans.
|
||||
emitted_otel_logger_ids: set = set()
|
||||
for callback in litellm.service_callback:
|
||||
if callback == "prometheus_system":
|
||||
await self.init_prometheus_services_logger_if_none()
|
||||
|
|
@ -144,19 +198,18 @@ class ServiceLogging(CustomLogger):
|
|||
end_time=end_time,
|
||||
event_metadata=event_metadata,
|
||||
)
|
||||
elif callback == "otel" or isinstance(callback, OpenTelemetry):
|
||||
_otel_logger_to_use: Optional[OpenTelemetry] = None
|
||||
if isinstance(callback, OpenTelemetry):
|
||||
_otel_logger_to_use = callback
|
||||
else:
|
||||
from litellm.proxy.proxy_server import open_telemetry_logger
|
||||
|
||||
if open_telemetry_logger is not None and isinstance(
|
||||
open_telemetry_logger, OpenTelemetry
|
||||
):
|
||||
_otel_logger_to_use = open_telemetry_logger
|
||||
|
||||
if _otel_logger_to_use is not None and parent_otel_span is not None:
|
||||
else:
|
||||
_otel_logger_to_use = self._resolve_otel_service_logger(callback)
|
||||
# No ``parent_otel_span is not None`` gate: a background service
|
||||
# call (no request on the stack) has no parent, and dropping it
|
||||
# here is what hid those calls from traces entirely. The OTel
|
||||
# logger decides what to do with a missing parent — legacy V1
|
||||
# no-ops, V2 emits a root span (and skips metrics-only pings).
|
||||
if (
|
||||
_otel_logger_to_use is not None
|
||||
and id(_otel_logger_to_use) not in emitted_otel_logger_ids
|
||||
):
|
||||
emitted_otel_logger_ids.add(id(_otel_logger_to_use))
|
||||
await _otel_logger_to_use.async_service_success_hook(
|
||||
payload=payload,
|
||||
parent_otel_span=parent_otel_span,
|
||||
|
|
@ -238,6 +291,9 @@ class ServiceLogging(CustomLogger):
|
|||
event_metadata=event_metadata,
|
||||
)
|
||||
|
||||
# Dedupe OTel loggers per event — see ``async_service_success_hook`` for why
|
||||
# the same logger can be referenced twice in ``service_callback``.
|
||||
emitted_otel_logger_ids: set = set()
|
||||
for callback in litellm.service_callback:
|
||||
if callback == "prometheus_system":
|
||||
await self.init_prometheus_services_logger_if_none()
|
||||
|
|
@ -255,22 +311,19 @@ class ServiceLogging(CustomLogger):
|
|||
end_time=end_time,
|
||||
event_metadata=event_metadata,
|
||||
)
|
||||
elif callback == "otel" or isinstance(callback, OpenTelemetry):
|
||||
_otel_logger_to_use: Optional[OpenTelemetry] = None
|
||||
if isinstance(callback, OpenTelemetry):
|
||||
_otel_logger_to_use = callback
|
||||
else:
|
||||
from litellm.proxy.proxy_server import open_telemetry_logger
|
||||
|
||||
if open_telemetry_logger is not None and isinstance(
|
||||
open_telemetry_logger, OpenTelemetry
|
||||
):
|
||||
_otel_logger_to_use = open_telemetry_logger
|
||||
else:
|
||||
_otel_logger_to_use = self._resolve_otel_service_logger(callback)
|
||||
|
||||
if not isinstance(error, str):
|
||||
error = str(error)
|
||||
|
||||
if _otel_logger_to_use is not None and parent_otel_span is not None:
|
||||
# See the success hook: no parent gate, so background failures
|
||||
# are traced too. V1 no-ops without a parent; V2 emits a root.
|
||||
if (
|
||||
_otel_logger_to_use is not None
|
||||
and id(_otel_logger_to_use) not in emitted_otel_logger_ids
|
||||
):
|
||||
emitted_otel_logger_ids.add(id(_otel_logger_to_use))
|
||||
await _otel_logger_to_use.async_service_failure_hook(
|
||||
payload=payload,
|
||||
error=error,
|
||||
|
|
@ -318,6 +371,8 @@ class ServiceLogging(CustomLogger):
|
|||
service=ServiceTypes.LITELLM,
|
||||
duration=_duration,
|
||||
call_type=kwargs.get("call_type", "unknown"),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ Always uses fastuuid for performance.
|
|||
|
||||
import fastuuid as _uuid # type: ignore
|
||||
|
||||
|
||||
# Expose a module-like alias so callers can use: uuid.uuid4()
|
||||
uuid = _uuid
|
||||
|
||||
|
|
|
|||
|
|
@ -20,9 +20,20 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
|
|||
)
|
||||
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
|
||||
|
||||
# litellm_params key carrying the authenticated principal (hashed virtual key) so
|
||||
# A2A provider configs can scope provider-side state (e.g. LangFlow session memory)
|
||||
# per key instead of trusting the client-supplied A2A contextId.
|
||||
A2A_USER_API_KEY_HASH_PARAM = "litellm_a2a_user_api_key_hash"
|
||||
|
||||
# Agent metadata fields stored in litellm_params that are not valid litellm.acompletion() kwargs
|
||||
_AGENT_ONLY_PARAMS = frozenset(
|
||||
{"is_public", "agent_name", "agent_id", "agent_card_params"}
|
||||
{
|
||||
"is_public",
|
||||
"agent_name",
|
||||
"agent_id",
|
||||
"agent_card_params",
|
||||
A2A_USER_API_KEY_HASH_PARAM,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -37,6 +48,8 @@ class A2ACompletionBridgeHandler:
|
|||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
*,
|
||||
_skip_a2a_provider_routing: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle non-streaming A2A request via litellm.acompletion.
|
||||
|
|
@ -50,25 +63,24 @@ class A2ACompletionBridgeHandler:
|
|||
Returns:
|
||||
A2A SendMessageResponse dict
|
||||
"""
|
||||
# Get provider config for custom_llm_provider
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=litellm_params.get("model"),
|
||||
)
|
||||
|
||||
# If provider config exists, use it
|
||||
if a2a_provider_config is not None:
|
||||
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
|
||||
|
||||
response_data = await a2a_provider_config.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
if not _skip_a2a_provider_routing:
|
||||
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=litellm_params.get("model"),
|
||||
)
|
||||
|
||||
return response_data
|
||||
if a2a_provider_config is not None:
|
||||
verbose_logger.info(
|
||||
f"A2A: Using provider config for {custom_llm_provider}"
|
||||
)
|
||||
|
||||
return await a2a_provider_config.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
# Extract message from params
|
||||
message = params.get("message", {})
|
||||
|
|
@ -107,6 +119,14 @@ class A2ACompletionBridgeHandler:
|
|||
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
# Apply forward metadata AFTER the litellm_params merge so the helper
|
||||
# sees any agent-owner-configured ``extra_body.metadata`` and can keep
|
||||
# those keys authoritative over the client-supplied A2A metadata.
|
||||
A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
|
||||
completion_params=completion_params,
|
||||
a2a_message=message,
|
||||
params=params,
|
||||
)
|
||||
|
||||
# Call litellm.acompletion
|
||||
response = await litellm.acompletion(**completion_params)
|
||||
|
|
@ -129,6 +149,8 @@ class A2ACompletionBridgeHandler:
|
|||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
*,
|
||||
_skip_a2a_provider_routing: bool = False,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""
|
||||
Handle streaming A2A request via litellm.acompletion with stream=True.
|
||||
|
|
@ -148,28 +170,27 @@ class A2ACompletionBridgeHandler:
|
|||
Yields:
|
||||
A2A streaming response events
|
||||
"""
|
||||
# Get provider config for custom_llm_provider
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=litellm_params.get("model"),
|
||||
)
|
||||
|
||||
# If provider config exists, use it
|
||||
if a2a_provider_config is not None:
|
||||
verbose_logger.info(
|
||||
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
|
||||
if not _skip_a2a_provider_routing:
|
||||
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model=litellm_params.get("model"),
|
||||
)
|
||||
|
||||
async for chunk in a2a_provider_config.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
):
|
||||
yield chunk
|
||||
if a2a_provider_config is not None:
|
||||
verbose_logger.info(
|
||||
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
|
||||
)
|
||||
|
||||
return
|
||||
async for chunk in a2a_provider_config.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
):
|
||||
yield chunk
|
||||
|
||||
return
|
||||
|
||||
# Extract message from params
|
||||
message = params.get("message", {})
|
||||
|
|
@ -214,6 +235,14 @@ class A2ACompletionBridgeHandler:
|
|||
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
# Apply forward metadata AFTER the litellm_params merge so the helper
|
||||
# sees any agent-owner-configured ``extra_body.metadata`` and can keep
|
||||
# those keys authoritative over the client-supplied A2A metadata.
|
||||
A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
|
||||
completion_params=completion_params,
|
||||
a2a_message=message,
|
||||
params=params,
|
||||
)
|
||||
|
||||
# 1. Emit initial task event (kind: "task", status: "submitted")
|
||||
task_event = A2ACompletionBridgeTransformation.create_task_event(ctx)
|
||||
|
|
|
|||
|
|
@ -45,10 +45,80 @@ class A2ACompletionBridgeTransformation:
|
|||
Static methods for transforming between A2A and OpenAI message formats.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _extract_text_from_a2a_parts(parts: List[Dict[str, Any]]) -> str:
|
||||
"""Extract text from A2A parts (with or without explicit ``kind``)."""
|
||||
content_parts: List[str] = []
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
kind = part.get("kind")
|
||||
text = part.get("text")
|
||||
if text is None:
|
||||
continue
|
||||
if kind in (None, "", "text"):
|
||||
content_parts.append(str(text))
|
||||
return "\n".join(content_parts)
|
||||
|
||||
@staticmethod
|
||||
def get_forward_metadata(
|
||||
a2a_message: Dict[str, Any],
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Merge A2A metadata from MessageSendParams and the message for downstream providers.
|
||||
|
||||
Forwarded once on the LangGraph run payload (``metadata``), not duplicated on
|
||||
each input message — see ``apply_forward_metadata_to_completion_params``.
|
||||
"""
|
||||
merged: Dict[str, Any] = {}
|
||||
if params and isinstance(params.get("metadata"), dict):
|
||||
merged.update(params["metadata"])
|
||||
message_metadata = a2a_message.get("metadata")
|
||||
if isinstance(message_metadata, dict):
|
||||
merged.update(message_metadata)
|
||||
return merged or None
|
||||
|
||||
@staticmethod
|
||||
def apply_forward_metadata_to_completion_params(
|
||||
completion_params: Dict[str, Any],
|
||||
a2a_message: Dict[str, Any],
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Attach A2A metadata to completion kwargs for provider bridges (e.g. LangGraph).
|
||||
|
||||
Uses ``extra_body`` so we do not collide with LiteLLM's spend-log ``metadata`` kwarg.
|
||||
"""
|
||||
forward_metadata = A2ACompletionBridgeTransformation.get_forward_metadata(
|
||||
a2a_message=a2a_message,
|
||||
params=params,
|
||||
)
|
||||
if not forward_metadata:
|
||||
return
|
||||
|
||||
extra_body = completion_params.get("extra_body")
|
||||
if not isinstance(extra_body, dict):
|
||||
extra_body = {}
|
||||
# Layer client-supplied A2A metadata under any agent-owner-configured
|
||||
# ``extra_body.metadata`` so the configured keys remain authoritative
|
||||
# and an A2A caller cannot overwrite server-set run metadata.
|
||||
existing_metadata = extra_body.get("metadata")
|
||||
existing_dict: Dict[str, Any] = (
|
||||
existing_metadata if isinstance(existing_metadata, dict) else {}
|
||||
)
|
||||
merged_metadata: Dict[str, Any] = {**forward_metadata, **existing_dict}
|
||||
extra_body = {**extra_body, "metadata": merged_metadata}
|
||||
completion_params["extra_body"] = extra_body
|
||||
|
||||
verbose_logger.debug(
|
||||
f"A2A -> completion forward metadata keys={list(forward_metadata.keys())}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def a2a_message_to_openai_messages(
|
||||
a2a_message: Dict[str, Any],
|
||||
) -> List[Dict[str, str]]:
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Transform an A2A message to OpenAI message format.
|
||||
|
||||
|
|
@ -70,21 +140,20 @@ class A2ACompletionBridgeTransformation:
|
|||
elif role == "system":
|
||||
openai_role = "system"
|
||||
|
||||
# Extract text content from parts
|
||||
content_parts = []
|
||||
for part in parts:
|
||||
kind = part.get("kind", "")
|
||||
if kind == "text":
|
||||
text = part.get("text", "")
|
||||
content_parts.append(text)
|
||||
if not isinstance(parts, list):
|
||||
parts = []
|
||||
|
||||
content = "\n".join(content_parts) if content_parts else ""
|
||||
content = A2ACompletionBridgeTransformation._extract_text_from_a2a_parts(parts)
|
||||
|
||||
# Do not attach A2A message.metadata here — the completion bridge forwards it
|
||||
# once at run level via extra_body.metadata (LangGraph POST /runs/wait shape).
|
||||
openai_message: Dict[str, Any] = {"role": openai_role, "content": content}
|
||||
|
||||
verbose_logger.debug(
|
||||
f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}"
|
||||
)
|
||||
|
||||
return [{"role": openai_role, "content": content}]
|
||||
return [openai_message]
|
||||
|
||||
@staticmethod
|
||||
def openai_response_to_a2a_response(
|
||||
|
|
@ -110,6 +179,7 @@ class A2ACompletionBridgeTransformation:
|
|||
|
||||
# Build A2A message
|
||||
a2a_message = {
|
||||
"kind": "message",
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": content}],
|
||||
"messageId": uuid4().hex,
|
||||
|
|
@ -119,9 +189,7 @@ class A2ACompletionBridgeTransformation:
|
|||
a2a_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"message": a2a_message,
|
||||
},
|
||||
"result": a2a_message,
|
||||
}
|
||||
|
||||
verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}")
|
||||
|
|
@ -235,50 +303,3 @@ class A2ACompletionBridgeTransformation:
|
|||
"taskId": ctx.task_id,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def openai_chunk_to_a2a_chunk(
|
||||
chunk: Any,
|
||||
request_id: Optional[str] = None,
|
||||
is_final: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Transform a LiteLLM streaming chunk to A2A streaming format.
|
||||
|
||||
NOTE: This method is deprecated for streaming. Use the event-based
|
||||
methods (create_task_event, create_status_update_event,
|
||||
create_artifact_update_event) instead for proper A2A streaming.
|
||||
|
||||
Args:
|
||||
chunk: LiteLLM ModelResponse chunk
|
||||
request_id: Original A2A request ID
|
||||
is_final: Whether this is the final chunk
|
||||
|
||||
Returns:
|
||||
A2A streaming chunk dict or None if no content
|
||||
"""
|
||||
# Extract delta content
|
||||
content = ""
|
||||
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
|
||||
choice = chunk.choices[0]
|
||||
if hasattr(choice, "delta") and choice.delta:
|
||||
content = choice.delta.content or ""
|
||||
|
||||
if not content and not is_final:
|
||||
return None
|
||||
|
||||
# Build A2A streaming chunk (legacy format)
|
||||
a2a_chunk = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"message": {
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": content}],
|
||||
"messageId": uuid4().hex,
|
||||
},
|
||||
"final": is_final,
|
||||
},
|
||||
}
|
||||
|
||||
return a2a_chunk
|
||||
|
|
|
|||
|
|
@ -159,7 +159,9 @@ async def _send_message_via_completion_bridge(
|
|||
api_base=api_base,
|
||||
)
|
||||
|
||||
return LiteLLMSendMessageResponse.from_dict(response_dict)
|
||||
return LiteLLMSendMessageResponse.from_dict(
|
||||
response_dict, request_id=str(request.id)
|
||||
)
|
||||
|
||||
|
||||
async def _execute_a2a_send_with_retry(
|
||||
|
|
@ -317,15 +319,6 @@ async def asend_message(
|
|||
)
|
||||
card_url = getattr(agent_card, "url", None) if agent_card else None
|
||||
|
||||
context_id = trace_id or str(uuid.uuid4())
|
||||
message = request.params.message
|
||||
if isinstance(message, dict):
|
||||
if message.get("context_id") is None:
|
||||
message["context_id"] = context_id
|
||||
else:
|
||||
if getattr(message, "context_id", None) is None:
|
||||
message.context_id = context_id
|
||||
|
||||
a2a_response = await _execute_a2a_send_with_retry(
|
||||
a2a_client=a2a_client,
|
||||
request=request,
|
||||
|
|
@ -338,7 +331,9 @@ async def asend_message(
|
|||
verbose_logger.info(f"A2A send_message completed, request_id={request.id}")
|
||||
|
||||
# Wrap in LiteLLM response type for _hidden_params support
|
||||
response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response)
|
||||
response = LiteLLMSendMessageResponse.from_a2a_response(
|
||||
a2a_response, request_id=str(request.id)
|
||||
)
|
||||
|
||||
# Calculate token usage from request and response
|
||||
response_dict = a2a_response.model_dump(mode="json", exclude_none=True)
|
||||
|
|
|
|||
|
|
@ -48,4 +48,16 @@ class A2AProviderConfigManager:
|
|||
|
||||
return BedrockAgentCoreA2AConfig()
|
||||
|
||||
if custom_llm_provider == "langflow":
|
||||
from litellm.a2a_protocol.providers.langflow.config import LangFlowA2AConfig
|
||||
|
||||
return LangFlowA2AConfig()
|
||||
|
||||
if custom_llm_provider == "watsonx_orchestrate":
|
||||
from litellm.a2a_protocol.providers.watsonx_orchestrate.config import (
|
||||
WatsonxOrchestrateA2AConfig,
|
||||
)
|
||||
|
||||
return WatsonxOrchestrateA2AConfig()
|
||||
|
||||
return None
|
||||
|
|
|
|||
0
litellm/a2a_protocol/providers/langflow/__init__.py
Normal file
0
litellm/a2a_protocol/providers/langflow/__init__.py
Normal file
62
litellm/a2a_protocol/providers/langflow/config.py
Normal file
62
litellm/a2a_protocol/providers/langflow/config.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
|
||||
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
|
||||
A2A_USER_API_KEY_HASH_PARAM,
|
||||
A2ACompletionBridgeHandler,
|
||||
)
|
||||
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
|
||||
from litellm.llms.langflow.a2a import merge_a2a_session_into_litellm_params
|
||||
|
||||
|
||||
class LangFlowA2AConfig(BaseA2AProviderConfig):
|
||||
"""A2A bridge for LangFlow: scopes contextId to the authenticated key as the
|
||||
LangFlow session_id, then uses completion."""
|
||||
|
||||
async def handle_non_streaming(
|
||||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, Any]:
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not litellm_params:
|
||||
raise ValueError(
|
||||
"litellm_params is required for LangFlowA2AConfig "
|
||||
"(must contain custom_llm_provider and model)"
|
||||
)
|
||||
litellm_params = merge_a2a_session_into_litellm_params(
|
||||
litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
|
||||
)
|
||||
return await A2ACompletionBridgeHandler.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
api_base=api_base,
|
||||
_skip_a2a_provider_routing=True,
|
||||
)
|
||||
|
||||
async def handle_streaming(
|
||||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not litellm_params:
|
||||
raise ValueError(
|
||||
"litellm_params is required for LangFlowA2AConfig "
|
||||
"(must contain custom_llm_provider and model)"
|
||||
)
|
||||
litellm_params = merge_a2a_session_into_litellm_params(
|
||||
litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
|
||||
)
|
||||
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
api_base=api_base,
|
||||
_skip_a2a_provider_routing=True,
|
||||
):
|
||||
yield chunk
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
# A2A to LiteLLM Completion Bridge
|
||||
|
||||
Routes A2A protocol requests through `litellm.acompletion`, enabling any LiteLLM-supported provider to be invoked via A2A.
|
||||
|
||||
## Flow
|
||||
|
||||
```
|
||||
A2A Request → Transform → litellm.acompletion → Transform → A2A Response
|
||||
```
|
||||
|
||||
## SDK Usage
|
||||
|
||||
Use the existing `asend_message` and `asend_message_streaming` functions with `litellm_params`:
|
||||
|
||||
```python
|
||||
from litellm.a2a_protocol import asend_message, asend_message_streaming
|
||||
from a2a.types import SendMessageRequest, SendStreamingMessageRequest, MessageSendParams
|
||||
from uuid import uuid4
|
||||
|
||||
# Non-streaming
|
||||
request = SendMessageRequest(
|
||||
id=str(uuid4()),
|
||||
params=MessageSendParams(
|
||||
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
|
||||
)
|
||||
)
|
||||
response = await asend_message(
|
||||
request=request,
|
||||
api_base="http://localhost:2024",
|
||||
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
|
||||
)
|
||||
|
||||
# Streaming
|
||||
stream_request = SendStreamingMessageRequest(
|
||||
id=str(uuid4()),
|
||||
params=MessageSendParams(
|
||||
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
|
||||
)
|
||||
)
|
||||
async for chunk in asend_message_streaming(
|
||||
request=stream_request,
|
||||
api_base="http://localhost:2024",
|
||||
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
|
||||
):
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Proxy Usage
|
||||
|
||||
Configure an agent with `custom_llm_provider` in `litellm_params`:
|
||||
|
||||
```yaml
|
||||
agents:
|
||||
- agent_name: my-langgraph-agent
|
||||
agent_card_params:
|
||||
name: "LangGraph Agent"
|
||||
url: "http://localhost:2024" # Used as api_base
|
||||
litellm_params:
|
||||
custom_llm_provider: langgraph
|
||||
model: agent
|
||||
```
|
||||
|
||||
When an A2A request hits `/a2a/{agent_id}/message/send`, the bridge:
|
||||
|
||||
1. Detects `custom_llm_provider` in agent's `litellm_params`
|
||||
2. Transforms A2A message → OpenAI messages
|
||||
3. Calls `litellm.acompletion(model="langgraph/agent", api_base="http://localhost:2024")`
|
||||
4. Transforms response → A2A format
|
||||
|
||||
## Classes
|
||||
|
||||
- `A2ACompletionBridgeTransformation` - Static methods for message format conversion
|
||||
- `A2ACompletionBridgeHandler` - Static methods for handling requests (streaming/non-streaming)
|
||||
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
"""
|
||||
LiteLLM Completion bridge provider for A2A protocol.
|
||||
|
||||
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
|
||||
"""
|
||||
|
|
@ -1,301 +0,0 @@
|
|||
"""
|
||||
Handler for A2A to LiteLLM completion bridge.
|
||||
|
||||
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
|
||||
|
||||
A2A Streaming Events (in order):
|
||||
1. Task event (kind: "task") - Initial task creation with status "submitted"
|
||||
2. Status update (kind: "status-update") - Status change to "working"
|
||||
3. Artifact update (kind: "artifact-update") - Content/artifact delivery
|
||||
4. Status update (kind: "status-update") - Final status "completed" with final=true
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.a2a_protocol.litellm_completion_bridge.pydantic_ai_transformation import (
|
||||
PydanticAITransformation,
|
||||
)
|
||||
from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
|
||||
A2ACompletionBridgeTransformation,
|
||||
A2AStreamingContext,
|
||||
)
|
||||
|
||||
|
||||
class A2ACompletionBridgeHandler:
|
||||
"""
|
||||
Static methods for handling A2A requests via LiteLLM completion.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
async def handle_non_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Handle non-streaming A2A request via litellm.acompletion.
|
||||
|
||||
Args:
|
||||
request_id: A2A JSON-RPC request ID
|
||||
params: A2A MessageSendParams containing the message
|
||||
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
|
||||
api_base: API base URL from agent_card_params
|
||||
|
||||
Returns:
|
||||
A2A SendMessageResponse dict
|
||||
"""
|
||||
# Check if this is a Pydantic AI agent request
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
if custom_llm_provider == "pydantic_ai_agents":
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
|
||||
verbose_logger.info(
|
||||
f"Pydantic AI: Routing to Pydantic AI agent at {api_base}"
|
||||
)
|
||||
|
||||
# Send request directly to Pydantic AI agent
|
||||
response_data = await PydanticAITransformation.send_non_streaming_request(
|
||||
api_base=api_base,
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
)
|
||||
|
||||
return response_data
|
||||
|
||||
# Extract message from params
|
||||
message = params.get("message", {})
|
||||
|
||||
# Transform A2A message to OpenAI format
|
||||
openai_messages = (
|
||||
A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
|
||||
)
|
||||
|
||||
# Get completion params
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
model = litellm_params.get("model", "agent")
|
||||
|
||||
# Build full model string if provider specified
|
||||
# Skip prepending if model already starts with the provider prefix
|
||||
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
|
||||
full_model = f"{custom_llm_provider}/{model}"
|
||||
else:
|
||||
full_model = model
|
||||
|
||||
verbose_logger.info(
|
||||
f"A2A completion bridge: model={full_model}, api_base={api_base}"
|
||||
)
|
||||
|
||||
# Build completion params dict
|
||||
completion_params = {
|
||||
"model": full_model,
|
||||
"messages": openai_messages,
|
||||
"api_base": api_base,
|
||||
"stream": False,
|
||||
}
|
||||
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
|
||||
litellm_params_to_add = {
|
||||
k: v
|
||||
for k, v in litellm_params.items()
|
||||
if k not in ("model", "custom_llm_provider")
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
|
||||
# Call litellm.acompletion
|
||||
response = await litellm.acompletion(**completion_params)
|
||||
|
||||
# Transform response to A2A format
|
||||
a2a_response = (
|
||||
A2ACompletionBridgeTransformation.openai_response_to_a2a_response(
|
||||
response=response,
|
||||
request_id=request_id,
|
||||
)
|
||||
)
|
||||
|
||||
verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}")
|
||||
|
||||
return a2a_response
|
||||
|
||||
@staticmethod
|
||||
async def handle_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""
|
||||
Handle streaming A2A request via litellm.acompletion with stream=True.
|
||||
|
||||
Emits proper A2A streaming events:
|
||||
1. Task event (kind: "task") - Initial task with status "submitted"
|
||||
2. Status update (kind: "status-update") - Status "working"
|
||||
3. Artifact update (kind: "artifact-update") - Content delivery
|
||||
4. Status update (kind: "status-update") - Final "completed" status
|
||||
|
||||
Args:
|
||||
request_id: A2A JSON-RPC request ID
|
||||
params: A2A MessageSendParams containing the message
|
||||
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
|
||||
api_base: API base URL from agent_card_params
|
||||
|
||||
Yields:
|
||||
A2A streaming response events
|
||||
"""
|
||||
# Check if this is a Pydantic AI agent request
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
if custom_llm_provider == "pydantic_ai_agents":
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Pydantic AI agents")
|
||||
|
||||
verbose_logger.info(
|
||||
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
|
||||
)
|
||||
|
||||
# Get non-streaming response first
|
||||
response_data = await PydanticAITransformation.send_non_streaming_request(
|
||||
api_base=api_base,
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
)
|
||||
|
||||
# Convert to fake streaming
|
||||
async for chunk in PydanticAITransformation.fake_streaming_from_response(
|
||||
response_data=response_data,
|
||||
request_id=request_id,
|
||||
):
|
||||
yield chunk
|
||||
|
||||
return
|
||||
|
||||
# Extract message from params
|
||||
message = params.get("message", {})
|
||||
|
||||
# Create streaming context
|
||||
ctx = A2AStreamingContext(
|
||||
request_id=request_id,
|
||||
input_message=message,
|
||||
)
|
||||
|
||||
# Transform A2A message to OpenAI format
|
||||
openai_messages = (
|
||||
A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
|
||||
)
|
||||
|
||||
# Get completion params
|
||||
custom_llm_provider = litellm_params.get("custom_llm_provider")
|
||||
model = litellm_params.get("model", "agent")
|
||||
|
||||
# Build full model string if provider specified
|
||||
# Skip prepending if model already starts with the provider prefix
|
||||
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
|
||||
full_model = f"{custom_llm_provider}/{model}"
|
||||
else:
|
||||
full_model = model
|
||||
|
||||
verbose_logger.info(
|
||||
f"A2A completion bridge streaming: model={full_model}, api_base={api_base}"
|
||||
)
|
||||
|
||||
# Build completion params dict
|
||||
completion_params = {
|
||||
"model": full_model,
|
||||
"messages": openai_messages,
|
||||
"api_base": api_base,
|
||||
"stream": True,
|
||||
}
|
||||
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
|
||||
litellm_params_to_add = {
|
||||
k: v
|
||||
for k, v in litellm_params.items()
|
||||
if k not in ("model", "custom_llm_provider")
|
||||
}
|
||||
completion_params.update(litellm_params_to_add)
|
||||
|
||||
# 1. Emit initial task event (kind: "task", status: "submitted")
|
||||
task_event = A2ACompletionBridgeTransformation.create_task_event(ctx)
|
||||
yield task_event
|
||||
|
||||
# 2. Emit status update (kind: "status-update", status: "working")
|
||||
working_event = A2ACompletionBridgeTransformation.create_status_update_event(
|
||||
ctx=ctx,
|
||||
state="working",
|
||||
final=False,
|
||||
message_text="Processing request...",
|
||||
)
|
||||
yield working_event
|
||||
|
||||
# Call litellm.acompletion with streaming
|
||||
response = await litellm.acompletion(**completion_params)
|
||||
|
||||
# 3. Accumulate content and emit artifact update
|
||||
accumulated_text = ""
|
||||
chunk_count = 0
|
||||
async for chunk in response: # type: ignore[union-attr]
|
||||
chunk_count += 1
|
||||
|
||||
# Extract delta content
|
||||
content = ""
|
||||
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
|
||||
choice = chunk.choices[0]
|
||||
if hasattr(choice, "delta") and choice.delta:
|
||||
content = choice.delta.content or ""
|
||||
|
||||
if content:
|
||||
accumulated_text += content
|
||||
|
||||
# Emit artifact update with accumulated content
|
||||
if accumulated_text:
|
||||
artifact_event = (
|
||||
A2ACompletionBridgeTransformation.create_artifact_update_event(
|
||||
ctx=ctx,
|
||||
text=accumulated_text,
|
||||
)
|
||||
)
|
||||
yield artifact_event
|
||||
|
||||
# 4. Emit final status update (kind: "status-update", status: "completed", final: true)
|
||||
completed_event = A2ACompletionBridgeTransformation.create_status_update_event(
|
||||
ctx=ctx,
|
||||
state="completed",
|
||||
final=True,
|
||||
)
|
||||
yield completed_event
|
||||
|
||||
verbose_logger.info(
|
||||
f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}"
|
||||
)
|
||||
|
||||
|
||||
# Convenience functions that delegate to the class methods
|
||||
async def handle_a2a_completion(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Convenience function for non-streaming A2A completion."""
|
||||
return await A2ACompletionBridgeHandler.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
|
||||
async def handle_a2a_completion_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""Convenience function for streaming A2A completion."""
|
||||
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
api_base=api_base,
|
||||
):
|
||||
yield chunk
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
"""
|
||||
Transformation utilities for A2A <-> OpenAI message format conversion.
|
||||
|
||||
A2A Message Format:
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "Hello!"}],
|
||||
"messageId": "abc123"
|
||||
}
|
||||
|
||||
OpenAI Message Format:
|
||||
{"role": "user", "content": "Hello!"}
|
||||
|
||||
A2A Streaming Events:
|
||||
- Task event (kind: "task") - Initial task creation with status "submitted"
|
||||
- Status update (kind: "status-update") - Status changes (working, completed)
|
||||
- Artifact update (kind: "artifact-update") - Content/artifact delivery
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
class A2AStreamingContext:
|
||||
"""
|
||||
Context holder for A2A streaming state.
|
||||
Tracks task_id, context_id, and message accumulation.
|
||||
"""
|
||||
|
||||
def __init__(self, request_id: str, input_message: Dict[str, Any]):
|
||||
self.request_id = request_id
|
||||
self.task_id = str(uuid4())
|
||||
self.context_id = str(uuid4())
|
||||
self.input_message = input_message
|
||||
self.accumulated_text = ""
|
||||
self.has_emitted_task = False
|
||||
self.has_emitted_working = False
|
||||
|
||||
|
||||
class A2ACompletionBridgeTransformation:
|
||||
"""
|
||||
Static methods for transforming between A2A and OpenAI message formats.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def a2a_message_to_openai_messages(
|
||||
a2a_message: Dict[str, Any],
|
||||
) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Transform an A2A message to OpenAI message format.
|
||||
|
||||
Args:
|
||||
a2a_message: A2A message with role, parts, and messageId
|
||||
|
||||
Returns:
|
||||
List of OpenAI-format messages
|
||||
"""
|
||||
role = a2a_message.get("role", "user")
|
||||
parts = a2a_message.get("parts", [])
|
||||
|
||||
# Map A2A roles to OpenAI roles
|
||||
openai_role = role
|
||||
if role == "user":
|
||||
openai_role = "user"
|
||||
elif role == "assistant":
|
||||
openai_role = "assistant"
|
||||
elif role == "system":
|
||||
openai_role = "system"
|
||||
|
||||
# Extract text content from parts
|
||||
content_parts = []
|
||||
for part in parts:
|
||||
kind = part.get("kind", "")
|
||||
if kind == "text":
|
||||
text = part.get("text", "")
|
||||
content_parts.append(text)
|
||||
|
||||
content = "\n".join(content_parts) if content_parts else ""
|
||||
|
||||
verbose_logger.debug(
|
||||
f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}"
|
||||
)
|
||||
|
||||
return [{"role": openai_role, "content": content}]
|
||||
|
||||
@staticmethod
|
||||
def openai_response_to_a2a_response(
|
||||
response: Any,
|
||||
request_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Transform a LiteLLM ModelResponse to A2A SendMessageResponse format.
|
||||
|
||||
Args:
|
||||
response: LiteLLM ModelResponse object
|
||||
request_id: Original A2A request ID
|
||||
|
||||
Returns:
|
||||
A2A SendMessageResponse dict
|
||||
"""
|
||||
# Extract content from response
|
||||
content = ""
|
||||
if hasattr(response, "choices") and response.choices:
|
||||
choice = response.choices[0]
|
||||
if hasattr(choice, "message") and choice.message:
|
||||
content = choice.message.content or ""
|
||||
|
||||
# Build A2A message
|
||||
a2a_message = {
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": content}],
|
||||
"messageId": uuid4().hex,
|
||||
}
|
||||
|
||||
# Build A2A response
|
||||
a2a_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"message": a2a_message,
|
||||
},
|
||||
}
|
||||
|
||||
verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}")
|
||||
|
||||
return a2a_response
|
||||
|
||||
@staticmethod
|
||||
def _get_timestamp() -> str:
|
||||
"""Get current timestamp in ISO format with timezone."""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
@staticmethod
|
||||
def create_task_event(
|
||||
ctx: A2AStreamingContext,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create the initial task event with status 'submitted'.
|
||||
|
||||
This is the first event emitted in an A2A streaming response.
|
||||
"""
|
||||
return {
|
||||
"id": ctx.request_id,
|
||||
"jsonrpc": "2.0",
|
||||
"result": {
|
||||
"contextId": ctx.context_id,
|
||||
"history": [
|
||||
{
|
||||
"contextId": ctx.context_id,
|
||||
"kind": "message",
|
||||
"messageId": ctx.input_message.get("messageId", uuid4().hex),
|
||||
"parts": ctx.input_message.get("parts", []),
|
||||
"role": ctx.input_message.get("role", "user"),
|
||||
"taskId": ctx.task_id,
|
||||
}
|
||||
],
|
||||
"id": ctx.task_id,
|
||||
"kind": "task",
|
||||
"status": {
|
||||
"state": "submitted",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def create_status_update_event(
|
||||
ctx: A2AStreamingContext,
|
||||
state: str,
|
||||
final: bool = False,
|
||||
message_text: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create a status update event.
|
||||
|
||||
Args:
|
||||
ctx: Streaming context
|
||||
state: Status state ('working', 'completed')
|
||||
final: Whether this is the final event
|
||||
message_text: Optional message text for 'working' status
|
||||
"""
|
||||
status: Dict[str, Any] = {
|
||||
"state": state,
|
||||
"timestamp": A2ACompletionBridgeTransformation._get_timestamp(),
|
||||
}
|
||||
|
||||
# Add message for 'working' status
|
||||
if state == "working" and message_text:
|
||||
status["message"] = {
|
||||
"contextId": ctx.context_id,
|
||||
"kind": "message",
|
||||
"messageId": str(uuid4()),
|
||||
"parts": [{"kind": "text", "text": message_text}],
|
||||
"role": "agent",
|
||||
"taskId": ctx.task_id,
|
||||
}
|
||||
|
||||
return {
|
||||
"id": ctx.request_id,
|
||||
"jsonrpc": "2.0",
|
||||
"result": {
|
||||
"contextId": ctx.context_id,
|
||||
"final": final,
|
||||
"kind": "status-update",
|
||||
"status": status,
|
||||
"taskId": ctx.task_id,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def create_artifact_update_event(
|
||||
ctx: A2AStreamingContext,
|
||||
text: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Create an artifact update event with content.
|
||||
|
||||
Args:
|
||||
ctx: Streaming context
|
||||
text: The text content for the artifact
|
||||
"""
|
||||
return {
|
||||
"id": ctx.request_id,
|
||||
"jsonrpc": "2.0",
|
||||
"result": {
|
||||
"artifact": {
|
||||
"artifactId": str(uuid4()),
|
||||
"name": "response",
|
||||
"parts": [{"kind": "text", "text": text}],
|
||||
},
|
||||
"contextId": ctx.context_id,
|
||||
"kind": "artifact-update",
|
||||
"taskId": ctx.task_id,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def openai_chunk_to_a2a_chunk(
|
||||
chunk: Any,
|
||||
request_id: Optional[str] = None,
|
||||
is_final: bool = False,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Transform a LiteLLM streaming chunk to A2A streaming format.
|
||||
|
||||
NOTE: This method is deprecated for streaming. Use the event-based
|
||||
methods (create_task_event, create_status_update_event,
|
||||
create_artifact_update_event) instead for proper A2A streaming.
|
||||
|
||||
Args:
|
||||
chunk: LiteLLM ModelResponse chunk
|
||||
request_id: Original A2A request ID
|
||||
is_final: Whether this is the final chunk
|
||||
|
||||
Returns:
|
||||
A2A streaming chunk dict or None if no content
|
||||
"""
|
||||
# Extract delta content
|
||||
content = ""
|
||||
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
|
||||
choice = chunk.choices[0]
|
||||
if hasattr(choice, "delta") and choice.delta:
|
||||
content = choice.delta.content or ""
|
||||
|
||||
if not content and not is_final:
|
||||
return None
|
||||
|
||||
# Build A2A streaming chunk (legacy format)
|
||||
a2a_chunk = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"message": {
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": content}],
|
||||
"messageId": uuid4().hex,
|
||||
},
|
||||
"final": is_final,
|
||||
},
|
||||
}
|
||||
|
||||
return a2a_chunk
|
||||
|
|
@ -289,16 +289,16 @@ class PydanticAITransformation:
|
|||
Transform Pydantic AI task response to standard A2A non-streaming format.
|
||||
|
||||
Pydantic AI returns a task with history/artifacts, but the standard A2A
|
||||
non-streaming format expects:
|
||||
non-streaming format expects ``result`` to be the Message directly
|
||||
(``kind="message"``), per the A2A spec / ``SendMessageResponse``:
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": "...",
|
||||
"result": {
|
||||
"message": {
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": "..."}],
|
||||
"messageId": "..."
|
||||
}
|
||||
"kind": "message",
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": "..."}],
|
||||
"messageId": "..."
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -316,6 +316,7 @@ class PydanticAITransformation:
|
|||
|
||||
# Build standard A2A message
|
||||
a2a_message = {
|
||||
"kind": "message",
|
||||
"role": "agent",
|
||||
"parts": parts if parts else [{"kind": "text", "text": full_text}],
|
||||
"messageId": message_id,
|
||||
|
|
@ -325,9 +326,7 @@ class PydanticAITransformation:
|
|||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"message": a2a_message,
|
||||
},
|
||||
"result": a2a_message,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
"""
|
||||
IBM watsonx Orchestrate (WXO) A2A provider.
|
||||
"""
|
||||
55
litellm/a2a_protocol/providers/watsonx_orchestrate/config.py
Normal file
55
litellm/a2a_protocol/providers/watsonx_orchestrate/config.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
"""
|
||||
A2A provider configuration for IBM watsonx Orchestrate (WXO).
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
|
||||
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
|
||||
from litellm.a2a_protocol.providers.watsonx_orchestrate.handler import (
|
||||
WatsonxOrchestrateHandler,
|
||||
)
|
||||
|
||||
|
||||
class WatsonxOrchestrateA2AConfig(BaseA2AProviderConfig):
|
||||
"""A2A bridge for IBM watsonx Orchestrate (REST runs API + poll/SSE)."""
|
||||
|
||||
async def handle_non_streaming(
|
||||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Handle a non-streaming A2A request via WXO runs API."""
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not litellm_params:
|
||||
raise ValueError(
|
||||
"litellm_params is required for WatsonxOrchestrateA2AConfig "
|
||||
"(must contain cp4d_host, instance_id, wxo_agent_id, api_key)"
|
||||
)
|
||||
return await WatsonxOrchestrateHandler.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
async def handle_streaming(
|
||||
self,
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
api_base: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""Handle a streaming A2A request via WXO streaming runs API."""
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not litellm_params:
|
||||
raise ValueError(
|
||||
"litellm_params is required for WatsonxOrchestrateA2AConfig "
|
||||
"(must contain cp4d_host, instance_id, wxo_agent_id, api_key)"
|
||||
)
|
||||
async for chunk in WatsonxOrchestrateHandler.handle_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
):
|
||||
yield chunk
|
||||
373
litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py
Normal file
373
litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py
Normal file
|
|
@ -0,0 +1,373 @@
|
|||
"""
|
||||
Handler for IBM watsonx Orchestrate (WXO) agent provider.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any, AsyncIterator, Dict, NamedTuple, Optional, Tuple, cast
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.a2a_protocol.providers.watsonx_orchestrate.transformation import (
|
||||
WatsonxOrchestrateTransformation,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
_IBM_CLOUD_IAM_URL = "https://iam.cloud.ibm.com/identity/token"
|
||||
_POLL_INTERVAL_S = 2.0
|
||||
_MAX_POLL_ATTEMPTS = 90
|
||||
_TOKEN_CACHE_TTL_BUFFER_S = 60
|
||||
_token_cache: Dict[str, Tuple[str, float]] = {}
|
||||
|
||||
|
||||
class WXORequestParams(NamedTuple):
|
||||
cp4d_host: str
|
||||
instance_id: str
|
||||
wxo_agent_id: str
|
||||
api_key: str
|
||||
username: Optional[str]
|
||||
auth_mode: str
|
||||
thread_id: Optional[str]
|
||||
|
||||
|
||||
class WatsonxOrchestrateHandler:
|
||||
@staticmethod
|
||||
def _http_client(timeout: float = 90.0) -> AsyncHTTPHandler:
|
||||
return get_async_httpx_client(
|
||||
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
|
||||
params={"timeout": timeout},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _token_cache_key(
|
||||
auth_mode: str,
|
||||
cp4d_host: str,
|
||||
api_key: str,
|
||||
username: Optional[str],
|
||||
) -> str:
|
||||
material = f"{auth_mode}:{cp4d_host}:{username or ''}:{api_key}"
|
||||
return hashlib.sha256(material.encode()).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _cp4d_token_ttl_seconds(
|
||||
expiration: Any, now_wall: Optional[float] = None
|
||||
) -> int:
|
||||
# CP4D returns expiration as absolute Unix epoch seconds, not a duration.
|
||||
expires_at = int(expiration)
|
||||
wall = now_wall if now_wall is not None else time.time()
|
||||
return max(expires_at - int(wall), 0)
|
||||
|
||||
@staticmethod
|
||||
async def _get_bearer_token(
|
||||
cp4d_host: str,
|
||||
auth_mode: str,
|
||||
api_key: str,
|
||||
username: Optional[str] = None,
|
||||
client: Optional[AsyncHTTPHandler] = None,
|
||||
) -> str:
|
||||
cache_key = WatsonxOrchestrateHandler._token_cache_key(
|
||||
auth_mode, cp4d_host, api_key, username
|
||||
)
|
||||
now = time.monotonic()
|
||||
cached = _token_cache.get(cache_key)
|
||||
if cached and cached[1] > now:
|
||||
return cached[0]
|
||||
|
||||
if client is None:
|
||||
client = WatsonxOrchestrateHandler._http_client(timeout=30.0)
|
||||
|
||||
if auth_mode == "ibm_cloud":
|
||||
response = await client.post(
|
||||
_IBM_CLOUD_IAM_URL,
|
||||
data={
|
||||
"grant_type": "urn:ibm:params:oauth:grant-type:apikey",
|
||||
"apikey": api_key,
|
||||
},
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = str(payload["access_token"])
|
||||
ttl_s = int(payload.get("expires_in", 3600))
|
||||
else:
|
||||
if not username:
|
||||
raise ValueError(
|
||||
"'username' is required in litellm_params when auth_mode='cp4d'"
|
||||
)
|
||||
token_url = f"{cp4d_host.rstrip('/')}/icp4d-api/v1/authorize"
|
||||
response = await client.post(
|
||||
token_url,
|
||||
json={"username": username, "api_key": api_key},
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
token = str(payload["token"])
|
||||
expiration = payload.get("expiration")
|
||||
if expiration is None:
|
||||
ttl_s = 3600
|
||||
else:
|
||||
ttl_s = WatsonxOrchestrateHandler._cp4d_token_ttl_seconds(expiration)
|
||||
|
||||
expires_at = now + max(ttl_s - _TOKEN_CACHE_TTL_BUFFER_S, 0)
|
||||
_token_cache[cache_key] = (token, expires_at)
|
||||
for stale_key, (_, stale_expires_at) in list(_token_cache.items()):
|
||||
if stale_expires_at <= now:
|
||||
del _token_cache[stale_key]
|
||||
return token
|
||||
|
||||
@staticmethod
|
||||
async def _poll_run(
|
||||
base_url: str,
|
||||
run_id: str,
|
||||
auth_headers: Dict[str, str],
|
||||
client: AsyncHTTPHandler,
|
||||
max_attempts: int = _MAX_POLL_ATTEMPTS,
|
||||
interval_s: float = _POLL_INTERVAL_S,
|
||||
) -> Dict[str, Any]:
|
||||
url = f"{base_url}/v1/orchestrate/runs/{run_id}"
|
||||
|
||||
for attempt in range(max_attempts):
|
||||
await asyncio.sleep(interval_s)
|
||||
response = await client.get(url, headers=auth_headers)
|
||||
response.raise_for_status()
|
||||
result: Dict[str, Any] = response.json()
|
||||
status = result.get("status", "")
|
||||
verbose_logger.debug(
|
||||
f"WXO: Poll {attempt + 1}/{max_attempts} run='{run_id}' status='{status}'"
|
||||
)
|
||||
if status in WatsonxOrchestrateTransformation.TERMINAL_STATES:
|
||||
return result
|
||||
|
||||
raise asyncio.TimeoutError(
|
||||
f"WXO run '{run_id}' did not reach a terminal state after "
|
||||
f"{max_attempts * interval_s:.0f}s"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _get_successful_run_data(
|
||||
run_data: Dict[str, Any],
|
||||
base_url: str,
|
||||
auth_headers: Dict[str, str],
|
||||
client: AsyncHTTPHandler,
|
||||
) -> Dict[str, Any]:
|
||||
status = run_data.get("status", "")
|
||||
if status not in WatsonxOrchestrateTransformation.TERMINAL_STATES:
|
||||
run_id = run_data.get("run_id") or run_data.get("id") or ""
|
||||
if not run_id:
|
||||
raise ValueError(f"WXO: No run_id in response: {run_data}")
|
||||
run_data = await WatsonxOrchestrateHandler._poll_run(
|
||||
base_url=base_url,
|
||||
run_id=run_id,
|
||||
auth_headers=auth_headers,
|
||||
client=client,
|
||||
)
|
||||
status = run_data.get("status", "")
|
||||
|
||||
if status not in WatsonxOrchestrateTransformation.SUCCESS_STATES:
|
||||
raise RuntimeError(
|
||||
f"WXO run ended with non-success status '{status}': {run_data}"
|
||||
)
|
||||
|
||||
return run_data
|
||||
|
||||
@staticmethod
|
||||
async def _accumulate_wxo_sse_text(response: Any) -> str:
|
||||
accumulated_text = ""
|
||||
async for line in response.aiter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data_str = line[5:].strip()
|
||||
if not data_str or data_str == "[DONE]":
|
||||
continue
|
||||
try:
|
||||
event = json.loads(data_str)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
chunk_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result(
|
||||
event
|
||||
)
|
||||
if chunk_text:
|
||||
accumulated_text += chunk_text
|
||||
return accumulated_text
|
||||
|
||||
@staticmethod
|
||||
def _extract_litellm_params(litellm_params: Dict[str, Any]) -> WXORequestParams:
|
||||
cp4d_host = litellm_params.get("cp4d_host") or ""
|
||||
instance_id = litellm_params.get("instance_id") or ""
|
||||
wxo_agent_id = litellm_params.get("wxo_agent_id") or ""
|
||||
api_key = litellm_params.get("api_key") or ""
|
||||
|
||||
if not cp4d_host:
|
||||
raise ValueError("'cp4d_host' is required in litellm_params for WXO agents")
|
||||
if not instance_id:
|
||||
raise ValueError(
|
||||
"'instance_id' is required in litellm_params for WXO agents"
|
||||
)
|
||||
if not wxo_agent_id:
|
||||
raise ValueError(
|
||||
"'wxo_agent_id' is required in litellm_params for WXO agents"
|
||||
)
|
||||
if not api_key:
|
||||
raise ValueError("'api_key' is required in litellm_params for WXO agents")
|
||||
|
||||
return WXORequestParams(
|
||||
cp4d_host=cp4d_host,
|
||||
instance_id=instance_id,
|
||||
wxo_agent_id=wxo_agent_id,
|
||||
api_key=api_key,
|
||||
username=litellm_params.get("username") or None,
|
||||
auth_mode=litellm_params.get("auth_mode") or "cp4d",
|
||||
thread_id=litellm_params.get("thread_id") or None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def handle_non_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
|
||||
|
||||
client = WatsonxOrchestrateHandler._http_client(timeout=90.0)
|
||||
token = await WatsonxOrchestrateHandler._get_bearer_token(
|
||||
cp4d_host=wxo.cp4d_host,
|
||||
auth_mode=wxo.auth_mode,
|
||||
api_key=wxo.api_key,
|
||||
username=wxo.username,
|
||||
client=client,
|
||||
)
|
||||
base_url = WatsonxOrchestrateTransformation.get_api_base_url(
|
||||
wxo.cp4d_host, wxo.instance_id
|
||||
)
|
||||
auth_headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params)
|
||||
body = WatsonxOrchestrateTransformation.build_wxo_run_body(
|
||||
wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id
|
||||
)
|
||||
|
||||
run_response = await client.post(
|
||||
f"{base_url}/v1/orchestrate/runs",
|
||||
json=body,
|
||||
headers=auth_headers,
|
||||
)
|
||||
run_response.raise_for_status()
|
||||
run_data: Dict[str, Any] = run_response.json()
|
||||
|
||||
run_data = await WatsonxOrchestrateHandler._get_successful_run_data(
|
||||
run_data=run_data,
|
||||
base_url=base_url,
|
||||
auth_headers=auth_headers,
|
||||
client=client,
|
||||
)
|
||||
|
||||
response_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result(
|
||||
run_data
|
||||
)
|
||||
return WatsonxOrchestrateTransformation.build_a2a_message_response(
|
||||
request_id=request_id, text=response_text
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def handle_streaming(
|
||||
request_id: str,
|
||||
params: Dict[str, Any],
|
||||
litellm_params: Dict[str, Any],
|
||||
chunk_size: int = 50,
|
||||
delay_ms: int = 10,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
|
||||
|
||||
client = WatsonxOrchestrateHandler._http_client(timeout=120.0)
|
||||
token = await WatsonxOrchestrateHandler._get_bearer_token(
|
||||
cp4d_host=wxo.cp4d_host,
|
||||
auth_mode=wxo.auth_mode,
|
||||
api_key=wxo.api_key,
|
||||
username=wxo.username,
|
||||
client=client,
|
||||
)
|
||||
base_url = WatsonxOrchestrateTransformation.get_api_base_url(
|
||||
wxo.cp4d_host, wxo.instance_id
|
||||
)
|
||||
auth_headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream, application/json",
|
||||
}
|
||||
text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params)
|
||||
body = WatsonxOrchestrateTransformation.build_wxo_run_body(
|
||||
wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id
|
||||
)
|
||||
|
||||
try:
|
||||
response = await client.post(
|
||||
f"{base_url}/v1/orchestrate/runs/stream",
|
||||
json=body,
|
||||
headers=auth_headers,
|
||||
stream=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.TransportError as exc:
|
||||
verbose_logger.warning(
|
||||
f"WXO: Streaming request failed before a run was submitted "
|
||||
f"({exc!r}), falling back to non-streaming + fake streaming",
|
||||
exc_info=True,
|
||||
)
|
||||
result = await WatsonxOrchestrateHandler.handle_non_streaming(
|
||||
request_id=request_id,
|
||||
params=params,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
response_text = (
|
||||
WatsonxOrchestrateTransformation.extract_text_from_a2a_message_response(
|
||||
result
|
||||
)
|
||||
)
|
||||
async for (
|
||||
chunk
|
||||
) in WatsonxOrchestrateTransformation.fake_streaming_from_text(
|
||||
text=response_text,
|
||||
request_id=request_id,
|
||||
chunk_size=chunk_size,
|
||||
delay_ms=delay_ms,
|
||||
):
|
||||
yield chunk
|
||||
return
|
||||
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if "text/event-stream" not in content_type:
|
||||
response_body = await response.aread()
|
||||
result = json.loads(response_body)
|
||||
result = await WatsonxOrchestrateHandler._get_successful_run_data(
|
||||
run_data=result,
|
||||
base_url=base_url,
|
||||
auth_headers=auth_headers,
|
||||
client=client,
|
||||
)
|
||||
accumulated_text = (
|
||||
WatsonxOrchestrateTransformation.extract_text_from_wxo_result(result)
|
||||
)
|
||||
else:
|
||||
accumulated_text = await WatsonxOrchestrateHandler._accumulate_wxo_sse_text(
|
||||
response
|
||||
)
|
||||
|
||||
async for chunk in WatsonxOrchestrateTransformation.fake_streaming_from_text(
|
||||
text=accumulated_text,
|
||||
request_id=request_id,
|
||||
chunk_size=chunk_size,
|
||||
delay_ms=delay_ms,
|
||||
):
|
||||
yield chunk
|
||||
|
|
@ -0,0 +1,224 @@
|
|||
"""
|
||||
Transformation layer for IBM watsonx Orchestrate (WXO) agent provider.
|
||||
|
||||
WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
|
||||
POST /v1/orchestrate/runs → submit run, get run_id
|
||||
GET /v1/orchestrate/runs/{id} → poll until terminal state
|
||||
POST /v1/orchestrate/runs/stream → native SSE streaming
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Any, AsyncIterator, Dict, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
|
||||
class WatsonxOrchestrateTransformation:
|
||||
"""
|
||||
Handles request/response transformation between A2A and the WXO REST API.
|
||||
"""
|
||||
|
||||
TERMINAL_STATES = frozenset(
|
||||
{"completed", "succeeded", "failed", "error", "cancelled"}
|
||||
)
|
||||
SUCCESS_STATES = frozenset({"completed", "succeeded"})
|
||||
|
||||
@staticmethod
|
||||
def get_api_base_url(cp4d_host: str, instance_id: str) -> str:
|
||||
"""Build the WXO API base URL from host and instance ID."""
|
||||
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_a2a_params(params: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Extract user message text from A2A MessageSendParams.
|
||||
|
||||
A2A format: params.message.parts[*] where part.kind == "text"
|
||||
"""
|
||||
message = params.get("message", {})
|
||||
parts = message.get("parts", [])
|
||||
texts = []
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
kind = part.get("kind")
|
||||
if kind in (None, "", "text") and part.get("text"):
|
||||
texts.append(part["text"])
|
||||
return " ".join(texts) or ""
|
||||
|
||||
@staticmethod
|
||||
def build_wxo_run_body(
|
||||
wxo_agent_id: str,
|
||||
text: str,
|
||||
thread_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the WXO POST /v1/orchestrate/runs request body."""
|
||||
body: Dict[str, Any] = {
|
||||
"agent_id": wxo_agent_id,
|
||||
"message": {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"response_type": "text",
|
||||
"text": text,
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
if thread_id:
|
||||
body["thread_id"] = thread_id
|
||||
return body
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_wxo_result(result: Any) -> str:
|
||||
"""
|
||||
Extract response text from a WXO run result.
|
||||
|
||||
WXO can return text in several locations; checks in priority order per the API spec.
|
||||
"""
|
||||
if not isinstance(result, dict):
|
||||
return ""
|
||||
|
||||
# Primary: last_message.content[0].text
|
||||
try:
|
||||
text = result["last_message"]["content"][0]["text"]
|
||||
if text:
|
||||
return str(text)
|
||||
except (KeyError, IndexError, TypeError):
|
||||
pass
|
||||
|
||||
# Secondary: result.data.message.content[0].text
|
||||
try:
|
||||
text = result["result"]["data"]["message"]["content"][0]["text"]
|
||||
if text:
|
||||
return str(text)
|
||||
except (KeyError, IndexError, TypeError):
|
||||
pass
|
||||
|
||||
# Tertiary: results as a raw string
|
||||
results = result.get("results")
|
||||
if results and isinstance(results, str):
|
||||
return results
|
||||
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_a2a_message_response(a2a_response: Dict[str, Any]) -> str:
|
||||
result = a2a_response.get("result")
|
||||
if not isinstance(result, dict):
|
||||
verbose_logger.warning("WXO: A2A response missing result object")
|
||||
return ""
|
||||
parts = result.get("parts")
|
||||
if not isinstance(parts, list):
|
||||
verbose_logger.warning("WXO: A2A result has no parts list")
|
||||
return ""
|
||||
for part in parts:
|
||||
if (
|
||||
isinstance(part, dict)
|
||||
and part.get("kind") == "text"
|
||||
and part.get("text")
|
||||
):
|
||||
return str(part["text"])
|
||||
verbose_logger.warning("WXO: A2A result parts contained no text")
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def build_a2a_message_response(request_id: str, text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Build a standard A2A non-streaming SendMessageResponse (kind=message).
|
||||
"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"kind": "message",
|
||||
"role": "agent",
|
||||
"parts": [{"kind": "text", "text": text}],
|
||||
"messageId": str(uuid4()),
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
async def fake_streaming_from_text(
|
||||
text: str,
|
||||
request_id: str,
|
||||
chunk_size: int = 50,
|
||||
delay_ms: int = 10,
|
||||
) -> AsyncIterator[Dict[str, Any]]:
|
||||
"""
|
||||
Emit standard A2A streaming events from a completed text response.
|
||||
|
||||
Event sequence:
|
||||
1. task (kind="task", state="submitted")
|
||||
2. status-update (kind="status-update", state="working")
|
||||
3. artifact-update chunks
|
||||
4. status-update (kind="status-update", state="completed", final=True)
|
||||
"""
|
||||
task_id = str(uuid4())
|
||||
context_id = str(uuid4())
|
||||
artifact_id = str(uuid4())
|
||||
|
||||
# 1. Task submitted
|
||||
yield {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"contextId": context_id,
|
||||
"id": task_id,
|
||||
"kind": "task",
|
||||
"status": {"state": "submitted"},
|
||||
},
|
||||
}
|
||||
|
||||
# 2. Working
|
||||
yield {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"contextId": context_id,
|
||||
"final": False,
|
||||
"kind": "status-update",
|
||||
"status": {"state": "working"},
|
||||
"taskId": task_id,
|
||||
},
|
||||
}
|
||||
await asyncio.sleep(delay_ms / 1000.0)
|
||||
|
||||
# 3. Artifact chunks (always emit at least one chunk, even for empty text)
|
||||
text_to_chunk = text or ""
|
||||
for i in range(0, max(len(text_to_chunk), 1), chunk_size):
|
||||
chunk_text = text_to_chunk[i : i + chunk_size]
|
||||
is_last = (i + chunk_size) >= max(len(text_to_chunk), 1)
|
||||
yield {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"contextId": context_id,
|
||||
"kind": "artifact-update",
|
||||
"taskId": task_id,
|
||||
"artifact": {
|
||||
"artifactId": artifact_id,
|
||||
"parts": [{"kind": "text", "text": chunk_text}],
|
||||
},
|
||||
},
|
||||
}
|
||||
if not is_last:
|
||||
await asyncio.sleep(delay_ms / 1000.0)
|
||||
|
||||
# 4. Completed
|
||||
yield {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"contextId": context_id,
|
||||
"final": True,
|
||||
"kind": "status-update",
|
||||
"status": {"state": "completed"},
|
||||
"taskId": task_id,
|
||||
},
|
||||
}
|
||||
|
||||
verbose_logger.debug(
|
||||
f"WXO: Fake streaming completed for request_id={request_id}"
|
||||
)
|
||||
|
|
@ -60,6 +60,12 @@ class A2ARequestUtils:
|
|||
if not isinstance(result, dict):
|
||||
return ""
|
||||
|
||||
# Direct message format (A2A spec): detect by explicit kind tag only.
|
||||
# The "parts" heuristic is too broad and would match any future result
|
||||
# type that happens to include a "parts" field.
|
||||
if result.get("kind") == "message":
|
||||
return A2ARequestUtils.extract_text_from_message(result)
|
||||
|
||||
message = result.get("message", {})
|
||||
return A2ARequestUtils.extract_text_from_message(message)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from typing import Dict, Optional
|
|||
|
||||
from .exceptions import AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
ANTHROPIC_ERROR_TYPE_MAP: Dict[int, AnthropicErrorType] = {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
from typing_extensions import Literal, Required, TypedDict
|
||||
|
||||
|
||||
# Known Anthropic error types
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
AnthropicErrorType = Literal[
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ This is an __init__.py file to allow the following interface
|
|||
|
||||
"""
|
||||
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
|
||||
from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional, Union
|
||||
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
|
||||
anthropic_messages as _async_anthropic_messages,
|
||||
|
|
@ -100,8 +100,11 @@ def create(
|
|||
**kwargs,
|
||||
) -> Union[
|
||||
AnthropicMessagesResponse,
|
||||
Iterator[bytes],
|
||||
AsyncIterator[Any],
|
||||
Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]],
|
||||
Coroutine[
|
||||
Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]
|
||||
],
|
||||
]:
|
||||
"""
|
||||
Async wrapper for Anthropic's messages API
|
||||
|
|
|
|||
|
|
@ -182,6 +182,14 @@ class ResponsesToCompletionBridgeHandler:
|
|||
client=kwargs.get("client"),
|
||||
)
|
||||
|
||||
# Pin the resolved provider so `responses()` doesn't re-run
|
||||
# `get_llm_provider()` on the model string and strip a second
|
||||
# provider prefix (see GitHub issue #28505). request_data already
|
||||
# carries `custom_llm_provider` via the spread of
|
||||
# `sanitized_litellm_params`; overwriting it on the dict (rather
|
||||
# than adding an explicit kwarg) avoids the duplicate-keyword
|
||||
# TypeError that would otherwise fire on the real bridge path.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
result = responses(
|
||||
**request_data,
|
||||
)
|
||||
|
|
@ -268,6 +276,13 @@ class ResponsesToCompletionBridgeHandler:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
# Pin the resolved provider so `aresponses()` doesn't re-run
|
||||
# `get_llm_provider()` on the model string and strip a second
|
||||
# provider prefix (see GitHub issue #28505). Set on request_data
|
||||
# rather than passed as a separate kwarg to avoid the duplicate-
|
||||
# keyword TypeError when `sanitized_litellm_params` already
|
||||
# carries `custom_llm_provider`.
|
||||
request_data["custom_llm_provider"] = custom_llm_provider
|
||||
result = await aresponses(
|
||||
**request_data,
|
||||
aresponses=True,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
|||
from litellm.llms.base_llm.bridges.completion_transformation import (
|
||||
CompletionTransformationBridge,
|
||||
)
|
||||
from litellm.responses.sse_output_recovery import (
|
||||
parse_sse_json_chunk,
|
||||
record_output_item_chunk,
|
||||
record_output_text_chunk,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionReasoningItem,
|
||||
|
|
@ -97,7 +102,7 @@ def _build_reasoning_item(
|
|||
|
||||
|
||||
def _reasoning_item_to_response_input(
|
||||
r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]]
|
||||
r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
|
||||
r_input: Dict[str, Any] = {
|
||||
|
|
@ -601,6 +606,79 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
return choices
|
||||
|
||||
@classmethod
|
||||
def _extract_output_from_completed_event(
|
||||
cls, parsed_chunk: Dict[str, Any]
|
||||
) -> Optional[List[Dict[str, Any]]]:
|
||||
response_payload = parsed_chunk.get("response")
|
||||
if not isinstance(response_payload, dict):
|
||||
return None
|
||||
response_output = response_payload.get("output")
|
||||
if not isinstance(response_output, list) or len(response_output) == 0:
|
||||
return None
|
||||
return cast(List[Dict[str, Any]], response_output)
|
||||
|
||||
@classmethod
|
||||
def _recover_output_items_from_raw_sse(
|
||||
cls, raw_sse: Optional[str]
|
||||
) -> List[Dict[str, Any]]:
|
||||
if not raw_sse or not isinstance(raw_sse, str):
|
||||
return []
|
||||
|
||||
recovered_output_items: Dict[int, Dict[str, Any]] = {}
|
||||
recovered_text_only_items: Dict[int, Dict[str, Any]] = {}
|
||||
|
||||
for chunk in raw_sse.splitlines():
|
||||
parsed_chunk = parse_sse_json_chunk(chunk)
|
||||
if parsed_chunk is None:
|
||||
continue
|
||||
|
||||
event_type = parsed_chunk.get("type")
|
||||
|
||||
if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
|
||||
recovered_output = cls._extract_output_from_completed_event(
|
||||
parsed_chunk
|
||||
)
|
||||
if recovered_output is not None:
|
||||
return recovered_output
|
||||
continue
|
||||
|
||||
if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
|
||||
record_output_item_chunk(
|
||||
parsed_chunk=parsed_chunk,
|
||||
output_items=recovered_output_items,
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE:
|
||||
record_output_text_chunk(
|
||||
parsed_chunk=parsed_chunk,
|
||||
output_items=recovered_output_items,
|
||||
text_only_items=recovered_text_only_items,
|
||||
)
|
||||
continue
|
||||
|
||||
# Merge text-only items into the recovered output items. Real
|
||||
# OUTPUT_ITEM_DONE events take precedence at any given output_index,
|
||||
# but text-only items at indices without a matching OUTPUT_ITEM_DONE
|
||||
# must still be preserved (e.g. multi-output responses where some
|
||||
# indices only emitted OUTPUT_TEXT_DONE).
|
||||
merged_items: Dict[int, Dict[str, Any]] = {**recovered_text_only_items}
|
||||
merged_items.update(recovered_output_items)
|
||||
|
||||
if merged_items:
|
||||
return [item for _, item in sorted(merged_items.items())]
|
||||
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def _recover_output_items_from_logging(
|
||||
cls, logging_obj: "LiteLLMLoggingObj"
|
||||
) -> List[Dict[str, Any]]:
|
||||
model_call_details = getattr(logging_obj, "model_call_details", {}) or {}
|
||||
original_response = model_call_details.get("original_response")
|
||||
return cls._recover_output_items_from_raw_sse(original_response)
|
||||
|
||||
def transform_response( # noqa: PLR0915
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -625,9 +703,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
if raw_response.error is not None:
|
||||
raise ValueError(f"Error in response: {raw_response.error}")
|
||||
|
||||
output_items = raw_response.output
|
||||
if len(output_items) == 0:
|
||||
recovered_output_items = self._recover_output_items_from_logging(
|
||||
logging_obj
|
||||
)
|
||||
if recovered_output_items:
|
||||
output_items = cast(Any, recovered_output_items)
|
||||
raw_response.output = cast(Any, recovered_output_items)
|
||||
verbose_logger.warning(
|
||||
"Recovered empty Responses API output from raw SSE for model=%s",
|
||||
model,
|
||||
)
|
||||
|
||||
# Convert response output to choices using the static helper
|
||||
choices = self._convert_response_output_to_choices(
|
||||
output_items=raw_response.output,
|
||||
output_items=output_items,
|
||||
handle_raw_dict_callback=self._handle_raw_dict_response_item,
|
||||
)
|
||||
|
||||
|
|
@ -641,7 +732,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown items in responses API response: {raw_response.output}"
|
||||
f"Unknown items in responses API response: {output_items}"
|
||||
)
|
||||
|
||||
setattr(model_response, "choices", choices)
|
||||
|
|
@ -1237,7 +1328,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
raise ValueError(
|
||||
f"Chat provider: Invalid function argument delta {parsed_chunk}"
|
||||
)
|
||||
elif event_type == "response.output_item.done":
|
||||
elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
|
||||
# New output item added
|
||||
output_item = parsed_chunk.get("item", {})
|
||||
if output_item.get("type") == "function_call":
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ Auto-detect content type per message: code, JSON, or text.
|
|||
import json
|
||||
import re
|
||||
|
||||
|
||||
_CODE_KEYWORDS = re.compile(
|
||||
r"\b(?:def |function |class |import |from |require\(|#include|fn |func |const |let |var |public |private |static )\b"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -585,6 +585,7 @@ LITELLM_CHAT_PROVIDERS = [
|
|||
"volcengine",
|
||||
"codestral",
|
||||
"text-completion-codestral",
|
||||
"text-completion-inception",
|
||||
"deepseek",
|
||||
"sambanova",
|
||||
"maritalk",
|
||||
|
|
@ -620,6 +621,7 @@ LITELLM_CHAT_PROVIDERS = [
|
|||
"oci",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"vercel_ai_gateway",
|
||||
"wandb",
|
||||
"ovhcloud",
|
||||
|
|
@ -676,6 +678,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [
|
|||
"extra_headers",
|
||||
"thinking",
|
||||
"web_search_options",
|
||||
"include_server_side_tool_invocations",
|
||||
"service_tier",
|
||||
"prompt_cache_key",
|
||||
"prompt_cache_retention",
|
||||
|
|
@ -737,6 +740,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
|
|||
"verbosity": None,
|
||||
"thinking": None,
|
||||
"web_search_options": None,
|
||||
"include_server_side_tool_invocations": None,
|
||||
"service_tier": None,
|
||||
"safety_identifier": None,
|
||||
"prompt_cache_key": None,
|
||||
|
|
@ -771,6 +775,7 @@ openai_compatible_endpoints: List = [
|
|||
"https://api.moonshot.ai/v1",
|
||||
"https://api.publicai.co/v1",
|
||||
"https://api.synthetic.new/openai/v1",
|
||||
"https://serverless.tensormesh.ai/v1",
|
||||
"https://api.stima.tech/v1",
|
||||
"https://nano-gpt.com/api/v1",
|
||||
"https://api.poe.com/v1",
|
||||
|
|
@ -778,6 +783,7 @@ openai_compatible_endpoints: List = [
|
|||
"https://api.v0.dev/v1",
|
||||
"https://api.morphllm.com/v1",
|
||||
"https://api.lambda.ai/v1",
|
||||
"https://api.inceptionlabs.ai/v1",
|
||||
"https://api.hyperbolic.xyz/v1",
|
||||
"https://ai-gateway.helicone.ai/",
|
||||
"https://ai-gateway.vercel.sh/v1",
|
||||
|
|
@ -820,6 +826,7 @@ openai_compatible_providers: List = [
|
|||
"meta_llama",
|
||||
"publicai", # PublicAI - JSON-configured provider
|
||||
"synthetic", # Synthetic - JSON-configured provider
|
||||
"tensormesh", # Tensormesh - JSON-configured provider
|
||||
"apertis", # Apertis - JSON-configured provider
|
||||
"nano-gpt", # Nano-GPT - JSON-configured provider
|
||||
"poe", # Poe - JSON-configured provider
|
||||
|
|
@ -833,6 +840,7 @@ openai_compatible_providers: List = [
|
|||
"helicone",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"hyperbolic",
|
||||
"vercel_ai_gateway",
|
||||
"aiml",
|
||||
|
|
@ -855,6 +863,7 @@ openai_text_completion_compatible_providers: List = (
|
|||
"moonshot",
|
||||
"publicai",
|
||||
"synthetic",
|
||||
"tensormesh",
|
||||
"apertis",
|
||||
"nano-gpt",
|
||||
"poe",
|
||||
|
|
@ -868,6 +877,7 @@ openai_text_completion_compatible_providers: List = (
|
|||
_openai_like_providers: List = [
|
||||
"predibase",
|
||||
"databricks",
|
||||
"lemonade",
|
||||
"watsonx",
|
||||
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
|
||||
# well supported replicate llms
|
||||
|
|
@ -1147,6 +1157,7 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
"openai.gpt-oss-120b-1:0",
|
||||
"anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
"anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
"anthropic.claude-opus-4-8",
|
||||
"anthropic.claude-opus-4-7",
|
||||
"anthropic.claude-opus-4-6-v1:0",
|
||||
"anthropic.claude-opus-4-6-v1",
|
||||
|
|
@ -1408,6 +1419,13 @@ LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
|
|||
# Prometheus metrics, audit trails, or any other downstream consumer.
|
||||
LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
|
||||
|
||||
# Marker placed in ``model_call_details`` on a synthetic ``Logging`` object that
|
||||
# records a proxy-gate error (auth/rate-limit rejection) for a request that never
|
||||
# reached an upstream provider. Tracing callbacks key off it to avoid fabricating
|
||||
# an LLM-call span for a call that did not happen. See
|
||||
# ``ProxyLogging._handle_logging_proxy_only_error``.
|
||||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL = "litellm_no_upstream_llm_call"
|
||||
|
||||
# Key Rotation Constants
|
||||
LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
|
||||
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(
|
||||
|
|
@ -1443,6 +1461,12 @@ CLI_JWT_EXPIRATION_HOURS = int(
|
|||
or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
|
||||
or 24
|
||||
)
|
||||
# Comma-separated allowlisted OIDC claim map for CLI SSO polling, e.g.
|
||||
# "employment_type->acme_employment_type,org_info.department->department"
|
||||
CLI_SSO_CLAIM_MAP = (
|
||||
os.getenv("CLI_SSO_CLAIM_MAP") or os.getenv("LITELLM_CLI_SSO_CLAIM_MAP") or ""
|
||||
)
|
||||
CLI_SSO_CLAIM_MAX_SCALAR_LENGTH = 1024
|
||||
|
||||
########################### UI SESSION DURATION ###########################
|
||||
# Duration for UI login session (username/password, SSO, invitation links). Format: "30s", "30m", "24h", "7d"
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
CostCalculatorUtils,
|
||||
_generic_cost_per_character,
|
||||
_get_regional_uplift_multiplier,
|
||||
_get_service_tier_cost_key,
|
||||
_parse_prompt_tokens_details,
|
||||
calculate_cost_component,
|
||||
|
|
@ -132,6 +133,8 @@ _VIDEO_CALL_TYPES = frozenset(
|
|||
{
|
||||
CallTypes.create_video.value,
|
||||
CallTypes.acreate_video.value,
|
||||
CallTypes.video_edit.value,
|
||||
CallTypes.avideo_edit.value,
|
||||
CallTypes.video_remix.value,
|
||||
CallTypes.avideo_remix.value,
|
||||
}
|
||||
|
|
@ -312,6 +315,10 @@ def cost_per_token( # noqa: PLR0915
|
|||
audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds
|
||||
### SERVICE TIER ###
|
||||
service_tier: Optional[str] = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
data_residency: Optional[
|
||||
str
|
||||
] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us")
|
||||
response: Optional[Any] = None,
|
||||
### REQUEST MODEL ###
|
||||
request_model: Optional[str] = None, # original request model for router detection
|
||||
|
|
@ -412,9 +419,36 @@ def cost_per_token( # noqa: PLR0915
|
|||
prompt_tokens_cost_usd_dollar: float = 0
|
||||
completion_tokens_cost_usd_dollar: float = 0
|
||||
model_cost_ref = litellm.model_cost
|
||||
# Only callers that explicitly pass `custom_llm_provider` get the
|
||||
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
|
||||
# behavior: `model_with_provider` stays equal to the raw `model` string
|
||||
# (provider is detected below for downstream use only).
|
||||
caller_supplied_provider = custom_llm_provider is not None
|
||||
|
||||
# `model` is normally a string, but callers that mock the transport can pass
|
||||
# non-string objects. Only run the string-based dedup/prefix-join when it is
|
||||
# actually a string — e.g. a MagicMock's `.startswith()` is always truthy and
|
||||
# its slices return new mocks, which would spin the dedup loop forever.
|
||||
model_is_str = isinstance(model, str)
|
||||
|
||||
# Router/proxy deployments may repeat the provider segment (e.g. model_name
|
||||
# "openai/openai/gpt-5.5"). Strip duplicated `{provider}/` chains before joining.
|
||||
if caller_supplied_provider and model_is_str:
|
||||
_dup_prefix = f"{custom_llm_provider}/"
|
||||
while model.startswith(_dup_prefix):
|
||||
_remainder = model[len(_dup_prefix) :]
|
||||
if _remainder.startswith(_dup_prefix):
|
||||
model = _remainder
|
||||
else:
|
||||
break
|
||||
|
||||
model_with_provider = model
|
||||
if custom_llm_provider is not None:
|
||||
model_with_provider = custom_llm_provider + "/" + model
|
||||
if caller_supplied_provider:
|
||||
_prov_prefix = f"{custom_llm_provider}/"
|
||||
if model_is_str and model.startswith(_prov_prefix):
|
||||
model_with_provider = model
|
||||
else:
|
||||
model_with_provider = f"{custom_llm_provider}/{model}"
|
||||
if region_name is not None:
|
||||
model_with_provider_and_region = (
|
||||
f"{custom_llm_provider}/{region_name}/{model}"
|
||||
|
|
@ -425,6 +459,9 @@ def cost_per_token( # noqa: PLR0915
|
|||
model_with_provider = model_with_provider_and_region
|
||||
else:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
|
||||
assert custom_llm_provider is not None # caller-supplied or get_llm_provider
|
||||
|
||||
model_without_prefix = model
|
||||
model_parts = model.split("/", 1)
|
||||
if len(model_parts) > 1:
|
||||
|
|
@ -493,6 +530,7 @@ def cost_per_token( # noqa: PLR0915
|
|||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
|
@ -521,7 +559,10 @@ def cost_per_token( # noqa: PLR0915
|
|||
or call_type == CallTypes.retrieve_batch
|
||||
):
|
||||
return batch_cost_calculator(
|
||||
usage=usage_block, model=model, custom_llm_provider=custom_llm_provider
|
||||
usage=usage_block,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
|
|
@ -529,6 +570,7 @@ def cost_per_token( # noqa: PLR0915
|
|||
model=model_without_prefix,
|
||||
usage=usage_block,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
|
||||
return openai_cost_per_second(
|
||||
|
|
@ -579,7 +621,10 @@ def cost_per_token( # noqa: PLR0915
|
|||
)
|
||||
elif custom_llm_provider == "openai":
|
||||
return openai_cost_per_token(
|
||||
model=model, usage=usage_block, service_tier=service_tier
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
elif custom_llm_provider == "databricks":
|
||||
return databricks_cost_per_token(model=model, usage=usage_block)
|
||||
|
|
@ -631,6 +676,7 @@ def cost_per_token( # noqa: PLR0915
|
|||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
|
||||
if (
|
||||
|
|
@ -1117,6 +1163,10 @@ def completion_cost( # noqa: PLR0915
|
|||
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: Optional[str] = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
data_residency: Optional[
|
||||
str
|
||||
] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us")
|
||||
) -> float:
|
||||
"""
|
||||
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
|
||||
|
|
@ -1516,6 +1566,7 @@ def completion_cost( # noqa: PLR0915
|
|||
combined_usage_object=cost_per_token_usage_object,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_model_name=model,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
elif call_type == _MCP_CALL_TYPE:
|
||||
from litellm.proxy._experimental.mcp_server.cost_calculator import (
|
||||
|
|
@ -1600,6 +1651,7 @@ def completion_cost( # noqa: PLR0915
|
|||
audio_transcription_file_duration=audio_transcription_file_duration,
|
||||
rerank_billed_units=rerank_billed_units,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
response=completion_response,
|
||||
request_model=request_model_for_cost,
|
||||
)
|
||||
|
|
@ -1811,6 +1863,10 @@ def response_cost_calculator(
|
|||
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: Optional[str] = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
data_residency: Optional[
|
||||
str
|
||||
] = None, # for OpenAI regional-processing uplift (e.g. "eu", "us")
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -1844,6 +1900,7 @@ def response_cost_calculator(
|
|||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
|
|
@ -1879,10 +1936,6 @@ def ocr_cost(
|
|||
if response.usage_info is None:
|
||||
raise ValueError("OCR response usage_info is None")
|
||||
|
||||
pages_processed = response.usage_info.pages_processed
|
||||
if pages_processed is None:
|
||||
raise ValueError("OCR response pages_processed is None")
|
||||
|
||||
try:
|
||||
model_info: Optional[ModelInfo] = litellm.get_model_info(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
@ -1890,9 +1943,49 @@ def ocr_cost(
|
|||
except Exception:
|
||||
model_info = None
|
||||
|
||||
ocr_cost_per_page: float = 0.0
|
||||
credits = getattr(response.usage_info, "credits", None)
|
||||
cost_per_credit = None
|
||||
if model_info is not None:
|
||||
ocr_cost_per_page = model_info.get("ocr_cost_per_page") or 0.0
|
||||
cost_per_credit = model_info.get("ocr_cost_per_credit")
|
||||
if credits is not None and cost_per_credit is not None:
|
||||
return cost_per_credit * credits, 0.0
|
||||
|
||||
ocr_cost_per_page: Optional[float] = None
|
||||
if model_info is not None:
|
||||
ocr_cost_per_page = model_info.get("ocr_cost_per_page")
|
||||
|
||||
pages_processed = response.usage_info.pages_processed
|
||||
if pages_processed is None:
|
||||
if cost_per_credit is not None or ocr_cost_per_page is None:
|
||||
# Surface missing usage data instead of silently under-reporting
|
||||
# cost. The previous behavior raised ValueError; we now return 0.0
|
||||
# for credit-priced or unpriced models, so log a warning to keep
|
||||
# the regression visible to operators.
|
||||
verbose_logger.warning(
|
||||
"OCR cost: model=%s custom_llm_provider=%s response.usage_info."
|
||||
"pages_processed is None and credits=%s; returning 0.0 cost.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
credits,
|
||||
)
|
||||
return 0.0, 0.0
|
||||
raise ValueError("OCR response pages_processed is None")
|
||||
|
||||
if ocr_cost_per_page is None:
|
||||
# No per-page pricing configured. Either the model is on credit-based
|
||||
# pricing (and credits weren't returned, so the credit branch above did
|
||||
# not match) or the model has no OCR pricing entry at all. Surface a
|
||||
# warning so that missing pricing entries are visible rather than
|
||||
# silently producing zero cost for billable usage.
|
||||
verbose_logger.warning(
|
||||
"OCR cost: model=%s custom_llm_provider=%s reported "
|
||||
"pages_processed=%s but no ocr_cost_per_page is configured; "
|
||||
"returning 0.0 cost.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
pages_processed,
|
||||
)
|
||||
return 0.0, 0.0
|
||||
|
||||
total_ocr_processing_cost: float = ocr_cost_per_page * pages_processed
|
||||
return total_ocr_processing_cost, 0.0
|
||||
|
|
@ -2166,6 +2259,7 @@ def batch_cost_calculator(
|
|||
model: str,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
model_info: Optional[ModelInfo] = None,
|
||||
data_residency: Optional[str] = None,
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculate the cost of a batch job.
|
||||
|
|
@ -2250,6 +2344,11 @@ def batch_cost_calculator(
|
|||
usage.completion_tokens * (output_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
uplift = _get_regional_uplift_multiplier(model_info, data_residency)
|
||||
if uplift != 1.0:
|
||||
total_prompt_cost *= uplift
|
||||
total_completion_cost *= uplift
|
||||
|
||||
return total_prompt_cost, total_completion_cost
|
||||
|
||||
|
||||
|
|
@ -2395,6 +2494,7 @@ def handle_realtime_stream_cost_calculation(
|
|||
combined_usage_object: Usage,
|
||||
custom_llm_provider: str,
|
||||
litellm_model_name: str,
|
||||
data_residency: Optional[str] = None,
|
||||
) -> float:
|
||||
"""
|
||||
Handles the cost calculation for realtime stream responses.
|
||||
|
|
@ -2425,6 +2525,7 @@ def handle_realtime_stream_cost_calculation(
|
|||
model=model_name,
|
||||
usage=combined_usage_object,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -421,8 +421,16 @@ class MCPClient:
|
|||
|
||||
return factory
|
||||
|
||||
async def list_tools(self) -> List[MCPTool]:
|
||||
"""List available tools from the server."""
|
||||
async def list_tools(self, raise_on_error: bool = False) -> List[MCPTool]:
|
||||
"""List available tools from the server.
|
||||
|
||||
Args:
|
||||
raise_on_error: When True, re-raise exceptions instead of returning
|
||||
an empty list. Used by the proxy's pass-through MCP flow so it
|
||||
can surface upstream HTTP 401 responses as a proper 401 to the
|
||||
MCP client (triggering the upstream OAuth flow) rather than
|
||||
masking them as "connected, no tools".
|
||||
"""
|
||||
verbose_logger.debug(
|
||||
f"MCP client listing tools from {self.server_url or 'stdio'}"
|
||||
)
|
||||
|
|
@ -458,6 +466,8 @@ class MCPClient:
|
|||
"the MCP server may have crashed, disconnected, or timed out"
|
||||
)
|
||||
|
||||
if raise_on_error:
|
||||
raise
|
||||
# Return empty list instead of raising to allow graceful degradation
|
||||
return []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from typing import AsyncIterator, Dict, Iterator, Literal, NamedTuple, Union
|
||||
|
||||
|
||||
FileContentProvider = Literal[
|
||||
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic", "manus"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
"""
|
||||
Google GenAI Adapters for LiteLLM
|
||||
|
||||
This module provides adapters for transforming Google GenAI generate_content requests
|
||||
This module provides adapters for transforming Google GenAI generate_content requests
|
||||
to/from LiteLLM completion format with full support for:
|
||||
- Text content transformation
|
||||
- Tool calling (function declarations, function calls, function responses)
|
||||
- Tool calling (function declarations, function calls, function responses)
|
||||
- Streaming (both regular and tool calling)
|
||||
- Mixed content (text + tool calls)
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
"""
|
||||
Handles Batching + sending Httpx Post requests to slack
|
||||
Handles Batching + sending Httpx Post requests to slack
|
||||
|
||||
Slack alerts are sent every 10s or when events are greater than X events
|
||||
Slack alerts are sent every 10s or when events are greater than X events
|
||||
|
||||
see custom_batch_logger.py for more details / defaults
|
||||
see custom_batch_logger.py for more details / defaults
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ else:
|
|||
|
||||
|
||||
def process_slack_alerting_variables(
|
||||
alert_to_webhook_url: Optional[Dict[AlertType, Union[List[str], str]]]
|
||||
alert_to_webhook_url: Optional[Dict[AlertType, Union[List[str], str]]],
|
||||
) -> Optional[Dict[AlertType, Union[List[str], str]]]:
|
||||
"""
|
||||
process alert_to_webhook_url
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
Base class for Additional Logging Utils for CustomLoggers
|
||||
Base class for Additional Logging Utils for CustomLoggers
|
||||
|
||||
- Health Check for the logging util
|
||||
- Get Request / Response Payload for the logging util
|
||||
|
|
|
|||
|
|
@ -8,18 +8,23 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
|
|||
BaseLLMObsOTELAttributes,
|
||||
safe_set_attribute,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
should_redact_message_logging,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span
|
||||
from litellm.integrations._types.open_inference import (
|
||||
MessageAttributes,
|
||||
ImageAttributes,
|
||||
SpanAttributes,
|
||||
AudioAttributes,
|
||||
EmbeddingAttributes,
|
||||
ImageAttributes,
|
||||
MessageAttributes,
|
||||
MessageContentAttributes,
|
||||
OpenInferenceSpanKindValues,
|
||||
SpanAttributes,
|
||||
ToolCallAttributes,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -53,40 +58,24 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes):
|
|||
msg.get("content", ""),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@override
|
||||
def set_response_output_messages(span: "Span", response_obj):
|
||||
"""
|
||||
Sets output message attributes on the span from the LLM response.
|
||||
Args:
|
||||
span: The OpenTelemetry span to set attributes on
|
||||
response_obj: The response object containing choices with messages
|
||||
"""
|
||||
from litellm.integrations._types.open_inference import (
|
||||
MessageAttributes,
|
||||
SpanAttributes,
|
||||
)
|
||||
# Additive: emit structured tool_calls / multimodal content
|
||||
# so Arize/Phoenix can render tool-using and image-bearing
|
||||
# turns. These set NEW attribute keys (MESSAGE_TOOL_CALLS /
|
||||
# MESSAGE_NAME / MESSAGE_TOOL_CALL_ID / MESSAGE_CONTENTS.*) —
|
||||
# never replace the MESSAGE_CONTENT write above.
|
||||
_safe_emit(
|
||||
f"input message extras (idx={idx})",
|
||||
_emit_input_message_extras,
|
||||
span,
|
||||
prefix,
|
||||
msg,
|
||||
)
|
||||
|
||||
for idx, choice in enumerate(response_obj.get("choices", [])):
|
||||
response_message = choice.get("message", {})
|
||||
safe_set_attribute(
|
||||
span,
|
||||
SpanAttributes.OUTPUT_VALUE,
|
||||
response_message.get("content", ""),
|
||||
)
|
||||
|
||||
# This shows up under `output_messages` tab on the span page.
|
||||
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{idx}"
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
|
||||
response_message.get("role"),
|
||||
)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
|
||||
response_message.get("content", ""),
|
||||
)
|
||||
# Note: `BaseLLMObsOTELAttributes.set_response_output_messages` is not
|
||||
# overridden here. The live code path uses `_set_choice_outputs` (called
|
||||
# via `_set_response_attributes` from `set_attributes`) which handles
|
||||
# tool_calls, multimodal output, embeddings, audio, images, and structured
|
||||
# outputs in a single place.
|
||||
|
||||
|
||||
def _set_response_attributes(span: "Span", response_obj):
|
||||
|
|
@ -106,11 +95,17 @@ def _set_response_attributes(span: "Span", response_obj):
|
|||
def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
|
||||
for idx, choice in enumerate(response_obj.get("choices", [])):
|
||||
response_message = choice.get("message", {})
|
||||
safe_set_attribute(
|
||||
span,
|
||||
span_attrs.OUTPUT_VALUE,
|
||||
response_message.get("content", ""),
|
||||
)
|
||||
content = response_message.get("content", "")
|
||||
|
||||
# Tool-only assistant responses have empty content; serialize the
|
||||
# tool_calls into OUTPUT_VALUE so Arize's "Output" pane isn't blank.
|
||||
output_value = content
|
||||
if not output_value:
|
||||
tool_calls = _get_tool_calls(response_message)
|
||||
if tool_calls:
|
||||
output_value = _summarize_tool_calls_for_output(tool_calls)
|
||||
|
||||
safe_set_attribute(span, span_attrs.OUTPUT_VALUE, output_value)
|
||||
prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{idx}"
|
||||
safe_set_attribute(
|
||||
span,
|
||||
|
|
@ -120,7 +115,18 @@ def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
|
|||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{msg_attrs.MESSAGE_CONTENT}",
|
||||
response_message.get("content", ""),
|
||||
content,
|
||||
)
|
||||
|
||||
# Additive: emit assistant tool_calls so tool-using turns render in
|
||||
# Arize/Phoenix. Sets new MESSAGE_TOOL_CALLS keys only — does not
|
||||
# change MESSAGE_CONTENT/MESSAGE_ROLE writes above.
|
||||
_safe_emit(
|
||||
f"output tool_calls (idx={idx})",
|
||||
_emit_message_tool_calls,
|
||||
span,
|
||||
prefix,
|
||||
response_message,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -278,6 +284,43 @@ def _set_usage_outputs(span: "Span", response_obj, span_attrs):
|
|||
reasoning_tokens,
|
||||
)
|
||||
|
||||
# Additive: cache token breakdown so prompt-caching savings render in
|
||||
# Arize. Sources covered:
|
||||
# - OpenAI Chat Completions: `prompt_tokens_details.cached_tokens`
|
||||
# - Anthropic / Bedrock-Anthropic: `cache_read_input_tokens`,
|
||||
# `cache_creation_input_tokens`
|
||||
# All emits are conditional, so when none of these fields exist (the
|
||||
# situation in the existing test fixtures) no extra attributes are set.
|
||||
prompt_token_details = _safe_get(usage, "prompt_tokens_details") or _safe_get(
|
||||
usage, "input_tokens_details"
|
||||
)
|
||||
cache_read = _safe_get(prompt_token_details, "cached_tokens") or _safe_get(
|
||||
usage, "cache_read_input_tokens"
|
||||
)
|
||||
if cache_read:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ,
|
||||
cache_read,
|
||||
)
|
||||
# Anthropic / Bedrock-Anthropic only — OpenAI's `prompt_tokens_details`
|
||||
# does not expose a cache-write count, so we read straight off `usage`.
|
||||
cache_write = _safe_get(usage, "cache_creation_input_tokens")
|
||||
if cache_write:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE,
|
||||
cache_write,
|
||||
)
|
||||
|
||||
audio_prompt_tokens = _safe_get(prompt_token_details, "audio_tokens")
|
||||
if audio_prompt_tokens:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_AUDIO,
|
||||
audio_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _infer_open_inference_span_kind(call_type: Optional[str]) -> str:
|
||||
"""
|
||||
|
|
@ -321,6 +364,10 @@ def _infer_open_inference_span_kind(call_type: Optional[str]) -> str:
|
|||
"videos",
|
||||
"realtime",
|
||||
"pass_through",
|
||||
# `passthrough` (no underscore) is what real call_types use:
|
||||
# `allm_passthrough_route`, `llm_passthrough_route`. Without
|
||||
# this they fell through to UNKNOWN, blanking span.kind.
|
||||
"passthrough",
|
||||
"anthropic_messages",
|
||||
"ocr",
|
||||
)
|
||||
|
|
@ -396,6 +443,18 @@ def set_attributes(
|
|||
"""
|
||||
Populates span with OpenInference-compliant LLM attributes for Arize and Phoenix tracing.
|
||||
"""
|
||||
# Coerce non-dict response objects (e.g. httpx.Response from passthrough
|
||||
# routes) into a dict so downstream `.get()` calls don't crash. Existing
|
||||
# dict / `.get()`-bearing objects (incl. Pydantic OpenAI Responses API
|
||||
# models) are returned unchanged, preserving the existing test behavior.
|
||||
response_obj_for_attrs = _coerce_response_obj_for_attrs(response_obj)
|
||||
|
||||
# Set span.kind defensively before anything else. If a downstream step
|
||||
# throws, the span still has a kind so Arize can render it correctly
|
||||
# (an LLM call instead of UNKNOWN). This is the single source of truth
|
||||
# for span.kind — no late re-write happens below.
|
||||
_safe_emit("early span kind", _set_early_span_kind, span, kwargs)
|
||||
|
||||
try:
|
||||
optional_params = _sanitize_optional_params(kwargs.get("optional_params"))
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
|
|
@ -415,25 +474,22 @@ def set_attributes(
|
|||
metadata_tools = _extract_metadata_tools(metadata)
|
||||
optional_tools = _extract_optional_tools(optional_params)
|
||||
|
||||
call_type = standard_logging_payload.get("call_type")
|
||||
_set_request_attributes(
|
||||
span=span,
|
||||
kwargs=kwargs,
|
||||
standard_logging_payload=standard_logging_payload,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
response_obj=response_obj,
|
||||
response_obj=response_obj_for_attrs,
|
||||
span_attrs=SpanAttributes,
|
||||
)
|
||||
|
||||
span_kind = _infer_open_inference_span_kind(call_type=call_type)
|
||||
# span.kind was already set above by `_set_early_span_kind`. We do
|
||||
# NOT re-write it here based on tool presence: a chat completion
|
||||
# that passes `tools=[...]` (or returns `tool_calls`) is still an
|
||||
# LLM call per the OpenInference spec — TOOL is reserved for actual
|
||||
# tool execution spans, not LLM calls that request tools.
|
||||
_set_tool_attributes(span, optional_tools, metadata_tools)
|
||||
if (
|
||||
optional_tools or metadata_tools
|
||||
) and span_kind != OpenInferenceSpanKindValues.TOOL.value:
|
||||
span_kind = OpenInferenceSpanKindValues.TOOL.value
|
||||
|
||||
safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, span_kind)
|
||||
attributes.set_messages(span, kwargs)
|
||||
|
||||
model_params = (
|
||||
|
|
@ -443,7 +499,7 @@ def set_attributes(
|
|||
)
|
||||
_set_model_params(span, model_params, SpanAttributes)
|
||||
|
||||
_set_response_attributes(span=span, response_obj=response_obj)
|
||||
_set_response_attributes(span=span, response_obj=response_obj_for_attrs)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.error(
|
||||
|
|
@ -452,6 +508,22 @@ def set_attributes(
|
|||
if hasattr(span, "record_exception"):
|
||||
span.record_exception(e)
|
||||
|
||||
# Additive emitters. Each is independently guarded so a failure can never
|
||||
# blank the attributes set by the main try-block above. New attributes are
|
||||
# written under new keys; existing attributes are not overwritten.
|
||||
slp = kwargs.get("standard_logging_object")
|
||||
_safe_emit("session/user attrs", _set_session_and_user_attrs, span, kwargs, slp)
|
||||
_safe_emit("response cost", _set_response_cost_attr, span, slp)
|
||||
_safe_emit(
|
||||
"passthrough normalization",
|
||||
_maybe_normalize_passthrough,
|
||||
span,
|
||||
kwargs,
|
||||
response_obj,
|
||||
response_obj_for_attrs,
|
||||
slp,
|
||||
)
|
||||
|
||||
|
||||
def _sanitize_optional_params(optional_params: Optional[dict]) -> dict:
|
||||
if not isinstance(optional_params, dict):
|
||||
|
|
@ -534,3 +606,529 @@ def _set_model_params(span: "Span", model_params: Optional[dict], span_attrs) ->
|
|||
user_id = model_params.get("user")
|
||||
if user_id is not None:
|
||||
safe_set_attribute(span, span_attrs.USER_ID, user_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Additive rendering helpers (introduced to enhance Arize/Phoenix rendering
|
||||
# without changing any previously-emitted attribute keys or values).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _safe_emit(label: str, fn, *args, **kwargs) -> None:
|
||||
"""Run an additive attribute emitter, swallowing any error so it cannot
|
||||
blank attributes set elsewhere on the span. Failures are logged at debug.
|
||||
"""
|
||||
try:
|
||||
fn(*args, **kwargs)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("[Arize] %s skipped: %s", label, e)
|
||||
|
||||
|
||||
def _set_early_span_kind(span: "Span", kwargs: dict) -> None:
|
||||
"""Defensively set OPENINFERENCE_SPAN_KIND before any other logic runs."""
|
||||
slp = kwargs.get("standard_logging_object")
|
||||
call_type = slp.get("call_type") if isinstance(slp, dict) else None
|
||||
safe_set_attribute(
|
||||
span,
|
||||
SpanAttributes.OPENINFERENCE_SPAN_KIND,
|
||||
_infer_open_inference_span_kind(call_type=call_type),
|
||||
)
|
||||
|
||||
|
||||
def _coerce_response_obj_for_attrs(response_obj):
|
||||
"""Return a `.get`-compatible view of `response_obj` when possible.
|
||||
|
||||
- dicts and Pydantic models that already expose `.get` are returned
|
||||
unchanged (preserves all current behavior, including the Responses API
|
||||
flow which relies on Pydantic attribute access).
|
||||
- `httpx.Response` and other text-only responses (passthrough routes)
|
||||
are JSON-decoded so the standard extraction paths can read fields like
|
||||
`id`, `model`, and `usage`. On failure the original object is returned
|
||||
so behavior is no worse than today.
|
||||
"""
|
||||
if response_obj is None or hasattr(response_obj, "get"):
|
||||
return response_obj
|
||||
text = getattr(response_obj, "text", None)
|
||||
if isinstance(text, str) and text:
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
pass
|
||||
return response_obj
|
||||
|
||||
|
||||
def _coerce_text(value) -> Optional[str]:
|
||||
"""Best-effort text extraction from a message-content value.
|
||||
|
||||
Returns None when no textual portion can be derived. Handles:
|
||||
- plain strings
|
||||
- lists of OpenAI-style content parts (`{"type": "text", "text": ...}`)
|
||||
- lists of Anthropic-style content parts (`{"type": "text", "text": ...}`
|
||||
or `{"type": "input_text", "text": ...}`)
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, list):
|
||||
parts = []
|
||||
for part in value:
|
||||
if isinstance(part, str):
|
||||
parts.append(part)
|
||||
elif isinstance(part, dict):
|
||||
text = part.get("text") or part.get("input_text")
|
||||
if isinstance(text, str):
|
||||
parts.append(text)
|
||||
if parts:
|
||||
return "\n".join(parts)
|
||||
return None
|
||||
|
||||
|
||||
def _to_plain_dict(value):
|
||||
"""Best-effort: coerce a value (Pydantic model / dict / None) to a dict.
|
||||
|
||||
Returns the original value when no safe conversion exists. Used to bridge
|
||||
OpenAI Pydantic message/tool_call objects into the dict-based helpers.
|
||||
"""
|
||||
if value is None or isinstance(value, dict):
|
||||
return value
|
||||
model_dump = getattr(value, "model_dump", None)
|
||||
if callable(model_dump):
|
||||
try:
|
||||
return model_dump()
|
||||
except Exception:
|
||||
pass
|
||||
return value
|
||||
|
||||
|
||||
def _get_tool_calls(message) -> Optional[list]:
|
||||
"""Return ``message.tool_calls`` only when it's a non-empty list.
|
||||
|
||||
Works for dicts and Pydantic message objects via ``_safe_get``.
|
||||
"""
|
||||
tool_calls = _safe_get(message, "tool_calls")
|
||||
return tool_calls if isinstance(tool_calls, list) and tool_calls else None
|
||||
|
||||
|
||||
def _normalize_tool_call(raw_tc) -> Optional[Dict[str, Any]]:
|
||||
"""Normalize a single tool_call (dict or Pydantic) into a stable shape:
|
||||
|
||||
{"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}}
|
||||
|
||||
Arguments are coerced to a JSON string per OpenInference convention.
|
||||
Returns ``None`` when ``raw_tc`` cannot be coerced to a dict.
|
||||
"""
|
||||
tc = _to_plain_dict(raw_tc)
|
||||
if not isinstance(tc, dict):
|
||||
return None
|
||||
function = _to_plain_dict(tc.get("function"))
|
||||
name = function.get("name") if isinstance(function, dict) else None
|
||||
args = function.get("arguments") if isinstance(function, dict) else None
|
||||
if args is not None and not isinstance(args, str):
|
||||
try:
|
||||
args = json.dumps(args)
|
||||
except Exception:
|
||||
args = str(args)
|
||||
return {
|
||||
"id": tc.get("id"),
|
||||
"type": tc.get("type", "function"),
|
||||
"function": {"name": name, "arguments": args},
|
||||
}
|
||||
|
||||
|
||||
def _summarize_tool_calls_for_output(tool_calls) -> str:
|
||||
"""Render a tool_calls list as a compact JSON string for OUTPUT_VALUE.
|
||||
|
||||
Best-effort: returns ``str(tool_calls)`` if anything unexpected happens
|
||||
so OUTPUT_VALUE is never blanked on a malformed payload.
|
||||
"""
|
||||
try:
|
||||
normalized = [n for n in (_normalize_tool_call(tc) for tc in tool_calls) if n]
|
||||
return json.dumps({"tool_calls": normalized})
|
||||
except Exception:
|
||||
return str(tool_calls)
|
||||
|
||||
|
||||
def _emit_message_tool_calls(span: "Span", prefix: str, message) -> None:
|
||||
"""Emit ``MESSAGE_TOOL_CALLS.*`` for an assistant message that requested
|
||||
tool calls. Pure addition: only writes when ``tool_calls`` is non-empty.
|
||||
|
||||
Accepts dicts or Pydantic message objects (e.g. ``litellm.Message``); the
|
||||
same applies to each tool_call entry.
|
||||
"""
|
||||
tool_calls = _get_tool_calls(message)
|
||||
if not tool_calls:
|
||||
return
|
||||
for tc_idx, raw_tc in enumerate(tool_calls):
|
||||
tc = _normalize_tool_call(raw_tc)
|
||||
if tc is None:
|
||||
continue
|
||||
tc_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tc_idx}"
|
||||
if tc["id"]:
|
||||
safe_set_attribute(
|
||||
span, f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tc["id"]
|
||||
)
|
||||
fn = tc["function"]
|
||||
if fn["name"]:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}",
|
||||
fn["name"],
|
||||
)
|
||||
if fn["arguments"] is not None:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}",
|
||||
fn["arguments"],
|
||||
)
|
||||
|
||||
|
||||
def _emit_input_message_extras(span: "Span", prefix: str, message: dict) -> None:
|
||||
"""Emit additive attributes for an input message:
|
||||
|
||||
- `MESSAGE_NAME` and `MESSAGE_TOOL_CALL_ID` (commonly set on tool-result
|
||||
messages so traces show which tool produced which result).
|
||||
- `MESSAGE_TOOL_CALLS.*` when an assistant message requested tools.
|
||||
- `MESSAGE_CONTENTS.*` structured content for list-shaped content
|
||||
(multimodal text + image parts). The plain `MESSAGE_CONTENT` write is
|
||||
still performed by the caller, so renderers that only read the legacy
|
||||
key continue to work.
|
||||
"""
|
||||
if not isinstance(message, dict):
|
||||
return
|
||||
|
||||
name = message.get("name")
|
||||
if name:
|
||||
safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_NAME}", name)
|
||||
|
||||
tool_call_id = message.get("tool_call_id")
|
||||
if tool_call_id:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALL_ID}",
|
||||
tool_call_id,
|
||||
)
|
||||
|
||||
_emit_message_tool_calls(span, prefix, message)
|
||||
|
||||
content = message.get("content")
|
||||
if isinstance(content, list):
|
||||
contents_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}"
|
||||
for part_idx, part in enumerate(content):
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
part_prefix = f"{contents_prefix}.{part_idx}"
|
||||
part_type = part.get("type")
|
||||
if part_type in ("text", "input_text"):
|
||||
text = part.get("text")
|
||||
if isinstance(text, str):
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}",
|
||||
"text",
|
||||
)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}",
|
||||
text,
|
||||
)
|
||||
elif part_type in ("image_url", "image", "input_image"):
|
||||
url = None
|
||||
image = part.get("image_url")
|
||||
if isinstance(image, dict):
|
||||
url = image.get("url")
|
||||
elif isinstance(image, str):
|
||||
url = image
|
||||
if not url:
|
||||
# Anthropic-style source.{type=base64,media_type,data}
|
||||
source = part.get("source")
|
||||
if isinstance(source, dict) and source.get("data"):
|
||||
media_type = source.get("media_type", "image/jpeg")
|
||||
url = f"data:{media_type};base64,{source['data']}"
|
||||
elif isinstance(part.get("url"), str):
|
||||
url = part["url"]
|
||||
if url:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}",
|
||||
"image",
|
||||
)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{part_prefix}.message_content.image.image.url",
|
||||
url,
|
||||
)
|
||||
|
||||
|
||||
def _set_session_and_user_attrs(
|
||||
span: "Span", kwargs: dict, standard_logging_payload
|
||||
) -> None:
|
||||
"""Emit `SESSION_ID` / `USER_ID` / team metadata when source data exists.
|
||||
|
||||
`SESSION_ID` is emitted only when an explicit end-user identifier exists
|
||||
(`metadata.user_api_key_end_user_id`). We deliberately do NOT fall back
|
||||
to `trace_id`, because that would create a distinct "session" for every
|
||||
single request and distort Arize's Session-grouping analytics. The
|
||||
`trace_id` is still emitted under its own `litellm.trace_id` key so
|
||||
spans remain filterable by trace.
|
||||
|
||||
USER_ID is *only* emitted when no upstream path (model_params.user or
|
||||
optional_params.user) has already set it, to avoid overwriting an
|
||||
existing value with a possibly-different one from API-key metadata.
|
||||
"""
|
||||
if not isinstance(standard_logging_payload, dict):
|
||||
return
|
||||
metadata = standard_logging_payload.get("metadata") or {}
|
||||
if not isinstance(metadata, dict):
|
||||
return
|
||||
|
||||
session_id = metadata.get("user_api_key_end_user_id")
|
||||
if session_id:
|
||||
safe_set_attribute(span, SpanAttributes.SESSION_ID, str(session_id))
|
||||
|
||||
trace_id = standard_logging_payload.get("trace_id")
|
||||
if trace_id:
|
||||
safe_set_attribute(span, "litellm.trace_id", str(trace_id))
|
||||
|
||||
optional_params = kwargs.get("optional_params") or {}
|
||||
model_params = standard_logging_payload.get("model_parameters") or {}
|
||||
has_user_already = bool(
|
||||
(isinstance(optional_params, dict) and optional_params.get("user"))
|
||||
or (isinstance(model_params, dict) and model_params.get("user"))
|
||||
)
|
||||
if not has_user_already:
|
||||
user_id = metadata.get("user_api_key_user_id")
|
||||
if user_id:
|
||||
safe_set_attribute(span, SpanAttributes.USER_ID, str(user_id))
|
||||
|
||||
team_id = metadata.get("user_api_key_team_id")
|
||||
if team_id:
|
||||
safe_set_attribute(span, "litellm.team_id", str(team_id))
|
||||
team_alias = metadata.get("user_api_key_team_alias")
|
||||
if team_alias:
|
||||
safe_set_attribute(span, "litellm.team_alias", str(team_alias))
|
||||
key_alias = metadata.get("user_api_key_alias")
|
||||
if key_alias:
|
||||
safe_set_attribute(span, "litellm.key_alias", str(key_alias))
|
||||
|
||||
|
||||
def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None:
|
||||
"""Emit cost attributes from the StandardLoggingPayload when present.
|
||||
|
||||
Uses the OpenInference `llm.cost.total` key so Arize / Phoenix can
|
||||
surface the cost in their "Total Cost" column. LiteLLM only tracks a
|
||||
single total in `StandardLoggingPayload.response_cost`, so we cannot
|
||||
split it into prompt/completion. We also keep the legacy
|
||||
`llm.response.cost` key for back-compat with any consumer querying it.
|
||||
"""
|
||||
if not isinstance(standard_logging_payload, dict):
|
||||
return
|
||||
cost = standard_logging_payload.get("response_cost")
|
||||
if cost is None:
|
||||
return
|
||||
try:
|
||||
cost_value = float(cost)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
safe_set_attribute(span, "llm.cost.total", cost_value)
|
||||
safe_set_attribute(span, "llm.response.cost", cost_value)
|
||||
|
||||
|
||||
def _is_passthrough_call_type(call_type: Optional[str]) -> bool:
|
||||
if not call_type:
|
||||
return False
|
||||
lowered = str(call_type).lower()
|
||||
return "passthrough" in lowered or "pass_through" in lowered
|
||||
|
||||
|
||||
def _maybe_normalize_passthrough(
|
||||
span: "Span",
|
||||
kwargs: dict,
|
||||
raw_response_obj,
|
||||
coerced_response_obj,
|
||||
standard_logging_payload,
|
||||
) -> None:
|
||||
"""Surface input/output text for passthrough routes (e.g. Bedrock
|
||||
InvokeModel) so the parent span renders as more than `usage` numbers.
|
||||
|
||||
Only runs when `call_type` is a passthrough variant. Reads from:
|
||||
- `kwargs["additional_args"]["complete_input_dict"]` for input
|
||||
- the coerced response (or `kwargs["original_response"]`) for output
|
||||
|
||||
All emits are best-effort: if the provider shape isn't recognized the
|
||||
helper exits silently. Existing chat/completion paths never enter this
|
||||
helper because their call_type doesn't contain "passthrough".
|
||||
|
||||
TEMPORARY BRIDGE: passthrough handlers don't populate the
|
||||
StandardLoggingPayload `messages` field today (they call
|
||||
`transform_response(messages=[])`), so the input is only available via
|
||||
`additional_args.complete_input_dict`. The proper fix is upstream in
|
||||
`base_passthrough_logging_handler._create_response_logging_payload()`:
|
||||
once that populates SLP `messages`/`response`, every callback gets
|
||||
passthrough I/O (with central redaction) for free and this helper's
|
||||
`complete_input_dict` fallback can be deleted. See follow-up issue.
|
||||
"""
|
||||
call_type = (
|
||||
standard_logging_payload.get("call_type")
|
||||
if isinstance(standard_logging_payload, dict)
|
||||
else None
|
||||
)
|
||||
if not _is_passthrough_call_type(call_type):
|
||||
return
|
||||
|
||||
# Respect LiteLLM's central message-redaction contract. The normal
|
||||
# chat/completion path is redacted by `perform_redaction` before
|
||||
# callbacks run, but `complete_input_dict` (read below) is NOT covered by
|
||||
# that layer — so without this gate, an operator who enabled redaction
|
||||
# would still see raw passthrough prompts in Arize. Skip entirely when
|
||||
# redaction is on so neither input nor output leaks through this bridge.
|
||||
if should_redact_message_logging(kwargs):
|
||||
return
|
||||
|
||||
# --- INPUT --------------------------------------------------------------
|
||||
additional_args = kwargs.get("additional_args") or {}
|
||||
complete_input_dict = (
|
||||
additional_args.get("complete_input_dict")
|
||||
if isinstance(additional_args, dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(complete_input_dict, dict):
|
||||
_set_passthrough_input_attributes(span, complete_input_dict.get("messages"))
|
||||
|
||||
# --- OUTPUT -------------------------------------------------------------
|
||||
parsed_response = _parse_passthrough_response(
|
||||
raw_response_obj, coerced_response_obj, kwargs
|
||||
)
|
||||
if not isinstance(parsed_response, dict):
|
||||
return
|
||||
|
||||
_set_passthrough_output_attributes(span, parsed_response)
|
||||
|
||||
|
||||
def _set_passthrough_input_attributes(span: "Span", messages) -> None:
|
||||
"""Render passthrough request messages into INPUT_VALUE + LLM_INPUT_MESSAGES."""
|
||||
if not (isinstance(messages, list) and messages):
|
||||
return
|
||||
# Set INPUT_VALUE from the last user message text if discoverable.
|
||||
last_text = None
|
||||
for msg in reversed(messages):
|
||||
if isinstance(msg, dict):
|
||||
last_text = _coerce_text(msg.get("content"))
|
||||
if last_text:
|
||||
break
|
||||
if last_text:
|
||||
safe_set_attribute(span, SpanAttributes.INPUT_VALUE, last_text)
|
||||
# Mirror messages into LLM_INPUT_MESSAGES so the input pane renders.
|
||||
for idx, msg in enumerate(messages):
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}"
|
||||
role = msg.get("role")
|
||||
if role:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
|
||||
role,
|
||||
)
|
||||
text = _coerce_text(msg.get("content"))
|
||||
if text is not None:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def _set_passthrough_output_attributes(span: "Span", parsed_response: dict) -> None:
|
||||
"""Render passthrough response into OUTPUT_VALUE + LLM_OUTPUT_MESSAGES."""
|
||||
# Anthropic / Bedrock-Anthropic: `content` is a list of typed parts.
|
||||
content_list = parsed_response.get("content")
|
||||
if isinstance(content_list, list) and content_list:
|
||||
texts = []
|
||||
for part in content_list:
|
||||
if isinstance(part, dict) and isinstance(part.get("text"), str):
|
||||
texts.append(part["text"])
|
||||
joined = "\n\n".join(t for t in texts if t)
|
||||
if joined:
|
||||
safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, joined)
|
||||
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0"
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
|
||||
parsed_response.get("role", "assistant"),
|
||||
)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
|
||||
joined,
|
||||
)
|
||||
|
||||
# OpenAI-style passthrough: `choices[0].message.content`
|
||||
choices = parsed_response.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
first = choices[0]
|
||||
if isinstance(first, dict):
|
||||
msg = first.get("message")
|
||||
if isinstance(msg, dict):
|
||||
text = _coerce_text(msg.get("content"))
|
||||
if text:
|
||||
safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, text)
|
||||
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0"
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
|
||||
msg.get("role", "assistant"),
|
||||
)
|
||||
safe_set_attribute(
|
||||
span,
|
||||
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def _parse_passthrough_response(raw_response_obj, coerced_response_obj, kwargs):
|
||||
"""Return a dict view of the provider response for passthrough routes."""
|
||||
# Prefer the coerced view (already JSON-parsed for httpx.Response).
|
||||
candidates = []
|
||||
if isinstance(coerced_response_obj, dict):
|
||||
candidates.append(coerced_response_obj)
|
||||
if (
|
||||
isinstance(raw_response_obj, dict)
|
||||
and raw_response_obj is not coerced_response_obj
|
||||
):
|
||||
candidates.append(raw_response_obj)
|
||||
|
||||
for candidate in candidates:
|
||||
# StandardPassThroughResponseObject wrapper: {"response": "..."}.
|
||||
if (
|
||||
"response" in candidate
|
||||
and "content" not in candidate
|
||||
and "choices" not in candidate
|
||||
):
|
||||
inner = candidate.get("response")
|
||||
if isinstance(inner, str):
|
||||
try:
|
||||
parsed = json.loads(inner)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(inner, dict):
|
||||
return inner
|
||||
else:
|
||||
return candidate
|
||||
|
||||
# Fallback: kwargs["original_response"] from the OTel base path.
|
||||
original = kwargs.get("original_response") if isinstance(kwargs, dict) else None
|
||||
if isinstance(original, dict):
|
||||
return original
|
||||
if isinstance(original, str):
|
||||
try:
|
||||
parsed = json.loads(original)
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import os
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.arize import _utils
|
||||
|
|
@ -8,8 +10,10 @@ from litellm.types.integrations.arize_phoenix import ArizePhoenixConfig
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SpanProcessor
|
||||
from opentelemetry.trace import Span as _Span
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace import Tracer
|
||||
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry as _OpenTelemetry
|
||||
from litellm.integrations.opentelemetry import (
|
||||
|
|
@ -21,20 +25,27 @@ if TYPE_CHECKING:
|
|||
OpenTelemetryConfig = _OpenTelemetryConfig
|
||||
Span = Union[_Span, Any]
|
||||
OpenTelemetry = _OpenTelemetry
|
||||
LITELLM_TRACER_NAME: str
|
||||
else:
|
||||
Protocol = Any
|
||||
OpenTelemetryConfig = Any
|
||||
Span = Any
|
||||
Tracer = Any
|
||||
TracerProvider = Any
|
||||
SpanKind = Any
|
||||
# Import OpenTelemetry at runtime
|
||||
SpanProcessor = Any
|
||||
try:
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.integrations.opentelemetry import (
|
||||
LITELLM_TRACER_NAME,
|
||||
OpenTelemetry,
|
||||
)
|
||||
except ImportError:
|
||||
LITELLM_TRACER_NAME = "litellm"
|
||||
OpenTelemetry = None # type: ignore
|
||||
|
||||
|
||||
ARIZE_HOSTED_PHOENIX_ENDPOINT = "https://otlp.arize.com/v1/traces"
|
||||
_MAX_PROJECT_PROVIDERS = 64
|
||||
|
||||
|
||||
class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
||||
|
|
@ -48,37 +59,142 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
|
||||
def _init_tracing(self, tracer_provider):
|
||||
"""
|
||||
Override to always create a *private* TracerProvider for Arize Phoenix.
|
||||
Override to create per-project TracerProviders (LRU-cached) for Arize Phoenix.
|
||||
|
||||
The base ``OpenTelemetry._init_tracing`` falls back to the global
|
||||
TracerProvider when one already exists. That causes whichever
|
||||
integration initialises second to silently reuse the first one's
|
||||
exporter, so spans only reach one destination.
|
||||
|
||||
By creating our own provider we guarantee Arize Phoenix always gets
|
||||
its own exporter pipeline, regardless of initialisation order.
|
||||
"""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
if tracer_provider is not None:
|
||||
# Explicitly supplied (e.g. in tests) — honour it.
|
||||
self.tracer = tracer_provider.get_tracer("litellm")
|
||||
self._use_injected_tracer_provider = True
|
||||
self._shared_span_processor = None
|
||||
self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
|
||||
self.span_kind = SpanKind
|
||||
return
|
||||
|
||||
# Always create a dedicated provider — never touch the global one.
|
||||
provider = TracerProvider(resource=self._get_litellm_resource(self.config))
|
||||
provider.add_span_processor(self._get_span_processor())
|
||||
self.tracer = provider.get_tracer("litellm")
|
||||
self._use_injected_tracer_provider = False
|
||||
self._project_providers: OrderedDict[str, TracerProvider] = OrderedDict()
|
||||
self._project_providers_lock = threading.Lock()
|
||||
self._shared_span_processor = self._get_span_processor()
|
||||
self.span_kind = SpanKind
|
||||
|
||||
default_project = self._resolve_project_name({})
|
||||
self.tracer = self._get_tracer_for(default_project)
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: Created dedicated TracerProvider "
|
||||
"(endpoint=%s, exporter=%s)",
|
||||
"ArizePhoenixLogger: Initialized per-project TracerProvider cache "
|
||||
"(default_project=%s, endpoint=%s, exporter=%s)",
|
||||
default_project,
|
||||
self.config.endpoint,
|
||||
self.config.exporter,
|
||||
)
|
||||
|
||||
def flush_tracer_providers(self) -> None:
|
||||
"""
|
||||
Flush all cached per-project providers and the shared span processor.
|
||||
|
||||
Call on graceful proxy shutdown. Do not call on LRU eviction — in-flight
|
||||
spans may still reference evicted providers.
|
||||
"""
|
||||
if getattr(self, "_use_injected_tracer_provider", False):
|
||||
return
|
||||
|
||||
shared_processor = getattr(self, "_shared_span_processor", None)
|
||||
if shared_processor is not None:
|
||||
try:
|
||||
shared_processor.force_flush()
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: shared span processor force_flush failed: %s",
|
||||
e,
|
||||
)
|
||||
|
||||
with getattr(self, "_project_providers_lock", threading.Lock()):
|
||||
providers = list(getattr(self, "_project_providers", {}).values())
|
||||
|
||||
for provider in providers:
|
||||
try:
|
||||
provider.force_flush()
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: TracerProvider force_flush failed: %s", e
|
||||
)
|
||||
|
||||
def _get_litellm_resource_for_project(self, project_name: str):
|
||||
"""
|
||||
Build an OTEL Resource with project routing attrs that win over env detector.
|
||||
|
||||
Phoenix uses ``openinference.project.name``; Arize AX uses ``model_id`` and
|
||||
``service.name``. Project attrs are merged last so OTEL_RESOURCE_ATTRIBUTES
|
||||
from init does not pin every provider to one project.
|
||||
"""
|
||||
from opentelemetry.sdk.resources import OTELResourceDetector, Resource
|
||||
|
||||
project_attributes: dict[str, str] = {
|
||||
"openinference.project.name": project_name,
|
||||
"model_id": project_name,
|
||||
"service.name": project_name,
|
||||
}
|
||||
deployment_environment = getattr(self.config, "deployment_environment", None)
|
||||
if deployment_environment is not None:
|
||||
project_attributes["deployment.environment"] = deployment_environment
|
||||
|
||||
env_resource = OTELResourceDetector().detect()
|
||||
project_resource = Resource.create(project_attributes) # type: ignore[arg-type]
|
||||
return env_resource.merge(project_resource)
|
||||
|
||||
def _build_tracer_provider_for_project(self, project_name: str) -> TracerProvider:
|
||||
"""Create a TracerProvider for *project_name* (caller holds no cache lock)."""
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
|
||||
provider = TracerProvider(
|
||||
resource=self._get_litellm_resource_for_project(project_name)
|
||||
)
|
||||
provider.add_span_processor(self._shared_span_processor)
|
||||
return provider
|
||||
|
||||
def _get_tracer_for(self, project_name: str) -> Tracer:
|
||||
"""Return a tracer for *project_name*, creating/caching a provider on miss."""
|
||||
if getattr(self, "_use_injected_tracer_provider", False):
|
||||
return self.tracer
|
||||
|
||||
with self._project_providers_lock:
|
||||
if project_name in self._project_providers:
|
||||
self._project_providers.move_to_end(project_name)
|
||||
return self._project_providers[project_name].get_tracer(
|
||||
LITELLM_TRACER_NAME
|
||||
)
|
||||
|
||||
# OTELResourceDetector().detect() is synchronous; build outside the lock so
|
||||
# concurrent requests for other projects are not blocked on cache misses.
|
||||
new_provider = self._build_tracer_provider_for_project(project_name)
|
||||
|
||||
with self._project_providers_lock:
|
||||
if project_name in self._project_providers:
|
||||
self._project_providers.move_to_end(project_name)
|
||||
return self._project_providers[project_name].get_tracer(
|
||||
LITELLM_TRACER_NAME
|
||||
)
|
||||
|
||||
if len(self._project_providers) >= _MAX_PROJECT_PROVIDERS:
|
||||
self._project_providers.popitem(last=False)
|
||||
|
||||
self._project_providers[project_name] = new_provider
|
||||
return new_provider.get_tracer(LITELLM_TRACER_NAME)
|
||||
|
||||
def _resolve_tracer_for_kwargs(self, kwargs: dict) -> Tuple[str, Tracer]:
|
||||
"""Resolve project name once and return the matching tracer."""
|
||||
project_name = self._resolve_project_name(kwargs)
|
||||
return project_name, self._get_tracer_for(project_name)
|
||||
|
||||
def get_tracer_to_use_for_request(self, kwargs: dict) -> Tracer:
|
||||
"""Route guardrail/raw-request spans to the same per-project tracer as the request."""
|
||||
if getattr(self, "_use_injected_tracer_provider", False):
|
||||
return self.tracer
|
||||
return self._resolve_tracer_for_kwargs(kwargs)[1]
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self):
|
||||
"""
|
||||
Override: Arize Phoenix should NOT overwrite the proxy's
|
||||
|
|
@ -93,56 +209,109 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
|
||||
@staticmethod
|
||||
def set_arize_phoenix_attributes(span: Span, kwargs, response_obj):
|
||||
from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import (
|
||||
safe_set_attribute,
|
||||
)
|
||||
|
||||
_utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes)
|
||||
|
||||
# Dynamic project name: check metadata first, then fall back to env var config
|
||||
dynamic_project_name = ArizePhoenixLogger._get_dynamic_project_name(kwargs)
|
||||
if dynamic_project_name:
|
||||
safe_set_attribute(span, "openinference.project.name", dynamic_project_name)
|
||||
else:
|
||||
# Fall back to static config from env var
|
||||
config = ArizePhoenixLogger.get_arize_phoenix_config()
|
||||
if config.project_name:
|
||||
safe_set_attribute(
|
||||
span, "openinference.project.name", config.project_name
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _get_dynamic_project_name(kwargs) -> Optional[str]:
|
||||
"""
|
||||
Retrieve dynamic Phoenix project name from request metadata.
|
||||
def _normalize_project_name(name: Optional[str]) -> Optional[str]:
|
||||
if name is None:
|
||||
return None
|
||||
normalized = str(name).strip()
|
||||
return normalized if normalized else None
|
||||
|
||||
Users can set `metadata.phoenix_project_name` in their request to route
|
||||
traces to different Phoenix projects dynamically.
|
||||
"""
|
||||
standard_logging_payload = kwargs.get("standard_logging_object")
|
||||
if isinstance(standard_logging_payload, dict):
|
||||
metadata = standard_logging_payload.get("metadata")
|
||||
@staticmethod
|
||||
def _iter_metadata_dicts_from_kwargs(kwargs: dict):
|
||||
"""Yield request metadata dicts; standard_logging_object before litellm_params."""
|
||||
for key in ("standard_logging_object", "litellm_params"):
|
||||
found_key = kwargs.get(key)
|
||||
if not isinstance(found_key, dict):
|
||||
continue
|
||||
metadata = found_key.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
project_name = metadata.get("phoenix_project_name")
|
||||
if project_name:
|
||||
return str(project_name)
|
||||
yield metadata
|
||||
|
||||
# Also check litellm_params.metadata for SDK usage
|
||||
@staticmethod
|
||||
def _is_proxy_request(kwargs: dict) -> bool:
|
||||
"""True when the call is routed through the LiteLLM proxy.
|
||||
|
||||
Proxy mode is determined solely by the server-set ``proxy_server_request``
|
||||
field in ``litellm_params``. Checking request metadata for
|
||||
``user_api_key_auth_metadata`` is intentionally avoided: that field is
|
||||
user-supplied and would let an authenticated caller fake proxy-mode
|
||||
detection to route their telemetry into arbitrary Arize/Phoenix projects.
|
||||
"""
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if isinstance(litellm_params, dict):
|
||||
metadata = litellm_params.get("metadata") or {}
|
||||
else:
|
||||
metadata = {}
|
||||
if isinstance(metadata, dict):
|
||||
project_name = metadata.get("phoenix_project_name")
|
||||
if project_name:
|
||||
return str(project_name)
|
||||
return isinstance(litellm_params, dict) and bool(
|
||||
litellm_params.get("proxy_server_request")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _project_from_metadata_dict(
|
||||
metadata: dict, metadata_key: str, *, proxy_mode: bool
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Read a Phoenix project field from proxy/SDK metadata.
|
||||
|
||||
On the proxy, only ``user_api_key_auth_metadata`` (team/key config) may
|
||||
select the project. SDK callers may still set project fields directly on
|
||||
``metadata``.
|
||||
"""
|
||||
auth_metadata = metadata.get("user_api_key_auth_metadata")
|
||||
if isinstance(auth_metadata, dict):
|
||||
project = ArizePhoenixLogger._normalize_project_name(
|
||||
auth_metadata.get(metadata_key)
|
||||
)
|
||||
if project:
|
||||
return project
|
||||
|
||||
if not proxy_mode:
|
||||
return ArizePhoenixLogger._normalize_project_name(
|
||||
metadata.get(metadata_key)
|
||||
)
|
||||
return None
|
||||
|
||||
def _get_phoenix_context(self, kwargs):
|
||||
@staticmethod
|
||||
def _metadata_project_from_kwargs(kwargs: dict, metadata_key: str) -> Optional[str]:
|
||||
proxy_mode = ArizePhoenixLogger._is_proxy_request(kwargs)
|
||||
for metadata in ArizePhoenixLogger._iter_metadata_dicts_from_kwargs(kwargs):
|
||||
project = ArizePhoenixLogger._project_from_metadata_dict(
|
||||
metadata, metadata_key, proxy_mode=proxy_mode
|
||||
)
|
||||
if project:
|
||||
return project
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_project_name(kwargs: dict) -> str:
|
||||
"""
|
||||
Resolve the target Phoenix/Arize project for this request.
|
||||
|
||||
Proxy priority: ``user_api_key_auth_metadata.phoenix_project_name_override``,
|
||||
``user_api_key_auth_metadata.phoenix_project_name``, env, then ``default``.
|
||||
SDK priority: request metadata fields, then env, then ``default``.
|
||||
"""
|
||||
override = ArizePhoenixLogger._metadata_project_from_kwargs(
|
||||
kwargs, "phoenix_project_name_override"
|
||||
)
|
||||
if override:
|
||||
return override
|
||||
|
||||
phoenix_name = ArizePhoenixLogger._metadata_project_from_kwargs(
|
||||
kwargs, "phoenix_project_name"
|
||||
)
|
||||
if phoenix_name:
|
||||
return phoenix_name
|
||||
|
||||
env_name = ArizePhoenixLogger._normalize_project_name(
|
||||
os.environ.get("PHOENIX_PROJECT_NAME")
|
||||
or os.environ.get("ARIZE_PROJECT_NAME")
|
||||
)
|
||||
if env_name:
|
||||
return env_name
|
||||
|
||||
return "default"
|
||||
|
||||
def _get_phoenix_context(self, kwargs, tracer: Optional[Tracer] = None):
|
||||
"""
|
||||
Build a trace context for Phoenix's dedicated TracerProvider.
|
||||
|
||||
|
|
@ -159,11 +328,13 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
"""
|
||||
from opentelemetry import trace
|
||||
|
||||
if tracer is None:
|
||||
tracer = self._resolve_tracer_for_kwargs(kwargs)[1]
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
proxy_server_request = litellm_params.get("proxy_server_request", {}) or {}
|
||||
headers = proxy_server_request.get("headers", {}) or {}
|
||||
|
||||
# Propagate distributed trace context if the caller sent a traceparent
|
||||
traceparent_ctx = (
|
||||
self.get_traceparent_from_header(headers=headers)
|
||||
if headers.get("traceparent")
|
||||
|
|
@ -173,10 +344,8 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
is_proxy_mode = bool(proxy_server_request)
|
||||
|
||||
if is_proxy_mode:
|
||||
# Create a parent span on Phoenix's own tracer so both parent
|
||||
# and child are exported to Phoenix.
|
||||
start_time_val = kwargs.get("start_time", kwargs.get("api_call_start_time"))
|
||||
parent_span = self.tracer.start_span(
|
||||
parent_span = tracer.start_span(
|
||||
name="litellm_proxy_request",
|
||||
start_time=(
|
||||
self._to_ns(start_time_val) if start_time_val is not None else None
|
||||
|
|
@ -187,100 +356,77 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
ctx = trace.set_span_in_context(parent_span)
|
||||
return ctx, parent_span
|
||||
|
||||
# SDK mode — no parent span needed
|
||||
return traceparent_ctx, None
|
||||
|
||||
def _handle_success(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Override to always create spans on ArizePhoenixLogger's dedicated TracerProvider.
|
||||
|
||||
The base class's ``_get_span_context`` would find the parent span created by
|
||||
the ``otel`` callback on the *global* TracerProvider. That span is invisible
|
||||
in Phoenix (different exporter pipeline), so we ignore it and build our own
|
||||
hierarchy via ``_get_phoenix_context``.
|
||||
"""
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: Logging kwargs: %s, OTEL config settings=%s",
|
||||
kwargs,
|
||||
self.config,
|
||||
self._handle_phoenix_trace(
|
||||
kwargs, response_obj, start_time, end_time, success=True
|
||||
)
|
||||
|
||||
ctx, parent_span = self._get_phoenix_context(kwargs)
|
||||
|
||||
# Create litellm_request span (child of our parent when in proxy mode)
|
||||
span = self.tracer.start_span(
|
||||
name=self._get_span_name(kwargs),
|
||||
start_time=self._to_ns(start_time),
|
||||
context=ctx,
|
||||
)
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(span, kwargs, response_obj)
|
||||
|
||||
# Raw-request sub-span (if enabled) — must be created before
|
||||
# ending the parent span so the hierarchy is valid.
|
||||
self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
|
||||
span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
# Guardrail span
|
||||
self._create_guardrail_span(kwargs=kwargs, context=ctx)
|
||||
|
||||
# Annotate and close our proxy parent span
|
||||
if parent_span is not None:
|
||||
parent_span.set_status(Status(StatusCode.OK))
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
parent_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
# Metrics & cost recording
|
||||
self._record_metrics(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
# Semantic logs
|
||||
if self.config.enable_events:
|
||||
self._emit_semantic_logs(kwargs, response_obj, span)
|
||||
|
||||
def _handle_failure(self, kwargs, response_obj, start_time, end_time):
|
||||
"""
|
||||
Override to always create failure spans on ArizePhoenixLogger's dedicated
|
||||
TracerProvider. Mirrors ``_handle_success`` but sets ERROR status.
|
||||
"""
|
||||
self._handle_phoenix_trace(
|
||||
kwargs, response_obj, start_time, end_time, success=False
|
||||
)
|
||||
|
||||
def _handle_phoenix_trace(
|
||||
self,
|
||||
kwargs,
|
||||
response_obj,
|
||||
start_time,
|
||||
end_time,
|
||||
*,
|
||||
success: bool,
|
||||
):
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
verbose_logger.debug(
|
||||
"ArizePhoenixLogger: Failure - Logging kwargs: %s, OTEL config settings=%s",
|
||||
"ArizePhoenixLogger: %s - kwargs: %s, OTEL config settings=%s",
|
||||
"success" if success else "failure",
|
||||
kwargs,
|
||||
self.config,
|
||||
)
|
||||
|
||||
ctx, parent_span = self._get_phoenix_context(kwargs)
|
||||
_project_name, tracer = self._resolve_tracer_for_kwargs(kwargs)
|
||||
ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer)
|
||||
|
||||
# Create litellm_request span (child of our parent when in proxy mode)
|
||||
span = self.tracer.start_span(
|
||||
status = Status(StatusCode.OK if success else StatusCode.ERROR)
|
||||
|
||||
span = tracer.start_span(
|
||||
name=self._get_span_name(kwargs),
|
||||
start_time=self._to_ns(start_time),
|
||||
context=ctx,
|
||||
)
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
span.set_status(status)
|
||||
self.set_attributes(span, kwargs, response_obj)
|
||||
self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
if not success:
|
||||
self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
|
||||
if success:
|
||||
self._maybe_log_raw_request(
|
||||
kwargs, response_obj, start_time, end_time, span
|
||||
)
|
||||
span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
# Guardrail span
|
||||
self._create_guardrail_span(kwargs=kwargs, context=ctx)
|
||||
|
||||
# Annotate and close our proxy parent span
|
||||
if parent_span is not None:
|
||||
parent_span.set_status(Status(StatusCode.ERROR))
|
||||
parent_span.set_status(status)
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
self._record_exception_on_span(span=parent_span, kwargs=kwargs)
|
||||
if not success:
|
||||
self._record_exception_on_span(span=parent_span, kwargs=kwargs)
|
||||
parent_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
if success:
|
||||
self._record_metrics(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
if self.config.enable_events:
|
||||
self._emit_semantic_logs(kwargs, response_obj, span)
|
||||
|
||||
@staticmethod
|
||||
def get_arize_phoenix_config() -> ArizePhoenixConfig:
|
||||
"""
|
||||
Retrieves the Arize Phoenix configuration based on environment variables.
|
||||
Returns:
|
||||
ArizePhoenixConfig: A Pydantic model containing Arize Phoenix configuration.
|
||||
"""
|
||||
api_key = os.environ.get("PHOENIX_API_KEY", None)
|
||||
|
||||
|
|
@ -295,18 +441,15 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
protocol: Protocol = "otlp_http"
|
||||
|
||||
if collector_endpoint:
|
||||
# Parse the endpoint to determine protocol
|
||||
if collector_endpoint.startswith("grpc://") or (
|
||||
":4317" in collector_endpoint and "/v1/traces" not in collector_endpoint
|
||||
):
|
||||
endpoint = collector_endpoint
|
||||
protocol = "otlp_grpc"
|
||||
else:
|
||||
# Phoenix Cloud endpoints (app.phoenix.arize.com) include the space in the URL
|
||||
if "app.phoenix.arize.com" in collector_endpoint:
|
||||
endpoint = collector_endpoint
|
||||
protocol = "otlp_http"
|
||||
# For other HTTP endpoints, ensure they have the correct path
|
||||
elif "/v1/traces" not in collector_endpoint:
|
||||
if collector_endpoint.endswith("/v1"):
|
||||
endpoint = collector_endpoint + "/traces"
|
||||
|
|
@ -318,7 +461,6 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
endpoint = collector_endpoint
|
||||
protocol = "otlp_http"
|
||||
else:
|
||||
# If no endpoint specified, self hosted phoenix
|
||||
endpoint = "http://localhost:6006/v1/traces"
|
||||
protocol = "otlp_http"
|
||||
verbose_logger.debug(
|
||||
|
|
@ -329,12 +471,11 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
if api_key is not None:
|
||||
otlp_auth_headers = f"Authorization=Bearer {api_key}"
|
||||
elif "app.phoenix.arize.com" in endpoint:
|
||||
# Phoenix Cloud requires an API key
|
||||
raise ValueError(
|
||||
"PHOENIX_API_KEY must be set when using Phoenix Cloud (app.phoenix.arize.com)."
|
||||
)
|
||||
|
||||
project_name = os.environ.get("PHOENIX_PROJECT_NAME", "default")
|
||||
project_name = os.environ.get("PHOENIX_PROJECT_NAME") or "default"
|
||||
|
||||
return ArizePhoenixConfig(
|
||||
otlp_auth_headers=otlp_auth_headers,
|
||||
|
|
@ -343,8 +484,6 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
|
|||
project_name=project_name,
|
||||
)
|
||||
|
||||
## cannot suppress additional proxy server spans, removed previous methods.
|
||||
|
||||
async def async_health_check(self):
|
||||
config = self.get_arize_phoenix_config()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
Custom Logger that handles batching logic
|
||||
Custom Logger that handles batching logic
|
||||
|
||||
Use this if you want your logs to be stored in memory and flushed periodically.
|
||||
"""
|
||||
|
|
@ -14,22 +14,38 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
|
||||
|
||||
class CustomBatchLogger(CustomLogger):
|
||||
preserve_events_added_during_flush = False
|
||||
|
||||
# Default cap on the in-memory log queue. Prevents unbounded memory growth
|
||||
# if ``async_send_batch`` consistently fails (e.g. the destination is
|
||||
# unreachable) and events are preserved across flush attempts. Subclasses
|
||||
# may override by passing ``max_queue_size`` or by setting the attribute
|
||||
# directly (see ``RubrikLogger`` for an example).
|
||||
DEFAULT_MAX_QUEUE_SIZE = 50_000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
flush_lock: Optional[asyncio.Lock] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
flush_interval: Optional[int] = None,
|
||||
max_queue_size: Optional[int] = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
flush_lock (Optional[asyncio.Lock], optional): Lock to use when flushing the queue. Defaults to None. Only used for custom loggers that do batching
|
||||
max_queue_size (Optional[int], optional): Maximum number of events to retain in ``log_queue``. When the limit is exceeded (e.g. because the send destination is unreachable and events are preserved for retry), the oldest events are dropped. Defaults to ``DEFAULT_MAX_QUEUE_SIZE``.
|
||||
"""
|
||||
self.log_queue: List = []
|
||||
self.flush_interval = flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
|
||||
self.batch_size: int = batch_size or litellm.DEFAULT_BATCH_SIZE
|
||||
self.last_flush_time = time.time()
|
||||
self.flush_lock = flush_lock
|
||||
self.max_queue_size: int = (
|
||||
max_queue_size
|
||||
if max_queue_size is not None
|
||||
else self.DEFAULT_MAX_QUEUE_SIZE
|
||||
)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
|
@ -47,11 +63,40 @@ class CustomBatchLogger(CustomLogger):
|
|||
|
||||
async with self.flush_lock:
|
||||
if self.log_queue:
|
||||
log_queue_length = len(self.log_queue)
|
||||
verbose_logger.debug(
|
||||
"CustomLogger: Flushing batch of %s events", len(self.log_queue)
|
||||
)
|
||||
await self.async_send_batch()
|
||||
self.log_queue.clear()
|
||||
try:
|
||||
await self.async_send_batch()
|
||||
except Exception:
|
||||
# If the underlying batch send raised, do NOT drop the
|
||||
# in-flight events. They will be retried on the next flush.
|
||||
# Most existing async_send_batch implementations swallow
|
||||
# their own errors, so this only affects loggers that opt
|
||||
# in to surfacing failures (e.g. Rubrik).
|
||||
verbose_logger.exception(
|
||||
"CustomLogger: async_send_batch raised; preserving "
|
||||
"%s events in queue for retry",
|
||||
log_queue_length,
|
||||
)
|
||||
# Guard against unbounded queue growth if the destination
|
||||
# is persistently unreachable. Drop the oldest events
|
||||
# beyond ``max_queue_size``.
|
||||
overflow = len(self.log_queue) - self.max_queue_size
|
||||
if overflow > 0:
|
||||
del self.log_queue[:overflow]
|
||||
verbose_logger.warning(
|
||||
"CustomLogger: log queue exceeded max_queue_size=%s; "
|
||||
"dropped %s oldest events.",
|
||||
self.max_queue_size,
|
||||
overflow,
|
||||
)
|
||||
return
|
||||
if self.preserve_events_added_during_flush:
|
||||
del self.log_queue[:log_queue_length]
|
||||
else:
|
||||
self.log_queue.clear()
|
||||
self.last_flush_time = time.time()
|
||||
|
||||
async def async_send_batch(self, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -662,6 +662,16 @@ class CustomGuardrail(CustomLogger):
|
|||
request_data["metadata"] = {}
|
||||
_append_guardrail_info(request_data["metadata"])
|
||||
|
||||
# Emit the otel guardrail span here, where every guardrail execution lands,
|
||||
# rather than relying on a post-call hook that does not fire on every path
|
||||
# (e.g. a pass-through request that passes its guardrails).
|
||||
try:
|
||||
from litellm.integrations.otel.logger import emit_guardrail_span
|
||||
|
||||
emit_guardrail_span(slg)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def apply_guardrail(
|
||||
self,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.integrations.datadog.datadog_handler import (
|
|||
)
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
MaskedHTTPStatusError,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -68,6 +69,22 @@ DD_LOGGED_SUCCESS_SERVICE_TYPES = [
|
|||
]
|
||||
|
||||
|
||||
def _resolve_dd_batch_size() -> int:
|
||||
raw = os.getenv("DD_BATCH_SIZE")
|
||||
if raw is None:
|
||||
return DD_MAX_BATCH_SIZE
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
verbose_logger.warning(
|
||||
"Datadog: ignoring invalid DD_BATCH_SIZE=%r, using %s",
|
||||
raw,
|
||||
DD_MAX_BATCH_SIZE,
|
||||
)
|
||||
return DD_MAX_BATCH_SIZE
|
||||
return max(1, min(value, DD_MAX_BATCH_SIZE))
|
||||
|
||||
|
||||
class DataDogLogger(
|
||||
CustomBatchLogger,
|
||||
AdditionalLoggingUtils,
|
||||
|
|
@ -128,7 +145,9 @@ class DataDogLogger(
|
|||
asyncio.create_task(self.periodic_flush())
|
||||
self.flush_lock = asyncio.Lock()
|
||||
super().__init__(
|
||||
**kwargs, flush_lock=self.flush_lock, batch_size=DD_MAX_BATCH_SIZE
|
||||
**kwargs,
|
||||
flush_lock=self.flush_lock,
|
||||
batch_size=_resolve_dd_batch_size(),
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
|
|
@ -339,28 +358,14 @@ class DataDogLogger(
|
|||
"[DATADOG MOCK] Mock mode enabled - API calls will be intercepted"
|
||||
)
|
||||
|
||||
response = await self.async_send_compressed_data(batch_to_send)
|
||||
if response.status_code == 413:
|
||||
verbose_logger.exception(DD_ERRORS.DATADOG_413_ERROR.value)
|
||||
self.log_queue = batch_to_send + self.log_queue
|
||||
return
|
||||
|
||||
response.raise_for_status()
|
||||
if response.status_code != 202:
|
||||
raise Exception(
|
||||
f"Response from datadog API status_code: {response.status_code}, text: {response.text}"
|
||||
)
|
||||
undelivered = await self._send_with_413_split(batch_to_send)
|
||||
if undelivered:
|
||||
self.log_queue = undelivered + self.log_queue
|
||||
|
||||
if self.is_mock_mode:
|
||||
verbose_logger.debug(
|
||||
f"[DATADOG MOCK] Batch of {len(batch_to_send)} events successfully mocked"
|
||||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
"Datadog: Response from datadog API status_code: %s, text: %s",
|
||||
response.status_code,
|
||||
response.text,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.log_queue = batch_to_send + self.log_queue
|
||||
|
|
@ -368,6 +373,62 @@ class DataDogLogger(
|
|||
f"Datadog Error sending batch API - {str(e)}\n{traceback.format_exc()}"
|
||||
)
|
||||
|
||||
async def _send_with_413_split(self, batch: List) -> List:
|
||||
"""
|
||||
Send a batch, halving any sub-batch that 413s (payload too large) and retrying the
|
||||
halves, since Datadog enforces a 5MB uncompressed limit per request.
|
||||
|
||||
A 413 surfaces as a raised MaskedHTTPStatusError (httpx raise_for_status), not a
|
||||
returned response, so both paths are handled. A lone event that still 413s is
|
||||
dropped to avoid wedging the queue on an undeliverable payload. Returns the events
|
||||
that could not be delivered because of a non-413 (transient) error, so the caller
|
||||
re-queues only those and never the events already accepted by Datadog.
|
||||
"""
|
||||
pending: List[List] = [batch]
|
||||
while pending:
|
||||
chunk = pending.pop()
|
||||
if not chunk:
|
||||
continue
|
||||
try:
|
||||
response = await self.async_send_compressed_data(chunk)
|
||||
except Exception as e:
|
||||
if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413:
|
||||
response = e.response
|
||||
else:
|
||||
verbose_logger.exception(
|
||||
f"Datadog Error sending batch API - {str(e)}"
|
||||
)
|
||||
return self._undelivered(chunk, pending)
|
||||
|
||||
if response.status_code == 413:
|
||||
if len(chunk) == 1:
|
||||
verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value)
|
||||
continue
|
||||
mid = len(chunk) // 2
|
||||
pending.append(chunk[mid:])
|
||||
pending.append(chunk[:mid])
|
||||
continue
|
||||
|
||||
if response.status_code != 202:
|
||||
verbose_logger.error(
|
||||
"Datadog: unexpected response status_code=%s, text=%s",
|
||||
response.status_code,
|
||||
response.text,
|
||||
)
|
||||
return self._undelivered(chunk, pending)
|
||||
|
||||
verbose_logger.debug(
|
||||
"Datadog: delivered %s events, status_code=%s, text=%s",
|
||||
len(chunk),
|
||||
response.status_code,
|
||||
response.text,
|
||||
)
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _undelivered(chunk: List, pending: List[List]) -> List:
|
||||
return chunk + [event for remaining in reversed(pending) for event in remaining]
|
||||
|
||||
async def flush_queue(self):
|
||||
if self.flush_lock is None:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -2,10 +2,17 @@ import asyncio
|
|||
import os
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.integrations.datadog.datadog_handler import (
|
||||
get_datadog_env,
|
||||
get_datadog_hostname,
|
||||
get_datadog_pod_name,
|
||||
get_datadog_service,
|
||||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -15,9 +22,30 @@ from litellm.types.integrations.datadog_cost_management import (
|
|||
)
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
||||
# Reserved tag keys whose values come from trusted sources (infra env, LiteLLM
|
||||
# core payload fields, or proxy-controlled auth metadata). User-supplied
|
||||
# request_tags / metadata cannot overwrite these, even when the key is
|
||||
# allowlisted via cost_tag_keys, because that would let an authenticated caller
|
||||
# spoof cost attribution (e.g. request_tags=["team:victim-team"]).
|
||||
_RESERVED_TAG_KEYS: frozenset = frozenset(
|
||||
{
|
||||
"env",
|
||||
"service",
|
||||
"host",
|
||||
"pod_name",
|
||||
"provider",
|
||||
"model",
|
||||
"model_id",
|
||||
"team",
|
||||
"user",
|
||||
"model_group",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DatadogCostManagementLogger(CustomBatchLogger):
|
||||
def __init__(self, **kwargs):
|
||||
def __init__(self, cost_tag_keys: Optional[List[str]] = None, **kwargs):
|
||||
self.cost_tag_keys: List[str] = list(cost_tag_keys) if cost_tag_keys else []
|
||||
self.dd_api_key = os.getenv("DD_API_KEY")
|
||||
self.dd_app_key = os.getenv("DD_APP_KEY")
|
||||
self.dd_site = os.getenv("DD_SITE", "datadoghq.com")
|
||||
|
|
@ -68,20 +96,21 @@ class DatadogCostManagementLogger(CustomBatchLogger):
|
|||
if not self.log_queue:
|
||||
return
|
||||
|
||||
batch_to_send = self.log_queue[:]
|
||||
self.log_queue = []
|
||||
|
||||
try:
|
||||
# Aggregate costs from the batch
|
||||
aggregated_entries = self._aggregate_costs(self.log_queue)
|
||||
|
||||
aggregated_entries = self._aggregate_costs(batch_to_send)
|
||||
if not aggregated_entries:
|
||||
verbose_logger.debug(
|
||||
"Datadog Cost Management: batch produced no aggregable entries; "
|
||||
"dropping %d log(s) from queue.",
|
||||
len(batch_to_send),
|
||||
)
|
||||
return
|
||||
|
||||
# Send to Datadog
|
||||
await self._upload_to_datadog(aggregated_entries)
|
||||
|
||||
# Clear queue only on success (or if we decide to drop on failure)
|
||||
# CustomBatchLogger clears queue in flush_queue, so we just process here
|
||||
|
||||
except Exception as e:
|
||||
self.log_queue = batch_to_send + self.log_queue
|
||||
verbose_logger.exception(
|
||||
f"Datadog Cost Management: Error in async_send_batch: {str(e)}"
|
||||
)
|
||||
|
|
@ -151,45 +180,81 @@ class DatadogCostManagementLogger(CustomBatchLogger):
|
|||
return list(aggregator.values())
|
||||
|
||||
def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]:
|
||||
from litellm.integrations.datadog.datadog_handler import (
|
||||
get_datadog_env,
|
||||
get_datadog_hostname,
|
||||
get_datadog_pod_name,
|
||||
get_datadog_service,
|
||||
)
|
||||
|
||||
tags = {
|
||||
tags: Dict[str, str] = {
|
||||
"env": get_datadog_env(),
|
||||
"service": get_datadog_service(),
|
||||
"host": get_datadog_hostname(),
|
||||
"pod_name": get_datadog_pod_name(),
|
||||
}
|
||||
|
||||
# Add metadata as tags
|
||||
metadata = log.get("metadata", {})
|
||||
if metadata:
|
||||
# Add user info
|
||||
# Add user info
|
||||
if metadata.get("user_api_key_alias"):
|
||||
tags["user"] = str(metadata["user_api_key_alias"])
|
||||
# Always-on canonical FOCUS dimensions from top-level payload fields.
|
||||
# Non-sensitive and required for Datadog Custom Costs per-model attribution.
|
||||
self._add_tag(tags, "provider", log.get("custom_llm_provider"))
|
||||
self._add_tag(tags, "model", log.get("model"))
|
||||
self._add_tag(tags, "model_id", log.get("model_id"))
|
||||
|
||||
# Add Team Tag
|
||||
team_tag = (
|
||||
metadata.get("user_api_key_team_alias")
|
||||
or metadata.get("team_alias") # type: ignore
|
||||
or metadata.get("user_api_key_team_id")
|
||||
or metadata.get("team_id") # type: ignore
|
||||
)
|
||||
# cast because StandardLoggingMetadata is a TypedDict; we iterate it
|
||||
# as a generic mapping below.
|
||||
metadata: Dict[str, Any] = cast(Dict[str, Any], log.get("metadata") or {})
|
||||
|
||||
if team_tag:
|
||||
tags["team"] = str(team_tag)
|
||||
# model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get()
|
||||
model_group = metadata.get("model_group") # type: ignore[misc]
|
||||
if model_group:
|
||||
tags["model_group"] = str(model_group)
|
||||
# Backwards-compat: team/user/model_group preserved regardless of allowlist.
|
||||
if metadata.get("user_api_key_alias"):
|
||||
tags["user"] = str(metadata["user_api_key_alias"])
|
||||
team_tag = (
|
||||
metadata.get("user_api_key_team_alias")
|
||||
or metadata.get("team_alias")
|
||||
or metadata.get("user_api_key_team_id")
|
||||
or metadata.get("team_id")
|
||||
)
|
||||
if team_tag:
|
||||
tags["team"] = str(team_tag)
|
||||
if metadata.get("model_group"):
|
||||
tags["model_group"] = str(metadata["model_group"])
|
||||
|
||||
# Allowlist-gated: request_tags (split on `:`) and arbitrary metadata.*.
|
||||
# Reserved keys are hard-blocked here regardless of allowlist membership —
|
||||
# see _RESERVED_TAG_KEYS for the rationale.
|
||||
if self.cost_tag_keys:
|
||||
allow = set(self.cost_tag_keys)
|
||||
for rt in log.get("request_tags") or []:
|
||||
if not isinstance(rt, str) or ":" not in rt:
|
||||
continue
|
||||
k, _, v = rt.partition(":")
|
||||
if k in allow and v:
|
||||
self._set_custom_tag(tags, k, v)
|
||||
for k, v in metadata.items():
|
||||
if k in allow and v is not None and not isinstance(v, (dict, list)):
|
||||
self._set_custom_tag(tags, k, str(v))
|
||||
for nested_key in ("spend_logs_metadata", "requester_metadata"):
|
||||
nested = metadata.get(nested_key)
|
||||
if isinstance(nested, dict):
|
||||
for k, v in nested.items():
|
||||
if (
|
||||
k in allow
|
||||
and v is not None
|
||||
and not isinstance(v, (dict, list))
|
||||
):
|
||||
self._set_custom_tag(tags, k, str(v))
|
||||
|
||||
return tags
|
||||
|
||||
@staticmethod
|
||||
def _set_custom_tag(tags: Dict[str, str], key: str, value: str) -> None:
|
||||
if key in _RESERVED_TAG_KEYS:
|
||||
verbose_logger.debug(
|
||||
"Datadog Cost Management: dropping user-supplied tag %r=%r — "
|
||||
"key is reserved for trusted cost attribution.",
|
||||
key,
|
||||
value,
|
||||
)
|
||||
return
|
||||
tags[key] = value
|
||||
|
||||
@staticmethod
|
||||
def _add_tag(tags: Dict[str, str], key: str, value: Any) -> None:
|
||||
if value:
|
||||
tags[key] = str(value)
|
||||
|
||||
async def _upload_to_datadog(self, payload: List[Dict]):
|
||||
if not self.dd_api_key or not self.dd_app_key:
|
||||
return
|
||||
|
|
@ -201,8 +266,6 @@ class DatadogCostManagementLogger(CustomBatchLogger):
|
|||
}
|
||||
|
||||
# The API endpoint expects a list of objects directly in the body (file content behavior)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
data_json = safe_dumps(payload)
|
||||
|
||||
response = await self.async_client.put(
|
||||
|
|
|
|||
|
|
@ -144,7 +144,26 @@ class DatadogMetricsLogger(CustomBatchLogger):
|
|||
}
|
||||
self.log_queue.append(series_llm_latency)
|
||||
|
||||
# 3. Request Count / Status Code
|
||||
# 3. LiteLLM Overhead Latency Metric (total - llm_api time)
|
||||
hidden_params = log.get("hidden_params", {}) or {}
|
||||
litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms")
|
||||
if litellm_overhead_time_ms is not None:
|
||||
overhead_tags = self._extract_tags(log) # no status_code on latency metric
|
||||
series_overhead: DatadogMetricSeries = {
|
||||
"metric": "litellm.overhead.latency",
|
||||
"type": 3, # gauge
|
||||
"points": [
|
||||
{
|
||||
"timestamp": timestamp,
|
||||
"value": litellm_overhead_time_ms
|
||||
/ 1000, # convert ms → seconds
|
||||
}
|
||||
],
|
||||
"tags": overhead_tags,
|
||||
}
|
||||
self.log_queue.append(series_overhead)
|
||||
|
||||
# 4. Request Count / Status Code
|
||||
series_count: DatadogMetricSeries = {
|
||||
"metric": "litellm.llm_api.request_count",
|
||||
"type": 1, # count
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import polars as pl
|
|||
|
||||
from .schema import FOCUS_NORMALIZED_SCHEMA
|
||||
|
||||
|
||||
_TAG_KEYS = (
|
||||
"team_id",
|
||||
"team_alias",
|
||||
|
|
@ -96,7 +95,9 @@ class FocusTransformer:
|
|||
pl.lit("Usage-Based").alias("ChargeFrequency"),
|
||||
fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"),
|
||||
fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"),
|
||||
dec(pl.lit(1.0)).alias("ConsumedQuantity"),
|
||||
dec(
|
||||
pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0)
|
||||
).alias("ConsumedQuantity"),
|
||||
pl.lit("Requests").alias("ConsumedUnit"),
|
||||
dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"),
|
||||
none_str.alias("ContractedUnitPrice"),
|
||||
|
|
@ -108,7 +109,9 @@ class FocusTransformer:
|
|||
none_str.alias("AvailabilityZone"),
|
||||
pl.lit("USD").alias("PricingCurrency"),
|
||||
none_str.alias("PricingCategory"),
|
||||
dec(pl.lit(1.0)).alias("PricingQuantity"),
|
||||
dec(
|
||||
pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0)
|
||||
).alias("PricingQuantity"),
|
||||
none_dec.alias("PricingCurrencyContractedUnitPrice"),
|
||||
dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"),
|
||||
none_dec.alias("PricingCurrencyListUnitPrice"),
|
||||
|
|
|
|||
|
|
@ -1,18 +1,29 @@
|
|||
import json
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
import re
|
||||
from typing import Any, Dict, List, Optional, Tuple, cast
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
convert_content_list_to_str,
|
||||
get_content_from_model_response,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai"
|
||||
# Cap the in-memory buffer so persistent flush failures (e.g. Galileo
|
||||
# unavailable, invalid credentials) cannot leak memory unboundedly.
|
||||
GALILEO_MAX_IN_MEMORY_RECORDS = 1000
|
||||
|
||||
|
||||
# from here: https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#structuring-your-records
|
||||
class LLMResponse(BaseModel):
|
||||
latency_ms: int
|
||||
status_code: int
|
||||
|
|
@ -37,65 +48,190 @@ class GalileoObserve(CustomLogger):
|
|||
def __init__(self) -> None:
|
||||
self.in_memory_records: List[dict] = []
|
||||
self.batch_size = 1
|
||||
self.base_url = os.getenv("GALILEO_BASE_URL", None)
|
||||
self.project_id = os.getenv("GALILEO_PROJECT_ID", None)
|
||||
self.api_key = os.getenv("GALILEO_API_KEY")
|
||||
self.project_id = os.getenv("GALILEO_PROJECT_ID")
|
||||
self.log_stream_id = os.getenv("GALILEO_LOG_STREAM_ID")
|
||||
self.username = os.getenv("GALILEO_USERNAME")
|
||||
self.password = os.getenv("GALILEO_PASSWORD")
|
||||
self.base_url = self._normalize_base_url(os.getenv("GALILEO_BASE_URL"))
|
||||
if self.api_key and not self.base_url:
|
||||
self.base_url = GALILEO_CLOUD_API_BASE_URL
|
||||
self.use_v2_api = bool(self.api_key)
|
||||
self.headers: Optional[Dict[str, str]] = None
|
||||
self.async_httpx_handler = get_async_httpx_client(
|
||||
llm_provider=httpxSpecialProvider.LoggingCallback
|
||||
)
|
||||
pass
|
||||
|
||||
def set_galileo_headers(self):
|
||||
# following https://docs.rungalileo.io/galileo/gen-ai-studio-products/galileo-observe/how-to/logging-data-via-restful-apis#logging-your-records
|
||||
@staticmethod
|
||||
def _normalize_base_url(base_url: Optional[str]) -> Optional[str]:
|
||||
if base_url:
|
||||
return base_url.rstrip("/")
|
||||
return None
|
||||
|
||||
headers = {
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
}
|
||||
galileo_login_response = litellm.module_level_client.post(
|
||||
def _is_configured(self) -> bool:
|
||||
if not self.project_id or not self.base_url:
|
||||
return False
|
||||
if self.use_v2_api:
|
||||
return bool(self.api_key)
|
||||
return bool(self.username and self.password)
|
||||
|
||||
async def async_set_galileo_headers(self) -> None:
|
||||
galileo_login_response = await self.async_httpx_handler.post(
|
||||
url=f"{self.base_url}/login",
|
||||
headers=headers,
|
||||
headers={
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
},
|
||||
data={
|
||||
"username": os.getenv("GALILEO_USERNAME"),
|
||||
"password": os.getenv("GALILEO_PASSWORD"),
|
||||
"username": self.username,
|
||||
"password": self.password,
|
||||
},
|
||||
)
|
||||
|
||||
galileo_login_response.raise_for_status()
|
||||
access_token = galileo_login_response.json()["access_token"]
|
||||
|
||||
self.headers = {
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {access_token}",
|
||||
}
|
||||
|
||||
def get_output_str_from_response(self, response_obj, kwargs):
|
||||
output = None
|
||||
if response_obj is not None and (
|
||||
kwargs.get("call_type", None) == "embedding"
|
||||
or isinstance(response_obj, litellm.EmbeddingResponse)
|
||||
):
|
||||
output = None
|
||||
elif response_obj is not None and isinstance(
|
||||
response_obj, litellm.ModelResponse
|
||||
):
|
||||
output = response_obj["choices"][0]["message"].json()
|
||||
elif response_obj is not None and isinstance(
|
||||
response_obj, litellm.TextCompletionResponse
|
||||
):
|
||||
output = response_obj.choices[0].text
|
||||
elif response_obj is not None and isinstance(
|
||||
response_obj, litellm.ImageResponse
|
||||
):
|
||||
output = response_obj["data"]
|
||||
async def _ensure_headers(self) -> bool:
|
||||
if self.headers is not None:
|
||||
return True
|
||||
|
||||
return output
|
||||
if self.use_v2_api:
|
||||
if not self.api_key:
|
||||
return False
|
||||
self.headers = {
|
||||
"accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"Galileo-API-Key": self.api_key,
|
||||
}
|
||||
return True
|
||||
|
||||
if not (self.username and self.password and self.base_url):
|
||||
return False
|
||||
|
||||
try:
|
||||
await self.async_set_galileo_headers()
|
||||
return True
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Galileo Logger: failed to authenticate: %s", e)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _galileo_input_messages(
|
||||
messages: Optional[List[Any]], input_text: str
|
||||
) -> List[Dict[str, str]]:
|
||||
if not messages:
|
||||
return [{"role": "user", "content": input_text}]
|
||||
|
||||
galileo_messages: List[Dict[str, str]] = []
|
||||
for message in messages:
|
||||
if not isinstance(message, dict):
|
||||
continue
|
||||
role = message.get("role")
|
||||
if not role:
|
||||
continue
|
||||
galileo_messages.append(
|
||||
{
|
||||
"role": str(role),
|
||||
"content": convert_content_list_to_str(
|
||||
message=cast(AllMessageValues, message)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
if galileo_messages:
|
||||
return galileo_messages
|
||||
return [{"role": "user", "content": input_text}]
|
||||
|
||||
@staticmethod
|
||||
def _record_to_v2_span(record: Dict[str, Any]) -> Dict[str, Any]:
|
||||
created_at = record.get("created_at", "")
|
||||
if created_at and not re.search(r"(Z|[+-]\d{2}:?\d{2})$", created_at):
|
||||
created_at = f"{created_at}Z"
|
||||
|
||||
span: Dict[str, Any] = {
|
||||
"type": "llm",
|
||||
"name": record.get("node_type", "litellm"),
|
||||
"created_at": created_at,
|
||||
"input": GalileoObserve._galileo_input_messages(
|
||||
record.get("messages"), record.get("input_text", "")
|
||||
),
|
||||
"output": {
|
||||
"role": "assistant",
|
||||
"content": record.get("output_text", ""),
|
||||
},
|
||||
"status_code": record.get("status_code", 200),
|
||||
"model": record.get("model"),
|
||||
"metrics": {
|
||||
"duration_ns": int(record.get("latency_ms", 0)) * 1_000_000,
|
||||
"num_input_tokens": record.get("num_input_tokens"),
|
||||
"num_output_tokens": record.get("num_output_tokens"),
|
||||
},
|
||||
}
|
||||
if record.get("tags"):
|
||||
span["tags"] = record["tags"]
|
||||
return span
|
||||
|
||||
def _get_ingest_request(self) -> Optional[Tuple[str, Dict[str, Any]]]:
|
||||
if not self.base_url or not self.project_id:
|
||||
return None
|
||||
|
||||
# Snapshot the records to be sent into a new list so concurrent appends
|
||||
# during the network round-trip (across the await points in
|
||||
# flush_in_memory_records) aren't silently dropped when we later clear
|
||||
# the in-memory buffer.
|
||||
records = list(self.in_memory_records)
|
||||
|
||||
if self.use_v2_api:
|
||||
payload: Dict[str, Any] = {
|
||||
"spans": [self._record_to_v2_span(record) for record in records],
|
||||
"reliable": False,
|
||||
}
|
||||
if self.log_stream_id:
|
||||
payload["log_stream_id"] = self.log_stream_id
|
||||
return (
|
||||
f"{self.base_url}/v2/projects/{self.project_id}/spans",
|
||||
payload,
|
||||
)
|
||||
|
||||
return (
|
||||
f"{self.base_url}/projects/{self.project_id}/observe/ingest",
|
||||
{"records": records},
|
||||
)
|
||||
|
||||
def get_output_str_from_response(
|
||||
self, response_obj: Any, kwargs: Dict[str, Any]
|
||||
) -> Optional[str]:
|
||||
if response_obj is None:
|
||||
return None
|
||||
if kwargs.get("call_type", None) == "embedding" or isinstance(
|
||||
response_obj, litellm.EmbeddingResponse
|
||||
):
|
||||
return None
|
||||
if isinstance(response_obj, litellm.TextCompletionResponse):
|
||||
return response_obj.choices[0].text
|
||||
if isinstance(response_obj, litellm.ImageResponse):
|
||||
return json.dumps(response_obj["data"], default=str)
|
||||
if isinstance(response_obj, (litellm.ModelResponse, dict)):
|
||||
return get_content_from_model_response(response_obj)
|
||||
return None
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any
|
||||
):
|
||||
verbose_logger.debug("On Async Success")
|
||||
|
||||
if not self._is_configured():
|
||||
verbose_logger.debug(
|
||||
"Galileo Logger: skipping flush — set GALILEO_PROJECT_ID and "
|
||||
"either GALILEO_API_KEY (hosted) or GALILEO_USERNAME/GALILEO_PASSWORD "
|
||||
"(enterprise Observe)."
|
||||
)
|
||||
return
|
||||
|
||||
_latency_ms = int((end_time - start_time).total_seconds() * 1000)
|
||||
_call_type = kwargs.get("call_type", "litellm")
|
||||
input_text = litellm.utils.get_formatted_prompt(
|
||||
|
|
@ -125,26 +261,69 @@ class GalileoObserve(CustomLogger):
|
|||
), # timestamp str constructed in "%Y-%m-%dT%H:%M:%S" format
|
||||
)
|
||||
|
||||
# dump to dict
|
||||
request_dict = request_record.model_dump()
|
||||
messages = kwargs.get("messages")
|
||||
if messages:
|
||||
request_dict["messages"] = messages
|
||||
self.in_memory_records.append(request_dict)
|
||||
|
||||
# Bound the buffer so persistent flush failures cannot grow it
|
||||
# without limit. Drop the oldest records once we exceed the cap.
|
||||
if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS:
|
||||
dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS
|
||||
self.in_memory_records = self.in_memory_records[
|
||||
-GALILEO_MAX_IN_MEMORY_RECORDS:
|
||||
]
|
||||
verbose_logger.warning(
|
||||
"Galileo Logger: in-memory buffer exceeded %s records; "
|
||||
"dropped %s oldest record(s). Check Galileo connectivity/credentials.",
|
||||
GALILEO_MAX_IN_MEMORY_RECORDS,
|
||||
dropped,
|
||||
)
|
||||
|
||||
if len(self.in_memory_records) >= self.batch_size:
|
||||
await self.flush_in_memory_records()
|
||||
|
||||
async def flush_in_memory_records(self):
|
||||
verbose_logger.debug("flushing in memory records")
|
||||
response = await self.async_httpx_handler.post(
|
||||
url=f"{self.base_url}/projects/{self.project_id}/observe/ingest",
|
||||
headers=self.headers,
|
||||
json={"records": self.in_memory_records},
|
||||
)
|
||||
if not self.in_memory_records:
|
||||
return
|
||||
|
||||
if response.status_code == 200:
|
||||
# Capture the number of records that will be sent BEFORE any await so
|
||||
# that concurrent appends made by other asyncio tasks during the
|
||||
# network round-trip aren't silently dropped on the success-clear.
|
||||
records_in_payload = len(self.in_memory_records)
|
||||
|
||||
ingest_request = self._get_ingest_request()
|
||||
if ingest_request is None:
|
||||
verbose_logger.debug(
|
||||
"Galileo Logger:successfully flushed in memory records"
|
||||
"Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID"
|
||||
)
|
||||
self.in_memory_records = []
|
||||
return
|
||||
|
||||
if not await self._ensure_headers():
|
||||
verbose_logger.debug("Galileo Logger: could not set request headers")
|
||||
return
|
||||
|
||||
url, payload = ingest_request
|
||||
verbose_logger.debug("flushing in memory records to %s", url)
|
||||
|
||||
try:
|
||||
response = await self.async_httpx_handler.post(
|
||||
url=url,
|
||||
headers=self.headers,
|
||||
json=payload,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
"Galileo Logger: failed to flush in memory records: %s", e
|
||||
)
|
||||
return
|
||||
|
||||
if response.is_success:
|
||||
verbose_logger.debug(
|
||||
"Galileo Logger: successfully flushed in memory records"
|
||||
)
|
||||
del self.in_memory_records[:records_in_payload]
|
||||
else:
|
||||
verbose_logger.debug("Galileo Logger: failed to flush in memory records")
|
||||
verbose_logger.debug(
|
||||
|
|
@ -152,6 +331,13 @@ class GalileoObserve(CustomLogger):
|
|||
response.text,
|
||||
response.status_code,
|
||||
)
|
||||
# Legacy enterprise auth caches a bearer token obtained from
|
||||
# /login. If the request was rejected for auth reasons, drop the
|
||||
# cached headers so the next flush re-authenticates instead of
|
||||
# silently failing forever on a stale token. The v2 API key path
|
||||
# uses a long-lived static key, so leave its headers in place.
|
||||
if not self.use_v2_api and response.status_code in (401, 403):
|
||||
self.headers = None
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
verbose_logger.debug("On Async Failure")
|
||||
|
|
|
|||
|
|
@ -64,6 +64,9 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
|
|||
HTTP_ROUTE_ATTRIBUTE = "http.route"
|
||||
URL_PATH_ATTRIBUTE = "url.path"
|
||||
PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
|
||||
TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata"
|
||||
MODEL_GROUP_ATTRIBUTE = "litellm.model_group"
|
||||
PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model"
|
||||
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
|
||||
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
|
||||
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
|
||||
|
|
@ -80,6 +83,19 @@ _VALID_CAPTURE_MODES = {
|
|||
}
|
||||
|
||||
|
||||
def _normalize_team_metadata_keys(value: Any) -> List[str]:
|
||||
"""Coerce a team-metadata allowlist from a list or comma-separated string.
|
||||
|
||||
config.yaml passes a YAML list; an env var passes a comma-separated string.
|
||||
Both collapse to a list of stripped, non-empty keys.
|
||||
"""
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpenTelemetryConfig:
|
||||
exporter: Union[str, SpanExporter] = "console"
|
||||
|
|
@ -97,6 +113,10 @@ class OpenTelemetryConfig:
|
|||
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
|
||||
capture_message_content: Optional[str] = None
|
||||
semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set)
|
||||
# Sub-keys of the team's free-form metadata stamped onto the inference span
|
||||
# under ``litellm.team.metadata``. Empty by default so none of a team's
|
||||
# metadata leaves the process until explicitly allowlisted.
|
||||
baggage_team_metadata_keys: List[str] = field(default_factory=list)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# If endpoint is specified but exporter is still the default "console",
|
||||
|
|
@ -127,6 +147,11 @@ class OpenTelemetryConfig:
|
|||
self.semconv_stability_opt_in |= parse_semconv_opt_in(
|
||||
os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV)
|
||||
)
|
||||
self.baggage_team_metadata_keys = _normalize_team_metadata_keys(
|
||||
self.baggage_team_metadata_keys
|
||||
) or _normalize_team_metadata_keys(
|
||||
os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS")
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
|
|
@ -185,8 +210,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
meter_provider: Optional[Any] = None,
|
||||
**kwargs,
|
||||
):
|
||||
team_metadata_keys_override = kwargs.pop("baggage_team_metadata_keys", None)
|
||||
if config is None:
|
||||
config = OpenTelemetryConfig.from_env()
|
||||
if team_metadata_keys_override is not None:
|
||||
config.baggage_team_metadata_keys = _normalize_team_metadata_keys(
|
||||
team_metadata_keys_override
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.callback_name = callback_name
|
||||
|
|
@ -673,6 +703,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
if parent_otel_span is not None:
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
|
||||
# Stamp team attributes onto the SERVER (root) span too, so the
|
||||
# trace root is team-filterable on the failure path like the
|
||||
# child exception span below.
|
||||
self._set_team_attributes_on_span(
|
||||
span=parent_otel_span,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
team_alias=user_api_key_dict.team_alias,
|
||||
)
|
||||
|
||||
# Stamp structured error attrs on the SERVER span itself; the
|
||||
# failure path otherwise only sets its status (_handle_failure
|
||||
# records on the litellm_request child span). Inline import:
|
||||
|
|
@ -693,6 +732,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
},
|
||||
)
|
||||
|
||||
# _record_exception_on_span only stamps when error_code is set;
|
||||
# bare TypeError etc. has none, and the span is about to be ended.
|
||||
error_code = (
|
||||
error_information.get("error_code") if error_information else None
|
||||
)
|
||||
if not error_code:
|
||||
self.set_response_status_code_attribute(parent_otel_span, 500)
|
||||
|
||||
# Pre-request latency (request_data carries the propagated
|
||||
# metadata on the failure path; omitted if it failed before handoff).
|
||||
self.set_preprocessing_duration_attribute(parent_otel_span, request_data)
|
||||
|
|
@ -709,12 +756,65 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
key="exception",
|
||||
value=str(original_exception),
|
||||
)
|
||||
self._set_team_attributes_on_span(
|
||||
span=exception_logging_span,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
team_alias=user_api_key_dict.team_alias,
|
||||
)
|
||||
exception_logging_span.set_status(Status(StatusCode.ERROR))
|
||||
exception_logging_span.end(end_time=self._to_ns(datetime.now()))
|
||||
|
||||
# Emit guardrail spans for any guardrail invocations that
|
||||
# ran during this request. _handle_failure typically does this,
|
||||
# but for pre-call guardrail blocks the standard_logging_object
|
||||
# may not carry guardrail_information by the time _handle_failure
|
||||
# fires (the data lives only in request_data["metadata"]). Pull
|
||||
# directly from request_data so the span is recorded either way;
|
||||
# _emit_once dedupes if _handle_failure already emitted it.
|
||||
self._emit_guardrail_spans_from_request_data(
|
||||
request_data=request_data,
|
||||
parent_span=parent_otel_span,
|
||||
)
|
||||
|
||||
# End Parent OTEL Sspan
|
||||
parent_otel_span.end(end_time=self._to_ns(datetime.now()))
|
||||
|
||||
def _emit_guardrail_spans_from_request_data(
|
||||
self,
|
||||
request_data: dict,
|
||||
parent_span: Optional[Any],
|
||||
) -> None:
|
||||
"""Emit ``guardrail`` spans from ``request_data["metadata"]
|
||||
["standard_logging_guardrail_information"]``.
|
||||
|
||||
Routed through ``_create_guardrail_span`` so the dedupe state in
|
||||
``_otel_internal`` is honoured — if ``_handle_failure`` already
|
||||
emitted these spans for the same kwargs, this is a no-op.
|
||||
"""
|
||||
from opentelemetry import trace as _trace
|
||||
|
||||
metadata = (request_data or {}).get("metadata") or {}
|
||||
guardrail_information = metadata.get("standard_logging_guardrail_information")
|
||||
if not guardrail_information:
|
||||
return
|
||||
|
||||
# _create_guardrail_span reads guardrail_information from
|
||||
# kwargs["standard_logging_object"] and shares its dedupe state via
|
||||
# kwargs["litellm_params"]["metadata"]["_otel_internal"]. Pass the
|
||||
# SAME metadata dict the proxy populated so _handle_failure and
|
||||
# this hook see the same dedupe markers.
|
||||
kwargs: Dict[str, Any] = {
|
||||
"litellm_params": {"metadata": metadata},
|
||||
"standard_logging_object": {
|
||||
"guardrail_information": guardrail_information,
|
||||
"metadata": metadata,
|
||||
},
|
||||
}
|
||||
context = (
|
||||
_trace.set_span_in_context(parent_span) if parent_span is not None else None
|
||||
)
|
||||
self._create_guardrail_span(kwargs=kwargs, context=context)
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self,
|
||||
data: dict,
|
||||
|
|
@ -736,11 +836,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# Pre-request latency on the SERVER span (success path).
|
||||
self.set_preprocessing_duration_attribute(parent_span, kwargs)
|
||||
|
||||
# http.response.status_code on the SERVER span (success path).
|
||||
# A successful proxy response is HTTP 200; the failure path sets
|
||||
# this from the error code in _record_exception_on_span.
|
||||
self.set_response_status_code_attribute(parent_span, 200)
|
||||
|
||||
# 3. Guardrail span
|
||||
self._create_guardrail_span(kwargs=kwargs, context=ctx)
|
||||
|
||||
|
|
@ -917,13 +1012,28 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
litellm_params = kwargs.get("litellm_params", {}) or {}
|
||||
_metadata = litellm_params.get("metadata", {}) or {}
|
||||
proxy_span = _metadata.get("litellm_parent_otel_span", None)
|
||||
|
||||
# Fallback: check litellm_metadata (used by /v1/messages and other
|
||||
# LITELLM_METADATA_ROUTES).
|
||||
if proxy_span is None:
|
||||
_litellm_metadata = litellm_params.get("litellm_metadata", {}) or {}
|
||||
proxy_span = _litellm_metadata.get("litellm_parent_otel_span", None)
|
||||
|
||||
if (
|
||||
proxy_span is not None
|
||||
and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
and hasattr(proxy_span, "is_recording")
|
||||
and proxy_span.is_recording()
|
||||
):
|
||||
proxy_span.end(end_time=self._to_ns(end_time))
|
||||
self._close_proxy_span_ok(proxy_span, end_time)
|
||||
|
||||
def _close_proxy_span_ok(self, span: Span, end_time) -> None:
|
||||
"""Stamp http.response.status_code=200 + status=OK, then end the span."""
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
self.set_response_status_code_attribute(span, 200)
|
||||
span.set_status(Status(StatusCode.OK))
|
||||
span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
def _handle_success(self, kwargs, response_obj, start_time, end_time):
|
||||
"""Create the litellm_request span then close the proxy span."""
|
||||
|
|
@ -1009,8 +1119,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
parent_span is not None
|
||||
and hasattr(parent_span, "name")
|
||||
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
and hasattr(parent_span, "is_recording")
|
||||
and parent_span.is_recording()
|
||||
):
|
||||
parent_span.end(end_time=self._to_ns(end_time))
|
||||
self._close_proxy_span_ok(parent_span, end_time)
|
||||
|
||||
# Stamp team attributes onto the SERVER (root) span before it is
|
||||
# closed, so the trace root carries them like every child span.
|
||||
self._set_team_attributes_on_proxy_span_from_kwargs(kwargs)
|
||||
|
||||
# close the proxy span explicitly from kwargs metadata
|
||||
# after all child spans (litellm_request, guardrail, raw_request)
|
||||
|
|
@ -1070,8 +1186,138 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
)
|
||||
raw_span.set_status(Status(StatusCode.OK))
|
||||
self.set_raw_request_attributes(raw_span, kwargs, response_obj)
|
||||
self._set_team_attributes_from_kwargs(raw_span, kwargs)
|
||||
raw_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
def _set_team_attributes_on_span(
|
||||
self,
|
||||
span: Span,
|
||||
team_id: Optional[str],
|
||||
team_alias: Optional[str],
|
||||
) -> None:
|
||||
"""Stamp team_id / team_alias onto a span so every child span of a
|
||||
litellm_request trace carries them, not just the root span.
|
||||
|
||||
Empty strings are treated as absent: a request made with the master
|
||||
key or a team-less virtual key carries ``user_api_key_team_id=""``
|
||||
in ``standard_logging_object.metadata``; propagating that to every
|
||||
span only adds noise that makes traces look mis-instrumented.
|
||||
"""
|
||||
if team_id:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key="metadata.user_api_key_team_id",
|
||||
value=team_id,
|
||||
)
|
||||
if team_alias:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key="metadata.user_api_key_team_alias",
|
||||
value=team_alias,
|
||||
)
|
||||
|
||||
def _set_team_attributes_from_kwargs(self, span: Span, kwargs: dict) -> None:
|
||||
"""Pull team_id / team_alias from the standard logging metadata in kwargs and stamp them onto span."""
|
||||
std_log = kwargs.get("standard_logging_object")
|
||||
md: dict = {}
|
||||
if isinstance(std_log, dict):
|
||||
md = std_log.get("metadata") or {}
|
||||
elif std_log is not None:
|
||||
md = getattr(std_log, "metadata", None) or {}
|
||||
self._set_team_attributes_on_span(
|
||||
span=span,
|
||||
team_id=md.get("user_api_key_team_id"),
|
||||
team_alias=md.get("user_api_key_team_alias"),
|
||||
)
|
||||
|
||||
def _set_team_attributes_on_proxy_span_from_kwargs(self, kwargs: dict) -> None:
|
||||
"""Stamp team attributes onto the proxy SERVER (root) span so the
|
||||
trace root is filterable by team, not just its children. The root
|
||||
span is created in auth before the team is resolved and is
|
||||
otherwise only closed (never re-attributed) on the success path.
|
||||
|
||||
Guarded to the LiteLLM-created proxy span (by name + recording) so
|
||||
externally provided parent spans are never mutated.
|
||||
"""
|
||||
litellm_params = kwargs.get("litellm_params") or {}
|
||||
metadata = litellm_params.get("metadata") or {}
|
||||
proxy_span = metadata.get("litellm_parent_otel_span")
|
||||
if (
|
||||
proxy_span is not None
|
||||
and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
|
||||
and hasattr(proxy_span, "is_recording")
|
||||
and proxy_span.is_recording()
|
||||
):
|
||||
self._set_team_attributes_from_kwargs(proxy_span, kwargs)
|
||||
|
||||
def _set_inference_identity_attributes(
|
||||
self,
|
||||
span: Span,
|
||||
standard_logging_payload: StandardLoggingPayload,
|
||||
litellm_params: dict,
|
||||
) -> None:
|
||||
"""Stamp request-identity attributes onto an inference span so every
|
||||
LLM-call span is filterable by the route it came in on, the team's
|
||||
metadata, and both the user-facing (model_group alias) and the
|
||||
dispatched (provider) model names. Empty/absent values are skipped.
|
||||
"""
|
||||
metadata = standard_logging_payload.get("metadata") or {}
|
||||
|
||||
http_route = metadata.get("user_api_key_request_route")
|
||||
if http_route:
|
||||
self.safe_set_attribute(
|
||||
span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
|
||||
)
|
||||
|
||||
# ``user_api_key_team_metadata`` is dropped from the standard logging
|
||||
# payload metadata, so read it from the raw request metadata in kwargs.
|
||||
# ``metadata`` and ``litellm_metadata`` are alternate names for the same
|
||||
# full metadata dict (the name varies by endpoint), so first-truthy wins.
|
||||
raw_metadata = (
|
||||
litellm_params.get("metadata")
|
||||
or litellm_params.get("litellm_metadata")
|
||||
or {}
|
||||
)
|
||||
team_metadata = self._team_metadata_json(
|
||||
raw_metadata.get("user_api_key_team_metadata"),
|
||||
self.config.baggage_team_metadata_keys,
|
||||
)
|
||||
if team_metadata:
|
||||
self.safe_set_attribute(
|
||||
span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata
|
||||
)
|
||||
|
||||
model_group = standard_logging_payload.get("model_group")
|
||||
if model_group:
|
||||
self.safe_set_attribute(
|
||||
span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group
|
||||
)
|
||||
|
||||
hidden_params = standard_logging_payload.get("hidden_params") or {}
|
||||
provider_model = hidden_params.get(
|
||||
"litellm_model_name"
|
||||
) or standard_logging_payload.get("model")
|
||||
if provider_model:
|
||||
self.safe_set_attribute(
|
||||
span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _team_metadata_json(value: Any, allowed_keys: List[str]) -> Optional[str]:
|
||||
"""JSON-serialize only the allowlisted sub-keys of a team's metadata.
|
||||
|
||||
Returns ``None`` when nothing is allowlisted or no allowlisted key is
|
||||
present, so the empty case is dropped rather than stamping a useless
|
||||
``"{}"`` (and so a team's metadata never leaves the process until an
|
||||
operator opts each sub-key in via ``baggage_team_metadata_keys``).
|
||||
"""
|
||||
if not isinstance(value, dict) or not value or not allowed_keys:
|
||||
return None
|
||||
filtered = {key: value[key] for key in allowed_keys if key in value}
|
||||
if not filtered:
|
||||
return None
|
||||
return safe_dumps(filtered)
|
||||
|
||||
def _record_metrics(self, kwargs, response_obj, start_time, end_time):
|
||||
duration_s = (end_time - start_time).total_seconds()
|
||||
params = kwargs.get("litellm_params") or {}
|
||||
|
|
@ -1531,12 +1777,45 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
"masked_entity_count", safe_dumps(masked_entity_count)
|
||||
)
|
||||
|
||||
guardrail_response = guardrail_information.get("guardrail_response")
|
||||
if guardrail_response is not None:
|
||||
guardrail_span.set_attribute(
|
||||
"guardrail_response", safe_dumps(guardrail_response)
|
||||
)
|
||||
|
||||
# Surface guardrail_status (success / guardrail_intervened /
|
||||
# guardrail_failed_to_respond / not_run) as a top-level span
|
||||
# attribute so trace backends can filter on it without parsing
|
||||
# guardrail_response.
|
||||
self.safe_set_attribute(
|
||||
span=guardrail_span,
|
||||
key="guardrail_response",
|
||||
value=guardrail_information.get("guardrail_response"),
|
||||
key="guardrail_status",
|
||||
value=guardrail_information.get("guardrail_status"),
|
||||
)
|
||||
|
||||
# Provider's raw top-level action (e.g. Bedrock's
|
||||
# ``GUARDRAIL_INTERVENED`` / ``NONE``). Populated by the provider
|
||||
# hook onto StandardLoggingGuardrailInformation so this integration
|
||||
# stays provider-agnostic — we only read a normalised string.
|
||||
guardrail_action = guardrail_information.get("guardrail_action")
|
||||
if guardrail_action:
|
||||
guardrail_span.set_attribute("guardrail_action", guardrail_action)
|
||||
|
||||
# The provider hook (e.g. Bedrock) extracts violation_categories
|
||||
# from the raw response BEFORE redaction and stamps them onto
|
||||
# StandardLoggingGuardrailInformation. Surfacing them here as a
|
||||
# queryable attribute lets dashboards group by violation category
|
||||
# without parsing the redacted guardrail_response blob.
|
||||
violation_categories = guardrail_information.get("violation_categories")
|
||||
if violation_categories:
|
||||
# OTel sequence attributes must be homogeneous primitives;
|
||||
# serialise to JSON once so set_attribute never coerces.
|
||||
guardrail_span.set_attribute(
|
||||
"guardrail_violation_categories", safe_dumps(violation_categories)
|
||||
)
|
||||
|
||||
self._set_team_attributes_from_kwargs(guardrail_span, kwargs)
|
||||
|
||||
guardrail_span.end(end_time=self._to_ns(end_time_datetime))
|
||||
|
||||
def _handle_failure(self, kwargs, response_obj, start_time, end_time):
|
||||
|
|
@ -1849,6 +2128,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
key="hidden_params",
|
||||
value=safe_dumps(hidden_params),
|
||||
)
|
||||
|
||||
self._set_inference_identity_attributes(
|
||||
span=span,
|
||||
standard_logging_payload=standard_logging_payload,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
# Cost breakdown tracking
|
||||
cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get(
|
||||
"cost_breakdown"
|
||||
|
|
@ -2390,6 +2675,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
)
|
||||
|
||||
def _to_ns(self, dt):
|
||||
if dt is None:
|
||||
return int(datetime.now().timestamp() * 1e9)
|
||||
if isinstance(dt, (int, float)):
|
||||
return int(dt * 1e9)
|
||||
return int(dt.timestamp() * 1e9)
|
||||
|
||||
def _get_span_name(self, kwargs):
|
||||
|
|
@ -2436,6 +2725,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
_metadata = litellm_params.get("metadata", {}) or {}
|
||||
parent_otel_span = _metadata.get("litellm_parent_otel_span", None)
|
||||
|
||||
# Fallback: check litellm_metadata (used by /v1/messages and other
|
||||
# LITELLM_METADATA_ROUTES that store proxy-internal metadata
|
||||
# separately from the provider's native "metadata" field).
|
||||
if parent_otel_span is None:
|
||||
_litellm_metadata = litellm_params.get("litellm_metadata", {}) or {}
|
||||
parent_otel_span = _litellm_metadata.get("litellm_parent_otel_span", None)
|
||||
|
||||
# Priority 1: Explicit parent span from metadata
|
||||
if parent_otel_span is not None:
|
||||
verbose_logger.debug(
|
||||
|
|
@ -2880,6 +3176,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
management_endpoint_span.set_status(Status(StatusCode.OK))
|
||||
management_endpoint_span.end(end_time=_end_time_ns)
|
||||
|
||||
# The management wrapper has no other hook that closes the SERVER span.
|
||||
self.set_response_status_code_attribute(parent_otel_span, 200)
|
||||
parent_otel_span.set_status(Status(StatusCode.OK))
|
||||
parent_otel_span.end(end_time=_end_time_ns)
|
||||
|
||||
async def async_management_endpoint_failure_hook(
|
||||
self,
|
||||
logging_payload: ManagementEndpointLoggingPayload,
|
||||
|
|
@ -2930,6 +3231,24 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
management_endpoint_span.set_status(Status(StatusCode.ERROR))
|
||||
management_endpoint_span.end(end_time=_end_time_ns)
|
||||
|
||||
# The management wrapper has no other hook that closes the SERVER span.
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
|
||||
error_information = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=_exception,
|
||||
)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
self._record_exception_on_span(
|
||||
span=parent_otel_span,
|
||||
kwargs={
|
||||
"exception": _exception,
|
||||
"standard_logging_object": {"error_information": error_information},
|
||||
},
|
||||
)
|
||||
parent_otel_span.end(end_time=_end_time_ns)
|
||||
|
||||
def create_litellm_proxy_request_started_span(
|
||||
self,
|
||||
start_time: datetime,
|
||||
|
|
@ -2986,6 +3305,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
value=int(status_code),
|
||||
)
|
||||
|
||||
def record_error_attributes_on_span(
|
||||
self,
|
||||
span: Optional[Span],
|
||||
exception: Optional[Exception],
|
||||
status_code: int,
|
||||
) -> None:
|
||||
"""Stamp structured ``error.*`` attributes on the SERVER span from the
|
||||
exception returned to the client, with ``error.code`` pinned to the real
|
||||
response status. Idempotent (overwrites); emits no exception event."""
|
||||
if span is None or exception is None:
|
||||
return
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
|
||||
error_information = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=exception
|
||||
)
|
||||
error_information["error_code"] = str(status_code)
|
||||
self._record_exception_on_span(
|
||||
span=span,
|
||||
kwargs={
|
||||
"standard_logging_object": {"error_information": error_information}
|
||||
},
|
||||
)
|
||||
|
||||
def set_preprocessing_duration_attribute(
|
||||
self, span: Optional[Span], container: Any
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -39,20 +39,32 @@ def extract_opik_metadata(
|
|||
standard_logging_metadata: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract and merge Opik metadata from request and requester.
|
||||
Merge Opik metadata from three sources in increasing priority order:
|
||||
|
||||
1. user_api_key_auth_metadata– lowest priority (operator-level defaults)
|
||||
2. litellm_metadata (request)– overrides auth-key defaults
|
||||
3. requester_metadata – highest priority (e.g. proxy header overrides)
|
||||
|
||||
Args:
|
||||
litellm_metadata: Metadata from litellm_params
|
||||
standard_logging_metadata: Metadata from standard_logging_object
|
||||
litellm_metadata: Metadata from litellm_params.mak
|
||||
standard_logging_metadata: Metadata from standard_logging_object.
|
||||
|
||||
Returns:
|
||||
Merged Opik metadata dictionary
|
||||
Merged Opik metadata dictionary.
|
||||
"""
|
||||
opik_meta = litellm_metadata.get("opik", {}).copy()
|
||||
# Start with auth-key defaults (lowest priority).
|
||||
auth_meta = standard_logging_metadata.get("user_api_key_auth_metadata") or {}
|
||||
opik_meta = (auth_meta.get("opik") or {}).copy()
|
||||
|
||||
# Request-level values override auth-key defaults.
|
||||
request_opik = litellm_metadata.get("opik") or {}
|
||||
opik_meta.update(request_opik)
|
||||
|
||||
# Requester-level values win over everything else.
|
||||
requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {}
|
||||
requester_opik = requester_metadata.get("opik", {}) or {}
|
||||
opik_meta.update(requester_opik)
|
||||
if requester_opik:
|
||||
opik_meta.update(requester_opik)
|
||||
|
||||
_logging.verbose_logger.debug(
|
||||
f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}"
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ def _remove_nulls(x: Dict[str, Any]) -> Dict[str, Any]:
|
|||
|
||||
|
||||
def get_traces_and_spans_from_payload(
|
||||
payload: List[Dict[str, Any]]
|
||||
payload: List[Dict[str, Any]],
|
||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
"""
|
||||
Separate traces and spans from payload.
|
||||
|
|
|
|||
261
litellm/integrations/otel/README.md
Normal file
261
litellm/integrations/otel/README.md
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
# OpenTelemetry instrumentation
|
||||
|
||||
This package produces OpenTelemetry traces for LiteLLM. It is enabled by the
|
||||
`LITELLM_OTEL_V2` environment variable (`is_otel_v2_enabled()` in
|
||||
[`config.py`](./model/config.py)); when unset, nothing in this package runs.
|
||||
|
||||
## What gets traced
|
||||
|
||||
A traced proxy request produces one trace with two kinds of spans:
|
||||
|
||||
```
|
||||
SERVER span "POST /v1/chat/completions" ← FastAPI instrumentation
|
||||
├── INTERNAL span "auth /v1/chat/completions" ← auth phase ┐
|
||||
│ ├── CLIENT span "postgres get_key_object" ← datastore call │
|
||||
│ └── CLIENT span "postgres get_team_membership" │
|
||||
├── INTERNAL span "execute_guardrail …" ← guardrail │ this package
|
||||
├── CLIENT span "chat gpt-4o" ← LLM call │
|
||||
└── CLIENT span "batch_write_to_db …" ← spend write ┘
|
||||
```
|
||||
|
||||
The gen-ai spans are siblings under the server span. In particular the guardrail
|
||||
span is a sibling of the LLM call, not a child of it: pre/during/post-call
|
||||
guardrail hooks are part of the request lifecycle (a pre-call guardrail runs
|
||||
before the LLM call even starts), so they belong directly under the server span,
|
||||
alongside the LLM call.
|
||||
|
||||
Request-level spans (LLM call, guardrail) parent to the server span via an
|
||||
**explicit anchor** — `context.set_request_root_span` captures the server span
|
||||
once at request entry, and `resolve_request_span_context` reads it — rather than
|
||||
to whatever span is momentarily active. Ambient-only parenting was wrong at two
|
||||
boundaries: inside the live `auth` phase span the active span is `auth` (so the
|
||||
span would nest under auth), and a pass-through request closes its span from a
|
||||
detached `asyncio.create_task` where the server span is no longer active (so the
|
||||
span orphaned into its own trace). The anchor — a contextvar inherited by those
|
||||
child tasks — gives a stable parent in both cases. DB/service spans keep ambient
|
||||
parenting so an auth DB lookup still nests under `auth`.
|
||||
|
||||
**Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's
|
||||
service-logging layer instruments many internal functions, but only some are
|
||||
traceable units of work:
|
||||
|
||||
- **`DB_CALL` (CLIENT)** — outbound datastore calls (redis, postgres,
|
||||
`batch_write_to_db`), carrying `db.system.name` / `db.operation.name` semconv.
|
||||
- **`SERVICE` (INTERNAL)** — genuine internal work worth a span (background
|
||||
budget/reset jobs, pod-lock manager).
|
||||
- **metrics-only (no span)** — `self` (the `track_llm_api_timing` wrapper, which
|
||||
duplicates the LLM-call span), `router` (duplicates the request), and
|
||||
`proxy_pre_call` (a guardrail's real span is `execute_guardrail …`). These
|
||||
still feed Prometheus/Datadog through their own hooks; they just never enter
|
||||
the trace. `auth` is also excluded here because it gets a **live phase span**
|
||||
instead (see below).
|
||||
|
||||
Spans are named `"{service} {call_type}"` (e.g. `"redis set"`) so repeated calls
|
||||
to one service stay distinguishable. Like every other span they parent to the
|
||||
**ambient** context, falling back to the threaded `litellm_parent_otel_span` only
|
||||
when ambient has no live span; a background job with neither starts its own root
|
||||
trace. Caller-supplied `event_metadata` is **sanitized** before it reaches a span
|
||||
(primitives only, no live objects, no secrets/headers, bounded) — see
|
||||
`payloads.sanitize_event_metadata`.
|
||||
|
||||
**Live phase spans.** `auth` is wrapped in a real, active span
|
||||
(`logger.phase_span`) for the duration of authentication, so the DB lookups it
|
||||
triggers nest **under** it instead of flattening onto the server span. Identity
|
||||
Baggage (team/key/user) is seeded once the key resolves, so every post-auth span
|
||||
inherits it; auth-internal DB lookups that run before the key is known stay
|
||||
unlabeled, which is correct.
|
||||
|
||||
**Status.** On success a span's status is left `UNSET` (the semconv default,
|
||||
matching the FastAPI server span); only a genuine error sets `ERROR`.
|
||||
|
||||
- **Server spans** (one per HTTP route) are created by the
|
||||
`opentelemetry-instrumentation-fastapi` package. It stamps `http.*` attributes
|
||||
and extracts inbound `traceparent` headers. This package does **not** create
|
||||
or modify server spans — request routes never touch spans.
|
||||
- **Gen-AI spans** (LLM calls, guardrails, internal service calls) are created
|
||||
by this package from LiteLLM's logging callbacks. Request-level spans parent to
|
||||
the server span via the captured anchor; DB/service spans parent to the active
|
||||
span (ambient) so they nest under the request phase that triggered them.
|
||||
|
||||
Both kinds share a single `TracerProvider`, so they belong to the same trace
|
||||
and export through the same configured exporters. FastAPI middleware can only be
|
||||
added before the app starts serving, so the app is instrumented at
|
||||
import time **without** a provider — it binds to the OTel global
|
||||
`ProxyTracerProvider`. Once config (and the callbacks) is loaded, the proxy
|
||||
publishes the chosen logger's `TracerProvider` as the global via
|
||||
`trace.set_tracer_provider(...)`, and the server spans delegate to it. When a
|
||||
preset callback (`arize`, `langfuse_otel`, …) is configured, its provider
|
||||
becomes the global, so server spans export to that backend too.
|
||||
|
||||
## How a request flows
|
||||
|
||||
1. **App creation** (`proxy_server` import): when the gate is on,
|
||||
`mount.instrument_fastapi_app(app)` calls `FastAPIInstrumentor.instrument_app`
|
||||
with no provider (the middleware stack is frozen once the app serves, so this
|
||||
can't wait for startup). It binds to the OTel global `ProxyTracerProvider`. Noisy
|
||||
non-LLM routes are excluded by default (`mount._DEFAULT_EXCLUDED_ROUTES`): health
|
||||
checks (`/health*`), the Prometheus scrape (`/metrics`), and static UI/docs assets
|
||||
(`/litellm-asset-prefix`, `/_next`, `/ui`, `/swagger`, `/docs`, `/redoc`,
|
||||
`/openapi.json`, favicons, `/.well-known`) — so load-balancer polling, metric
|
||||
scrapes, and asset fetches don't flood traces. Entries are substring-matched, so
|
||||
`/metrics` also drops the `/model/metrics` admin-analytics spans. Set
|
||||
`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` to override the whole set (e.g. `""` to trace
|
||||
everything, or your own comma-separated path list).
|
||||
2. **Startup** (`proxy_server.proxy_startup_event`): after the config (and
|
||||
callbacks) is loaded, the already-registered preset `OpenTelemetryV2` logger
|
||||
is reused — or a generic one reading `OTEL_*` envs is built when no preset is
|
||||
configured — and its `TracerProvider` is published as the OTel global with
|
||||
`trace.set_tracer_provider(...)`. The proxy tracer then delegates to it, so
|
||||
server spans and gen-ai spans share one provider and the same trace.
|
||||
3. **Request**: the FastAPI instrumentation starts the server span and makes it
|
||||
the active context for the request task. The proxy's first call into the V2
|
||||
logger (`create_litellm_proxy_request_started_span`, at the auth boundary)
|
||||
**captures it as the request anchor** (`set_request_root_span`), so every later
|
||||
request-level span has a stable explicit parent regardless of what is active
|
||||
when it emits.
|
||||
4. **LLM call span (born at the boundary)**: `OpenTelemetryV2.log_pre_api_call`
|
||||
runs synchronously in the request task, just before the upstream call, and
|
||||
**opens** the LLM-call span there, parented to the anchored server span
|
||||
(`resolve_request_span_context`). The open span is held in a bounded cache keyed
|
||||
by `litellm_call_id` (a primitive the callback kwargs carry at both `pre_call`
|
||||
and close), so no live `Span` ever travels through a `litellm_params` metadata
|
||||
dict. For the boundary hook to fire at all, the logger is registered into
|
||||
`litellm.input_callback` — the list `Logging.pre_call` iterates. The async
|
||||
success/failure callback later
|
||||
**closes** it: it builds an `LLMCallSpanData` from the typed
|
||||
`standard_logging_object` (token usage and cost are computed only by then),
|
||||
stamps the attributes, sets status, and ends the span. The sync callback is a
|
||||
no-op (closing is async-only). When `pre_call` runs off the request task — a
|
||||
sync-only provider driven through a thread pool, where contextvars (and so the
|
||||
anchor) don't follow — no parent is visible there, so creation is **deferred**
|
||||
to the async callback, whose worker context was copied from the request task at
|
||||
enqueue and so still carries the anchor. **Pass-through** endpoints call
|
||||
`logging_obj.pre_call` in the request task too, then close from a detached
|
||||
`asyncio.create_task`; the anchor (not the by-then-inactive server span) keeps
|
||||
their LLM-call span in the request's trace. `pre_call` is litellm's generic
|
||||
"log the attempt" hook, so it also fires for synthetic proxy-gate error logs
|
||||
(auth/rate-limit rejections); those carry `LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL`
|
||||
and are skipped, so a request rejected before reaching a provider never produces
|
||||
a phantom CLIENT span.
|
||||
5. **Guardrails / services**: the post-call and service hooks emit guardrail and
|
||||
service spans the same way — typed data → engine → span. Service spans
|
||||
(Redis/Postgres) are dispatched by `litellm/_service_logger.py`, which
|
||||
recognizes the V2 `OpenTelemetryV2` logger (a plain `CustomLogger`, not a
|
||||
subclass of the legacy `OpenTelemetry`). It hands every service call to the
|
||||
logger — including calls with no parent span — and the V2 adapter decides the
|
||||
role (`DB_CALL` vs `SERVICE`), the parent (ambient → threaded → root), and
|
||||
whether the call is a traceable operation or a metrics-only ping. Guardrail
|
||||
span data is built from the typed, provider-agnostic
|
||||
`StandardLoggingGuardrailInformation` — no single provider's field shape is
|
||||
assumed.
|
||||
6. **Export**: each span ends and is handed to the provider's span processors,
|
||||
which export to the configured backends (OTLP, console, in-memory, …).
|
||||
|
||||
## Components
|
||||
|
||||
### Sources of truth (`model/`, no OpenTelemetry import)
|
||||
|
||||
These define the shape of a span without depending on the OTel SDK, so they can
|
||||
be imported anywhere. They live in [`model/`](./model) and form a closed set —
|
||||
nothing here imports outside it:
|
||||
|
||||
- [`semconv.py`](./model/semconv.py) — attribute-key constants (`gen_ai.*`, `http.*`,
|
||||
`litellm.*`), the GenAI operation/provider enums, and the functions that map
|
||||
LiteLLM provider/call-type strings onto convention values.
|
||||
- [`spans.py`](./model/spans.py) — the span registry: every span role, its OTel span
|
||||
kind, its place in the hierarchy, and its name builder.
|
||||
- [`payloads.py`](./model/payloads.py) — frozen dataclasses (`LLMCallSpanData`,
|
||||
`GuardrailSpanData`, `ServiceSpanData`, …) built from heterogeneous logging
|
||||
payloads via `from_*` classmethods.
|
||||
- [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings
|
||||
model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate.
|
||||
`capture_span_content` gates whether prompt/response bodies may be written as
|
||||
span attributes; it defaults **off** (`no_content`). The Baggage allowlists are
|
||||
configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` /
|
||||
`LITELLM_OTEL_BAGGAGE_METADATA_KEYS` /
|
||||
`LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or
|
||||
`baggage_promoted_keys` / `baggage_metadata_keys` /
|
||||
`baggage_team_metadata_keys` (YAML lists) under `callback_settings.otel` in
|
||||
`config.yaml` — the latter reach the config through the logger's constructor
|
||||
kwargs. `baggage_team_metadata_keys` is empty by default, so none of a team's
|
||||
free-form metadata is promoted until each sub-key is explicitly allowlisted.
|
||||
- [`baggage.py`](./model/baggage.py) — the single definition of which request-identity
|
||||
values are promoted into Baggage (so child spans inherit them) and under which
|
||||
attribute keys.
|
||||
- [`utils.py`](./model/utils.py) — value coercion, JSON serialization, and
|
||||
extractor-table application, shared across the package.
|
||||
|
||||
### Engine
|
||||
|
||||
- [`emitter.py`](./emitter.py) — `SpanEmitter.emit(role, data)`: dedupe → start
|
||||
the span → run the mapper chain to stamp attributes → set status → end. It
|
||||
owns no attribute keys. The dedupe set (which coalesces the sync+async firing
|
||||
of one request) is a bounded LRU so it can't grow without limit.
|
||||
- [`mappers/`](./mappers) — each mapper turns typed span data into a flat
|
||||
`{attribute key: value}` dict. They compose: listing several mapper names in
|
||||
the config layers multiple attribute vocabularies onto the same span.
|
||||
- `genai` — the canonical OpenTelemetry GenAI vocabulary, always present.
|
||||
- `legacy` — an additional vocabulary using the older semconv-ai / Traceloop
|
||||
attribute key names, for backends that read those.
|
||||
- `openinference`, `langfuse`, `weave`, `langtrace` — vendor vocabularies.
|
||||
- `resolve_mappers(names)` turns config names into mapper instances.
|
||||
|
||||
### Plumbing (`plumbing/`)
|
||||
|
||||
The OTel-SDK wiring. Everything here imports only `model/` and each other; it
|
||||
lives in [`plumbing/`](./plumbing):
|
||||
|
||||
- [`providers.py`](./plumbing/providers.py) — builds the `TracerProvider`, its exporters
|
||||
(from `ExporterSpec`s), and the span processor that copies allowlisted Baggage
|
||||
entries onto every span. `register_exporter_factory(kind, factory)` lets a
|
||||
preset contribute a custom exporter `kind` (e.g. one that fetches an auth
|
||||
token lazily) without coupling this module to any vendor.
|
||||
- [`context.py`](./plumbing/context.py) — trace-context and Baggage read/write helpers.
|
||||
- [`routing.py`](./plumbing/routing.py) — `TenantTracerCache`: when a request carries
|
||||
team/key-scoped vendor credentials, route its spans through a credential-keyed
|
||||
`TracerProvider` so one logger serves many tenants. The cache is a bounded LRU
|
||||
that flushes + shuts down evicted providers, since the key derives from
|
||||
request-supplied credentials and must not grow (or leak threads) without limit.
|
||||
- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments.
|
||||
|
||||
### Adapter
|
||||
|
||||
- [`logger.py`](./logger.py) — `OpenTelemetryV2`, a `CustomLogger` that
|
||||
translates LiteLLM's logging callbacks into typed span data and hands them to
|
||||
the engine. The LLM-call span is opened at the `log_pre_api_call` boundary
|
||||
(parented to the live server span via ambient context) and closed at the async
|
||||
success/failure callback; the open span is held in a bounded cache keyed by
|
||||
`litellm_call_id`, never threaded through a metadata dict. The logger registers
|
||||
itself into `litellm.input_callback` so `Logging.pre_call` fires the boundary
|
||||
hook.
|
||||
- [`mount.py`](./mount.py) — `instrument_fastapi_app(app)`, the single call site
|
||||
that attaches `opentelemetry-instrumentation-fastapi` for SERVER spans. It owns
|
||||
the health-check exclusion default (`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`) and the
|
||||
passthrough span-naming hook (`PASSTHROUGH_PREFIXES`) so `proxy_server` carries
|
||||
no OTel detail. A safe no-op when the gate is off or the instrumentation package
|
||||
is absent; must be called at app-creation time (the middleware stack freezes
|
||||
once the app serves).
|
||||
|
||||
### Presets
|
||||
|
||||
- [`presets/`](./presets) — each preset reads one integration's env vars and
|
||||
returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies
|
||||
+ resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`,
|
||||
`"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped
|
||||
credentials also provide a per-request OTLP header builder
|
||||
(`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time:
|
||||
AgentOps, for example, mints its JWT lazily inside a custom exporter on the
|
||||
first export (in the `BatchSpanProcessor` worker thread), never on the event
|
||||
loop.
|
||||
|
||||
## Extending
|
||||
|
||||
- **A new attribute vocabulary for a backend**: add a mapper in `mappers/`
|
||||
(a class with a `map(data) -> AttributeMap` method, typically built from
|
||||
`key -> extractor` tables) and register it in `mappers/__init__._MAPPER_BY_NAME`.
|
||||
- **A new integration**: add a preset in `presets/` that returns an
|
||||
`OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`.
|
||||
If it supports dynamic credentials, add a header builder to
|
||||
`DYNAMIC_HEADERS_BY_CALLBACK`.
|
||||
- **A new span kind**: add a role to `spans.py` (registry entry + name builder),
|
||||
a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s).
|
||||
118
litellm/integrations/otel/__init__.py
Normal file
118
litellm/integrations/otel/__init__.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
"""Typed, semconv-aligned OpenTelemetry instrumentation for LiteLLM.
|
||||
|
||||
The three sources of truth — attribute keys (:mod:`semconv`), the span and
|
||||
hierarchy registry (:mod:`spans`), and the typed span-data inputs
|
||||
(:mod:`payloads`) — plus :mod:`config` are exported here and are free of any
|
||||
``opentelemetry`` import. The engine layer (``emitter``, ``providers``,
|
||||
``context``, ``metrics``) and the ``CustomLogger`` adapter (``logger``) are
|
||||
reached via their submodule paths so that importing this package never
|
||||
requires the OTel SDK.
|
||||
|
||||
The ``LITELLM_OTEL_V2`` env var gates whether the factory in
|
||||
``litellm_core_utils.litellm_logging`` constructs the ``OpenTelemetryV2``
|
||||
class (from :mod:`logger`).
|
||||
"""
|
||||
|
||||
from litellm.integrations.otel.model.config import (
|
||||
OTEL_V2_ENV,
|
||||
OpenTelemetryV2Config,
|
||||
is_otel_v2_enabled,
|
||||
)
|
||||
from litellm.integrations.otel.model.baggage import (
|
||||
BAGGAGE_PROMOTED_KEYS,
|
||||
DEFAULT_BAGGAGE_METADATA_KEYS,
|
||||
promoted_baggage,
|
||||
)
|
||||
from litellm.integrations.otel.model.metadata import (
|
||||
RequestContext,
|
||||
RequestIdentity,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
LLMRequestParams,
|
||||
LLMUsage,
|
||||
MCPToolCallSpanData,
|
||||
ProxyRequestSpanData,
|
||||
ServerInfo,
|
||||
ServiceSpanData,
|
||||
SpanError,
|
||||
is_mcp_tool_call,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
HTTP,
|
||||
MCP,
|
||||
Client,
|
||||
Error,
|
||||
GenAI,
|
||||
GenAIOperation,
|
||||
GenAIProvider,
|
||||
JsonRpc,
|
||||
LiteLLM,
|
||||
MCPMethod,
|
||||
Metric,
|
||||
Network,
|
||||
NetworkTransport,
|
||||
Server,
|
||||
resolve_operation,
|
||||
resolve_provider,
|
||||
)
|
||||
from litellm.integrations.otel.model.spans import (
|
||||
SPAN_REGISTRY,
|
||||
LiteLLMSpanKind,
|
||||
SpanRole,
|
||||
SpanSpec,
|
||||
db_system,
|
||||
span_role_for_service,
|
||||
validate_registry,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# config
|
||||
"OTEL_V2_ENV",
|
||||
"OpenTelemetryV2Config",
|
||||
"is_otel_v2_enabled",
|
||||
# semconv
|
||||
"BAGGAGE_PROMOTED_KEYS",
|
||||
"DB",
|
||||
"DEFAULT_BAGGAGE_METADATA_KEYS",
|
||||
"Client",
|
||||
"Error",
|
||||
"GenAI",
|
||||
"GenAIOperation",
|
||||
"GenAIProvider",
|
||||
"HTTP",
|
||||
"JsonRpc",
|
||||
"LiteLLM",
|
||||
"MCP",
|
||||
"MCPMethod",
|
||||
"Metric",
|
||||
"Network",
|
||||
"NetworkTransport",
|
||||
"Server",
|
||||
"resolve_operation",
|
||||
"resolve_provider",
|
||||
# spans
|
||||
"SPAN_REGISTRY",
|
||||
"LiteLLMSpanKind",
|
||||
"SpanRole",
|
||||
"SpanSpec",
|
||||
"db_system",
|
||||
"span_role_for_service",
|
||||
"validate_registry",
|
||||
# payloads
|
||||
"GuardrailSpanData",
|
||||
"LLMCallSpanData",
|
||||
"LLMRequestParams",
|
||||
"LLMUsage",
|
||||
"MCPToolCallSpanData",
|
||||
"ProxyRequestSpanData",
|
||||
"RequestContext",
|
||||
"RequestIdentity",
|
||||
"ServerInfo",
|
||||
"ServiceSpanData",
|
||||
"SpanError",
|
||||
"is_mcp_tool_call",
|
||||
"promoted_baggage",
|
||||
]
|
||||
190
litellm/integrations/otel/emitter.py
Normal file
190
litellm/integrations/otel/emitter.py
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
"""The span engine: dedup, start, run the mapper chain, set status, end."""
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import Callable, Sequence
|
||||
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import Span, Tracer
|
||||
from opentelemetry.trace.status import Status, StatusCode
|
||||
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
MCPToolCallSpanData,
|
||||
ServiceSpanData,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
from litellm.integrations.otel.model.semconv import Error
|
||||
from litellm.integrations.otel.model.spans import (
|
||||
SPAN_REGISTRY,
|
||||
SpanRole,
|
||||
guardrail_span_name,
|
||||
llm_call_span_name,
|
||||
mcp_tool_call_span_name,
|
||||
service_span_name,
|
||||
)
|
||||
|
||||
# Roles emit() knows how to name and emit. PROXY_REQUEST and the management
|
||||
# routes are SERVER spans owned by the mounted FastAPI instrumentor, so they
|
||||
# have no builder here.
|
||||
_NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = {
|
||||
SpanRole.LLM_CALL: llm_call_span_name,
|
||||
SpanRole.MCP_TOOL_CALL: mcp_tool_call_span_name,
|
||||
SpanRole.GUARDRAIL: guardrail_span_name,
|
||||
# DB_CALL and SERVICE are both built from ServiceSpanData; they differ only in
|
||||
# span kind (CLIENT vs INTERNAL) and attribute vocabulary, not in naming.
|
||||
SpanRole.DB_CALL: service_span_name,
|
||||
SpanRole.SERVICE: service_span_name,
|
||||
}
|
||||
|
||||
# Cap on the dedup cache. It only needs to coalesce the sync+async firing window
|
||||
# of a single in-flight request, so a bounded LRU keeps memory flat on a
|
||||
# long-running proxy while still covering every concurrently-open call.
|
||||
_DEDUP_CACHE_MAX = 10_000
|
||||
|
||||
|
||||
class SpanEmitter:
|
||||
def __init__(
|
||||
self,
|
||||
tracer: Tracer,
|
||||
config: OpenTelemetryV2Config,
|
||||
mappers: Sequence[AttributeMapper] | None = None,
|
||||
) -> None:
|
||||
self._tracer = tracer
|
||||
self._config = config
|
||||
# The mapper chain is the sole source of span attributes. When not
|
||||
# passed in, resolve it from the config so there's one source of truth.
|
||||
self._mappers: list[AttributeMapper] = (
|
||||
list(mappers)
|
||||
if mappers is not None
|
||||
else resolve_mappers(config.mapper_names)
|
||||
)
|
||||
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
|
||||
# only — the value is unused — so it behaves like a capped set.
|
||||
self._emitted: "OrderedDict[tuple[str, SpanRole], None]" = OrderedDict()
|
||||
|
||||
# -- low-level helpers --------------------------------------------------- #
|
||||
|
||||
def start_span(
|
||||
self,
|
||||
role: SpanRole,
|
||||
name: str,
|
||||
parent_context: Context | None = None,
|
||||
start_time_ns: int | None = None,
|
||||
*,
|
||||
tracer: Tracer | None = None,
|
||||
) -> Span:
|
||||
"""Start a span for ``role`` without dedup or attribute mapping.
|
||||
|
||||
For callers that own and manage their own span lifecycle. ``tracer``
|
||||
overrides the bound tracer for this span only, used for per-request
|
||||
multi-tenant credential routing.
|
||||
"""
|
||||
return (tracer or self._tracer).start_span(
|
||||
name,
|
||||
context=parent_context,
|
||||
kind=to_otel_span_kind(SPAN_REGISTRY[role].kind),
|
||||
start_time=start_time_ns,
|
||||
)
|
||||
|
||||
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
|
||||
"""Return True once a ``(dedup_key, role)`` pair has been emitted.
|
||||
|
||||
Guards against emitting the same span twice when a streaming call
|
||||
fires both a sync and an async logging callback.
|
||||
"""
|
||||
if not dedup_key:
|
||||
return False
|
||||
marker = (dedup_key, role)
|
||||
if marker in self._emitted:
|
||||
self._emitted.move_to_end(marker)
|
||||
return True
|
||||
self._emitted[marker] = None
|
||||
if len(self._emitted) > _DEDUP_CACHE_MAX:
|
||||
self._emitted.popitem(last=False) # evict least-recently-used
|
||||
return False
|
||||
|
||||
# -- the engine ---------------------------------------------------------- #
|
||||
|
||||
def emit(
|
||||
self,
|
||||
role: SpanRole,
|
||||
data: SpanData,
|
||||
parent_context: Context | None = None,
|
||||
*,
|
||||
start_time_ns: int | None = None,
|
||||
end_time_ns: int | None = None,
|
||||
tracer: Tracer | None = None,
|
||||
) -> Span | None:
|
||||
"""Emit one complete span: dedup, start, map attributes, status, end.
|
||||
|
||||
Return the span, or ``None`` if it was deduplicated away. ``tracer``
|
||||
overrides the bound tracer for this span, used for per-request routing.
|
||||
"""
|
||||
# LLM-call and MCP tool-call spans carry a dedup key (their request's
|
||||
# call id), so a sync+async double-firing coalesces. ``isinstance`` narrows
|
||||
# the type for mypy and keeps the engine free of duck-typed attribute reads.
|
||||
dedup_key = (
|
||||
data.identity.call_id
|
||||
if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData))
|
||||
else None
|
||||
)
|
||||
if self._seen(dedup_key, role):
|
||||
return None
|
||||
span = self.start_span(
|
||||
role,
|
||||
_NAME_BUILDERS[role](data),
|
||||
parent_context=parent_context,
|
||||
start_time_ns=start_time_ns,
|
||||
tracer=tracer,
|
||||
)
|
||||
self.finish_span(role, span, data, end_time_ns=end_time_ns)
|
||||
return span
|
||||
|
||||
def finish_span(
|
||||
self,
|
||||
role: SpanRole,
|
||||
span: Span,
|
||||
data: SpanData,
|
||||
*,
|
||||
end_time_ns: int | None = None,
|
||||
) -> None:
|
||||
"""Stamp attributes + status on an already-started ``span`` and end it.
|
||||
|
||||
The counterpart to :meth:`start_span` for callers that own a span's
|
||||
lifecycle — the LLM-call span is opened at the request's ``pre_call``
|
||||
boundary (so it parents to the live server span via real ambient context,
|
||||
never a span threaded through a metadata dict) and closed here once the
|
||||
typed payload is available. The span name is (re)built from the now-known
|
||||
data, since the boundary opener only has a provisional name.
|
||||
"""
|
||||
span.update_name(_NAME_BUILDERS[role](data))
|
||||
for mapper in self._mappers:
|
||||
for key, value in mapper.map(data).items():
|
||||
span.set_attribute(key, value)
|
||||
error = (
|
||||
data.error
|
||||
if isinstance(
|
||||
data,
|
||||
(
|
||||
LLMCallSpanData,
|
||||
MCPToolCallSpanData,
|
||||
ServiceSpanData,
|
||||
GuardrailSpanData,
|
||||
),
|
||||
)
|
||||
else None
|
||||
)
|
||||
if error and (error.error_type or error.message):
|
||||
span.set_attribute(Error.TYPE, error.error_type or "error")
|
||||
span.set_status(
|
||||
Status(StatusCode.ERROR, error.message or error.error_type or "error")
|
||||
)
|
||||
# On success leave the status UNSET (the semconv default) rather than
|
||||
# forcing OK — that matches the FastAPI server span and avoids implying a
|
||||
# span-level health signal litellm doesn't actually evaluate. Only a
|
||||
# genuine error sets a status.
|
||||
span.end(end_time=end_time_ns)
|
||||
544
litellm/integrations/otel/logger.py
Normal file
544
litellm/integrations/otel/logger.py
Normal file
|
|
@ -0,0 +1,544 @@
|
|||
"""``CustomLogger`` adapter on the OpenTelemetry span engine."""
|
||||
|
||||
from collections import OrderedDict
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast
|
||||
|
||||
from opentelemetry.context import attach, get_current
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.trace import Span, Tracer, get_current_span, use_span
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.model.baggage import promoted_baggage
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.plumbing.context import (
|
||||
is_recordable_span,
|
||||
resolve_parent_context,
|
||||
resolve_request_span_context,
|
||||
set_request_baggage,
|
||||
set_request_root_span,
|
||||
)
|
||||
from litellm.integrations.otel.emitter import SpanEmitter
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.model.metadata import (
|
||||
LLMCallEvent,
|
||||
RequestIdentity,
|
||||
model_from_request_data,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
MCPToolCallSpanData,
|
||||
ServiceSpanData,
|
||||
SpanError,
|
||||
is_mcp_tool_call,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import (
|
||||
build_tracer_provider,
|
||||
get_tracer,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
|
||||
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
|
||||
from litellm.integrations.otel.model.utils import to_ns
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import (
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingPayload,
|
||||
)
|
||||
|
||||
LITELLM_TRACER_NAME = "litellm"
|
||||
|
||||
# Any callback whose class belongs to one of these modules is "the OTel
|
||||
# callback" for proxy-global-registration purposes.
|
||||
_OTEL_MODULES = (
|
||||
"litellm.integrations.otel",
|
||||
"litellm.integrations.opentelemetry",
|
||||
)
|
||||
|
||||
|
||||
# Cap on the open-call carrier map. A span opened at ``pre_call`` that never
|
||||
# reaches a success/failure callback (e.g. a stream that only fires stream
|
||||
# events) would otherwise linger; bounding the map evicts the oldest so memory
|
||||
# stays flat on a long-running proxy while covering every concurrent in-flight
|
||||
# call.
|
||||
_OPEN_CALLS_MAX = 10_000
|
||||
|
||||
|
||||
class _LLMCallSpan:
|
||||
"""The state carried from the ``pre_call`` boundary to span close.
|
||||
|
||||
``span`` is the live span when it could be opened at the boundary (the server
|
||||
span was ambient), or ``None`` when creation was deferred because no ambient
|
||||
parent was visible — in which case the async callback creates it against its
|
||||
own (worker-copied) ambient context using ``start_time_ns``. The presence of
|
||||
a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an
|
||||
upstream call was actually attempted.
|
||||
"""
|
||||
|
||||
__slots__ = ("span", "start_time_ns")
|
||||
|
||||
def __init__(self, span: "Span | None", start_time_ns: int | None) -> None:
|
||||
self.span = span
|
||||
self.start_time_ns = start_time_ns
|
||||
|
||||
|
||||
class OpenTelemetryV2(CustomLogger):
|
||||
"""The ``CustomLogger`` for OpenTelemetry."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OpenTelemetryV2Config | None = None,
|
||||
callback_name: str | None = None,
|
||||
tracer_provider: TracerProvider | None = None,
|
||||
logger_provider: Any | None = None, # reserved for OTel logs
|
||||
meter_provider: Any | None = None, # reserved for metrics
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)
|
||||
self.callback_name = callback_name
|
||||
self._tracer_provider: TracerProvider = (
|
||||
tracer_provider
|
||||
if tracer_provider is not None
|
||||
else build_tracer_provider(self.config)
|
||||
)
|
||||
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
|
||||
self._emitter = SpanEmitter(
|
||||
self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)
|
||||
)
|
||||
self._tenant_tracers = TenantTracerCache(
|
||||
self.config, callback_name, LITELLM_TRACER_NAME
|
||||
)
|
||||
self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
|
||||
self._init_otel_logger_on_litellm_proxy()
|
||||
|
||||
# ====================================================================== #
|
||||
# Proxy global registration
|
||||
# ====================================================================== #
|
||||
|
||||
def _register_in_callback_list(self, callbacks: list) -> None:
|
||||
already_otel = any(
|
||||
cb.__class__.__module__.startswith(_OTEL_MODULES)
|
||||
for cb in callbacks
|
||||
if hasattr(cb, "__class__")
|
||||
)
|
||||
if not already_otel:
|
||||
callbacks.append(self)
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self) -> None:
|
||||
try:
|
||||
from litellm.proxy import proxy_server
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
self._register_in_callback_list(litellm.service_callback)
|
||||
self._register_in_callback_list(litellm.input_callback)
|
||||
self._register_in_callback_list(litellm._async_success_callback)
|
||||
self._register_in_callback_list(litellm._async_failure_callback)
|
||||
except Exception:
|
||||
pass
|
||||
if getattr(proxy_server, "open_telemetry_logger", None) is None:
|
||||
setattr(proxy_server, "open_telemetry_logger", self)
|
||||
|
||||
# ====================================================================== #
|
||||
# LLM-call callbacks — the span is opened at the ``pre_call`` boundary and
|
||||
# closed here. See ``log_pre_api_call``.
|
||||
# ====================================================================== #
|
||||
|
||||
def log_pre_api_call(self, model, messages, kwargs):
|
||||
"""Open the LLM-call span at the call boundary.
|
||||
|
||||
Runs synchronously inside the request task, before the upstream call —
|
||||
the one place where the live server span is genuinely the ambient OTel
|
||||
context — so the span parents to it natively, with no span threaded
|
||||
through a metadata dict. The open span is stashed on the per-request
|
||||
``LiteLLMLoggingObj`` (a typed object) and closed in the async callback.
|
||||
|
||||
When no recordable parent is visible (``pre_call`` was driven from a thread
|
||||
pool for a sync-only provider, where contextvars — and so the anchor —
|
||||
don't follow), creation is deferred: only the start time is recorded, and
|
||||
the async callback — whose worker context was copied from the request task
|
||||
and so still carries the anchor — creates the span then.
|
||||
|
||||
Synthetic proxy-gate error logs (auth/rate-limit rejections) also fire this
|
||||
hook but never made an upstream call; they are tagged and skipped so no
|
||||
phantom LLM-call span is produced.
|
||||
"""
|
||||
call = LLMCallEvent.from_dict(kwargs)
|
||||
if call.is_no_upstream_call:
|
||||
return
|
||||
call_id = call.call_id
|
||||
if call_id is None:
|
||||
return
|
||||
# Idempotent: a retried call may re-enter ``pre_call`` with the same
|
||||
# call id; keep the first span so its start time is the true one.
|
||||
if call_id in self._open_llm_calls:
|
||||
return
|
||||
start_time_ns = to_ns(datetime.now())
|
||||
span: Span | None = None
|
||||
# Parent to the request's anchored root span (stable across the request),
|
||||
# falling back to ambient on the SDK path. Open the span live only when
|
||||
# that resolves to a recordable parent; otherwise defer to the close
|
||||
# callback (the thread-pool case, where the anchor isn't visible here).
|
||||
parent_context = resolve_request_span_context()
|
||||
if is_recordable_span(get_current_span(parent_context)):
|
||||
span = self._emitter.start_span(
|
||||
SpanRole.LLM_CALL,
|
||||
call.provisional_span_name,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=start_time_ns,
|
||||
tracer=self._tenant_tracers.tracer_for(
|
||||
self.tracer, call.dynamic_params
|
||||
),
|
||||
)
|
||||
self._open_llm_calls[call_id] = _LLMCallSpan(
|
||||
span=span, start_time_ns=start_time_ns
|
||||
)
|
||||
# Evict the oldest open call if the map is over budget. A call that opens
|
||||
# but never closes (a stream that only fires stream events) would linger
|
||||
# otherwise; the evicted span is simply dropped (never exported).
|
||||
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
|
||||
self._open_llm_calls.popitem(last=False)
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
|
||||
return
|
||||
self._close_llm_call(kwargs, start_time, end_time)
|
||||
|
||||
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
|
||||
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
|
||||
return
|
||||
self._close_llm_call(kwargs, start_time, end_time)
|
||||
|
||||
def _emit_mcp_tool_call(
|
||||
self,
|
||||
kwargs: Mapping[str, Any],
|
||||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
) -> bool:
|
||||
"""Emit an MCP tool-call span when the closed request was a tool call.
|
||||
|
||||
MCP tool calls reach the success/failure callbacks like any other request
|
||||
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
|
||||
no ``pre_call`` carrier — so they get their own CLIENT span here, parented
|
||||
to the request's server span. Returns whether it handled the event, so the
|
||||
caller skips the LLM-call path. The whole span is emitted at once (there is
|
||||
no boundary to open it at), deduped on the call id by the emitter.
|
||||
"""
|
||||
raw_payload = kwargs.get("standard_logging_object")
|
||||
if not raw_payload or not is_mcp_tool_call(
|
||||
cast(Mapping[str, object], raw_payload)
|
||||
):
|
||||
return False
|
||||
payload = cast("StandardLoggingPayload", raw_payload)
|
||||
data = MCPToolCallSpanData.from_standard_logging_payload(
|
||||
payload, capture_content=self.config.capture_span_content
|
||||
)
|
||||
# A stray LLM carrier from a ``pre_call`` that mis-fired for this id would
|
||||
# otherwise linger until evicted; drop it so it's neither leaked nor closed
|
||||
# as a phantom LLM span.
|
||||
if data.identity.call_id:
|
||||
self._open_llm_calls.pop(data.identity.call_id, None)
|
||||
self._emitter.emit(
|
||||
SpanRole.MCP_TOOL_CALL,
|
||||
data,
|
||||
parent_context=resolve_request_span_context(),
|
||||
start_time_ns=to_ns(start_time),
|
||||
end_time_ns=to_ns(end_time),
|
||||
)
|
||||
return True
|
||||
|
||||
def _close_llm_call(
|
||||
self,
|
||||
kwargs: Mapping[str, Any],
|
||||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
) -> Span | None:
|
||||
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred).
|
||||
|
||||
No carrier for this call id means ``pre_call`` never ran — the request was
|
||||
rejected at the gate or blocked by a pre-call guardrail before any upstream
|
||||
call — so there is nothing to record and no phantom span.
|
||||
"""
|
||||
call = LLMCallEvent.from_dict(kwargs)
|
||||
call_id = call.call_id
|
||||
# ``pop`` is the dedup: this method runs from both the success and failure
|
||||
# paths, and whichever fires first removes the carrier and closes the span.
|
||||
carrier = self._open_llm_calls.pop(call_id, None) if call_id else None
|
||||
if carrier is None:
|
||||
return None
|
||||
payload = call.payload
|
||||
if payload is None:
|
||||
if carrier.span is not None:
|
||||
# Opened at the boundary but the payload never materialized — end
|
||||
# it (named provisionally) so it isn't leaked as an open span.
|
||||
carrier.span.end(end_time=to_ns(end_time))
|
||||
return None
|
||||
data = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload, capture_content=self.config.capture_span_content
|
||||
)
|
||||
end_time_ns = to_ns(end_time)
|
||||
if carrier.span is not None:
|
||||
# Born at the boundary: stamp attributes from the typed payload, set
|
||||
# status, and end it. Its parent (the server span) was captured at
|
||||
# creation from real ambient context.
|
||||
self._emitter.finish_span(
|
||||
SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns
|
||||
)
|
||||
return carrier.span
|
||||
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
|
||||
# The worker copied the request task's context, which carries the anchored
|
||||
# root span — parent to it (ambient fallback on the SDK path). Seed identity
|
||||
# Baggage so the span — and the SDK path, which has none — is labeled
|
||||
# consistently.
|
||||
parent_ctx = resolve_request_span_context()
|
||||
bag = promoted_baggage(
|
||||
data.identity,
|
||||
data.request_model,
|
||||
promoted_keys=tuple(self.config.baggage_promoted_keys),
|
||||
metadata_keys=tuple(self.config.baggage_metadata_keys),
|
||||
team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
|
||||
)
|
||||
if bag:
|
||||
parent_ctx = set_request_baggage(bag, context=parent_ctx)
|
||||
return self._emitter.emit(
|
||||
SpanRole.LLM_CALL,
|
||||
data,
|
||||
parent_context=parent_ctx,
|
||||
start_time_ns=carrier.start_time_ns,
|
||||
end_time_ns=end_time_ns,
|
||||
tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
|
||||
)
|
||||
|
||||
# ====================================================================== #
|
||||
# Service hooks
|
||||
# ====================================================================== #
|
||||
|
||||
async def async_service_success_hook(
|
||||
self,
|
||||
payload: Any,
|
||||
parent_otel_span: Span | None = None,
|
||||
start_time: datetime | float | None = None,
|
||||
end_time: datetime | float | None = None,
|
||||
event_metadata: dict | None = None,
|
||||
) -> None:
|
||||
self._emit_service(
|
||||
payload,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
event_metadata=event_metadata,
|
||||
error_override=None,
|
||||
)
|
||||
|
||||
async def async_service_failure_hook(
|
||||
self,
|
||||
payload: Any,
|
||||
error: str | None = "",
|
||||
parent_otel_span: Span | None = None,
|
||||
start_time: datetime | float | None = None,
|
||||
end_time: datetime | float | None = None,
|
||||
event_metadata: dict | None = None,
|
||||
) -> None:
|
||||
self._emit_service(
|
||||
payload,
|
||||
parent_otel_span=parent_otel_span,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
event_metadata=event_metadata,
|
||||
error_override=error or "error",
|
||||
)
|
||||
|
||||
def _emit_service(
|
||||
self,
|
||||
payload: Any,
|
||||
*,
|
||||
parent_otel_span: Span | None,
|
||||
start_time: datetime | float | None,
|
||||
end_time: datetime | float | None,
|
||||
event_metadata: dict | None,
|
||||
error_override: str | None,
|
||||
) -> Span | None:
|
||||
data = ServiceSpanData.from_payload(payload, event_metadata=event_metadata)
|
||||
# Decide whether this service call is a span at all, and of what kind.
|
||||
# ``None`` means metrics-only (framework instrumentation that duplicates a
|
||||
# gen-AI span — ``self``/``router``/``proxy_pre_call`` — or ``auth``, which
|
||||
# gets a live phase span instead). Those still feed Prometheus/Datadog via
|
||||
# their own hooks; they just never enter the trace.
|
||||
role = span_role_for_service(data.service_name)
|
||||
if role is None:
|
||||
return None
|
||||
# A metrics-only ping with neither timing nor a parent (in-memory queue
|
||||
# gauges) is not a traceable operation; a span for it would be a
|
||||
# zero-duration root with no context, so skip it. Real background work
|
||||
# (budget/reset jobs, spend flush) passes start/end times and still emits
|
||||
# as a root; anything with a parent emits regardless.
|
||||
if (
|
||||
error_override is None
|
||||
and start_time is None
|
||||
and end_time is None
|
||||
and parent_otel_span is None
|
||||
):
|
||||
return None
|
||||
if error_override is not None and data.error is None:
|
||||
data = ServiceSpanData(
|
||||
service_name=data.service_name,
|
||||
call_type=data.call_type,
|
||||
error=SpanError(message=error_override),
|
||||
event_metadata=data.event_metadata,
|
||||
)
|
||||
# Parent like every other span: ambient context first (so identity Baggage
|
||||
# rides along and the call nests under whatever request phase is active —
|
||||
# e.g. a DB lookup under the live ``auth`` span), falling back to the
|
||||
# server span the proxy threaded as ``parent_otel_span``. A background
|
||||
# service call has neither, so it starts its own root trace.
|
||||
parent_context = resolve_parent_context(threaded=parent_otel_span)
|
||||
return self._emitter.emit(
|
||||
role,
|
||||
data,
|
||||
parent_context=parent_context,
|
||||
start_time_ns=to_ns(start_time),
|
||||
end_time_ns=to_ns(end_time),
|
||||
)
|
||||
|
||||
# ====================================================================== #
|
||||
# async_post_call_* hooks — emit guardrail spans. The server span's status
|
||||
# / errors are the FastAPI instrumentor's job, so we don't touch it here.
|
||||
# ====================================================================== #
|
||||
|
||||
def seed_request_identity(self, user_api_key_dict: Any, model: Any = None) -> None:
|
||||
"""Attach request-identity Baggage to the current context + server span.
|
||||
|
||||
Seeding identity into Baggage makes **every** span emitted afterwards for
|
||||
this request — LLM call, guardrail, DB call — inherit it via
|
||||
``LiteLLMBaggageSpanProcessor``. Called once at the auth boundary (as soon
|
||||
as the key resolves) so post-auth spans are labeled consistently; the
|
||||
Baggage rides the request task's contextvar from there on. Auth-internal
|
||||
DB lookups that run before the key is known stay unlabeled — identity
|
||||
isn't determined yet, which is correct.
|
||||
"""
|
||||
try:
|
||||
identity = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
|
||||
bag = promoted_baggage(
|
||||
identity,
|
||||
model,
|
||||
promoted_keys=tuple(self.config.baggage_promoted_keys),
|
||||
metadata_keys=tuple(self.config.baggage_metadata_keys),
|
||||
team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
|
||||
)
|
||||
if bag:
|
||||
# Attach (no detach): the contextvar is scoped to this request's
|
||||
# asyncio task and is reclaimed when the task ends.
|
||||
attach(set_request_baggage(bag, context=get_current()))
|
||||
# The server span was started by the instrumentor before this ran,
|
||||
# so the Baggage processor (which only fires at span start) won't
|
||||
# backfill it — stamp identity on it directly.
|
||||
server_span = get_current_span()
|
||||
if is_recordable_span(server_span):
|
||||
# Re-capture the anchor here too: this runs post-auth with the
|
||||
# server span active and covers entrypoints that bypass
|
||||
# ``create_litellm_proxy_request_started_span`` (e.g. the SDK
|
||||
# path's ``async_pre_call_hook``). Idempotent.
|
||||
set_request_root_span(server_span)
|
||||
for key, value in bag.items():
|
||||
server_span.set_attribute(key, value)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@contextmanager
|
||||
def start_phase_span(self, name: str) -> "Iterator[Span]":
|
||||
span = self._emitter.start_span(SpanRole.SERVICE, name)
|
||||
with use_span(span, end_on_exit=True):
|
||||
yield span
|
||||
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: Any,
|
||||
cache: Any,
|
||||
data: dict,
|
||||
call_type: Any,
|
||||
) -> dict:
|
||||
self.seed_request_identity(
|
||||
user_api_key_dict,
|
||||
model=model_from_request_data(data),
|
||||
)
|
||||
return data
|
||||
|
||||
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
|
||||
# Emitted by the guardrail-recording code the moment a guardrail finishes,
|
||||
# not from a post-call hook — that hook does not fire on every path (a
|
||||
# pass-through request that passes its guardrails never reaches it), which
|
||||
# left passing guardrails without a span.
|
||||
#
|
||||
# A guardrail is a sibling of the LLM call under the request's root span,
|
||||
# so parent it to the explicit anchor — never the active span, which during
|
||||
# a pre_call guardrail can be the live ``auth`` phase span. Emit with the
|
||||
# guardrail's actual execution window so a pre_call guardrail is placed
|
||||
# before the LLM call rather than at emission time. One entry in, one span
|
||||
# out — the module-level entry point routes each entry to this single
|
||||
# registered logger so a guardrail is never emitted more than once.
|
||||
data = GuardrailSpanData.from_logging_entry(entry)
|
||||
self._emitter.emit(
|
||||
SpanRole.GUARDRAIL,
|
||||
data,
|
||||
parent_context=resolve_request_span_context(),
|
||||
start_time_ns=to_ns(data.start_time),
|
||||
end_time_ns=to_ns(data.end_time),
|
||||
)
|
||||
|
||||
def create_litellm_proxy_request_started_span(
|
||||
self, start_time: datetime, headers: Mapping[str, str] | None
|
||||
) -> Span | None:
|
||||
span = get_current_span()
|
||||
if not is_recordable_span(span):
|
||||
return None
|
||||
set_request_root_span(span)
|
||||
return span
|
||||
|
||||
|
||||
def _registered_v2_logger() -> "OpenTelemetryV2 | None":
|
||||
try:
|
||||
from litellm.proxy import proxy_server
|
||||
except Exception:
|
||||
return None
|
||||
logger = getattr(proxy_server, "open_telemetry_logger", None)
|
||||
return logger if isinstance(logger, OpenTelemetryV2) else None
|
||||
|
||||
|
||||
def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None:
|
||||
"""Emit a guardrail span on the registered v2 OTel logger.
|
||||
|
||||
Called by the guardrail-recording code the moment a guardrail finishes, so a
|
||||
span is produced regardless of whether a post-call hook later runs (it does
|
||||
not on the pass-through allow path). Routes through the single canonical
|
||||
logger — the same one every other v2 entry point uses — so a guardrail
|
||||
recorded once yields exactly one span; fanning out across every reachable
|
||||
``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span
|
||||
emission must never break guardrail evaluation.
|
||||
"""
|
||||
logger = _registered_v2_logger()
|
||||
if logger is None:
|
||||
return
|
||||
try:
|
||||
logger.emit_guardrail_span(entry)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None:
|
||||
logger = _registered_v2_logger()
|
||||
if logger is not None:
|
||||
logger.seed_request_identity(user_api_key_dict, model=model)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def phase_span(name: str) -> "Iterator[Span | None]":
|
||||
logger = _registered_v2_logger()
|
||||
if logger is None:
|
||||
yield None
|
||||
return
|
||||
with logger.start_phase_span(name) as span:
|
||||
yield span
|
||||
58
litellm/integrations/otel/mappers/__init__.py
Normal file
58
litellm/integrations/otel/mappers/__init__.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
"""Attribute mappers: pure ``LLMCallSpanData -> {attribute key: value}`` functions.
|
||||
|
||||
Composition over inheritance: vocabularies layer onto the same span. Listing
|
||||
``["genai", "openinference"]`` in ``config.mapper_names`` makes every span
|
||||
carry both the canonical ``gen_ai.*`` keys and the OpenInference (Arize +
|
||||
Phoenix) keys. Add ``"langfuse"`` and it works for all three backends at once.
|
||||
"""
|
||||
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from litellm.integrations.otel.mappers.base import (
|
||||
AttributeMap,
|
||||
AttributeMapper,
|
||||
AttrValue,
|
||||
)
|
||||
from litellm.integrations.otel.mappers.genai import GenAIMapper
|
||||
from litellm.integrations.otel.mappers.langfuse import LangfuseMapper
|
||||
from litellm.integrations.otel.mappers.langtrace import LangtraceMapper
|
||||
from litellm.integrations.otel.mappers.legacy import LegacyMapper
|
||||
from litellm.integrations.otel.mappers.openinference import OpenInferenceMapper
|
||||
from litellm.integrations.otel.mappers.weave import WeaveMapper
|
||||
|
||||
# Registry keyed by ``config.mapper_names`` entries.
|
||||
_MAPPER_BY_NAME: dict[str, Callable[[], AttributeMapper]] = {
|
||||
"genai": GenAIMapper,
|
||||
"legacy": LegacyMapper,
|
||||
"openinference": OpenInferenceMapper,
|
||||
"langfuse": LangfuseMapper,
|
||||
"weave": WeaveMapper,
|
||||
"langtrace": LangtraceMapper,
|
||||
}
|
||||
|
||||
|
||||
def resolve_mappers(names: Iterable[str]) -> list[AttributeMapper]:
|
||||
"""Resolve mapper names to instances. Unknown names raise ``ValueError``."""
|
||||
out: list[AttributeMapper] = []
|
||||
for name in names:
|
||||
factory = _MAPPER_BY_NAME.get(name)
|
||||
if factory is None:
|
||||
raise ValueError(
|
||||
f"unknown mapper name {name!r}; known: " f"{sorted(_MAPPER_BY_NAME)}"
|
||||
)
|
||||
out.append(factory())
|
||||
return out
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AttributeMap",
|
||||
"AttributeMapper",
|
||||
"AttrValue",
|
||||
"GenAIMapper",
|
||||
"LangfuseMapper",
|
||||
"LangtraceMapper",
|
||||
"LegacyMapper",
|
||||
"OpenInferenceMapper",
|
||||
"WeaveMapper",
|
||||
"resolve_mappers",
|
||||
]
|
||||
37
litellm/integrations/otel/mappers/base.py
Normal file
37
litellm/integrations/otel/mappers/base.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""Mapper protocol and attribute value types."""
|
||||
|
||||
from typing import Sequence
|
||||
|
||||
from typing_extensions import Protocol, runtime_checkable
|
||||
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
MCPToolCallSpanData,
|
||||
ServiceSpanData,
|
||||
)
|
||||
|
||||
AttrScalar = str | bool | int | float
|
||||
# Mirrors ``opentelemetry.util.types.AttributeValue`` (homogeneous sequences)
|
||||
# without importing the SDK, so mappers stay OTel-free.
|
||||
AttrValue = (
|
||||
AttrScalar | Sequence[str] | Sequence[bool] | Sequence[int] | Sequence[float]
|
||||
)
|
||||
AttributeMap = dict[str, AttrValue]
|
||||
|
||||
# The closed set of span-data types the engine routes through the mapper chain.
|
||||
# Server spans (PROXY_REQUEST + management routes) belong to the mounted FastAPI
|
||||
# instrumentor, not the mapper chain.
|
||||
SpanData = LLMCallSpanData | MCPToolCallSpanData | GuardrailSpanData | ServiceSpanData
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class AttributeMapper(Protocol):
|
||||
"""Maps a typed span input to a flat dict of OTel span attributes.
|
||||
|
||||
One method per mapper, dispatched internally on the ``data`` type. The
|
||||
engine calls this uniformly for every span kind — mappers that don't speak
|
||||
a given type return ``{}``. This is why the engine contains no attribute keys.
|
||||
"""
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap: ...
|
||||
159
litellm/integrations/otel/mappers/genai.py
Normal file
159
litellm/integrations/otel/mappers/genai.py
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
"""Canonical OpenTelemetry GenAI semantic-convention mapper (always active).
|
||||
|
||||
Owns the attribute schema for every span kind the engine emits — LLM call,
|
||||
guardrail, and service — so the engine itself never references attribute keys.
|
||||
|
||||
Each span kind declares its schema as a flat ``attribute key -> extractor``
|
||||
table: one lambda per mapping operation, applied against the typed span data.
|
||||
"""
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import collect, drop_none
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
MCPToolCallSpanData,
|
||||
ServiceSpanData,
|
||||
ToolDefinition,
|
||||
)
|
||||
from litellm.integrations.otel.model.semconv import (
|
||||
DB,
|
||||
MCP,
|
||||
Error,
|
||||
GenAI,
|
||||
LiteLLM,
|
||||
Server,
|
||||
)
|
||||
from litellm.integrations.otel.model.spans import db_system
|
||||
|
||||
|
||||
class GenAIMapper:
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
GenAI.OPERATION_NAME: lambda d: d.operation.value,
|
||||
GenAI.PROVIDER_NAME: lambda d: d.provider or None,
|
||||
GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
|
||||
GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
|
||||
GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
|
||||
GenAI.REQUEST_TOP_K: lambda d: d.request_params.top_k,
|
||||
GenAI.REQUEST_MAX_TOKENS: lambda d: d.request_params.max_tokens,
|
||||
GenAI.REQUEST_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty,
|
||||
GenAI.REQUEST_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty,
|
||||
GenAI.REQUEST_STOP_SEQUENCES: lambda d: (
|
||||
list(d.request_params.stop_sequences)
|
||||
if d.request_params.stop_sequences
|
||||
else None
|
||||
),
|
||||
GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
|
||||
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
|
||||
GenAI.RESPONSE_ID: lambda d: d.response_id,
|
||||
GenAI.RESPONSE_FINISH_REASONS: lambda d: (
|
||||
list(d.finish_reasons) if d.finish_reasons else None
|
||||
),
|
||||
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
|
||||
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
|
||||
Error.TYPE: lambda d: d.error.error_type if d.error else None,
|
||||
Server.ADDRESS: lambda d: d.server.address if d.server else None,
|
||||
Server.PORT: lambda d: d.server.port if d.server else None,
|
||||
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
|
||||
# The provider/underlying model is only known once routing has picked a
|
||||
# deployment, so it can't ride identity Baggage (seeded at auth, before
|
||||
# routing) onto the boundary-born LLM span — stamp it directly here.
|
||||
LiteLLM.PROVIDER_MODEL: lambda d: d.identity.provider_model or None,
|
||||
f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
|
||||
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
|
||||
}
|
||||
|
||||
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
|
||||
"name": lambda t: t.name,
|
||||
"description": lambda t: t.description or None,
|
||||
"parameters": lambda t: t.parameters_json or None,
|
||||
}
|
||||
|
||||
_MCP_ATTRS: dict[str, Callable[[MCPToolCallSpanData], AttrValue | None]] = {
|
||||
GenAI.OPERATION_NAME: lambda d: d.operation.value,
|
||||
MCP.METHOD_NAME: lambda d: d.method,
|
||||
MCP.SESSION_ID: lambda d: d.session_id,
|
||||
GenAI.TOOL_NAME: lambda d: d.tool_name or None,
|
||||
GenAI.TOOL_CALL_ARGUMENTS: lambda d: d.arguments_json,
|
||||
GenAI.TOOL_CALL_RESULT: lambda d: d.result_json,
|
||||
LiteLLM.MCP_SERVER_NAME: lambda d: d.server_name,
|
||||
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
|
||||
f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
|
||||
}
|
||||
|
||||
_GUARDRAIL_ATTRS: dict[str, Callable[[GuardrailSpanData], AttrValue | None]] = {
|
||||
LiteLLM.GUARDRAIL_NAME: lambda d: d.guardrail_name,
|
||||
LiteLLM.GUARDRAIL_MODE: lambda d: d.mode,
|
||||
LiteLLM.GUARDRAIL_STATUS: lambda d: d.status,
|
||||
LiteLLM.GUARDRAIL_PROVIDER: lambda d: d.provider,
|
||||
LiteLLM.GUARDRAIL_ACTION: lambda d: d.action,
|
||||
LiteLLM.GUARDRAIL_RESPONSE: lambda d: d.response_json,
|
||||
LiteLLM.GUARDRAIL_VIOLATION_CATEGORIES: lambda d: (
|
||||
list(d.violation_categories) if d.violation_categories else None
|
||||
),
|
||||
LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score,
|
||||
LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score,
|
||||
LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count,
|
||||
LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration,
|
||||
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
|
||||
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
|
||||
LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
|
||||
}
|
||||
|
||||
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {
|
||||
LiteLLM.SERVICE_NAME: lambda d: d.service_name,
|
||||
LiteLLM.SERVICE_CALL_TYPE: lambda d: d.call_type,
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case MCPToolCallSpanData():
|
||||
return collect(self._MCP_ATTRS, data)
|
||||
case GuardrailSpanData():
|
||||
return self._guardrail(data)
|
||||
case ServiceSpanData():
|
||||
return self._service(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
attrs = collect(cls._LLM_CALL_ATTRS, data)
|
||||
attrs.update(
|
||||
drop_none(
|
||||
{
|
||||
f"gen_ai.tool.{idx}.{suffix}": extract(tool)
|
||||
for idx, tool in enumerate(data.tools)
|
||||
for suffix, extract in cls._TOOL_ATTRS.items()
|
||||
}
|
||||
)
|
||||
)
|
||||
return attrs
|
||||
|
||||
@classmethod
|
||||
def _guardrail(cls, data: GuardrailSpanData) -> AttributeMap:
|
||||
return collect(cls._GUARDRAIL_ATTRS, data)
|
||||
|
||||
@classmethod
|
||||
def _service(cls, data: ServiceSpanData) -> AttributeMap:
|
||||
attrs = collect(cls._SERVICE_ATTRS, data)
|
||||
# An outbound datastore call (DB_CALL / CLIENT span) also carries db.*
|
||||
# semconv. Internal services (router, budget jobs, …) have no db.system,
|
||||
# so they get only the litellm.service.* keys above.
|
||||
system = db_system(data.service_name)
|
||||
if system is not None:
|
||||
attrs[DB.SYSTEM_NAME] = system
|
||||
if data.call_type:
|
||||
attrs[DB.OPERATION_NAME] = data.call_type
|
||||
attrs.update(
|
||||
{
|
||||
f"{LiteLLM.METADATA_PREFIX}{key}": value
|
||||
for key, value in data.event_metadata.items()
|
||||
}
|
||||
)
|
||||
return attrs
|
||||
84
litellm/integrations/otel/mappers/langfuse.py
Normal file
84
litellm/integrations/otel/mappers/langfuse.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
"""Langfuse OTLP attribute mapper.
|
||||
|
||||
Langfuse ingests OTLP spans and reads from its own vendor namespace
|
||||
(``langfuse.observation.*``, ``langfuse.trace.*``). Compose this mapper after
|
||||
``GenAIMapper`` to send canonical + Langfuse-flavored spans simultaneously.
|
||||
|
||||
Every attribute is declared as a ``key -> extractor`` table entry (one callable
|
||||
per mapping operation): ``_LLM_CALL_ATTRS`` for scalars and ``_BLOB_ATTRS`` for
|
||||
the JSON-serialized payloads. ``_llm_call`` just applies both tables.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Callable
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import (
|
||||
collect,
|
||||
json_if,
|
||||
output_messages,
|
||||
serialize_messages,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
LLMCallSpanData,
|
||||
LLMRequestParams,
|
||||
LLMUsage,
|
||||
)
|
||||
|
||||
|
||||
class LangfuseMapper:
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"langfuse.observation.type": lambda d: "generation",
|
||||
"langfuse.observation.model.name": lambda d: d.request_model or None,
|
||||
"langfuse.observation.metadata.provider": lambda d: d.provider or None,
|
||||
"langfuse.observation.id": lambda d: d.identity.call_id or None,
|
||||
"langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None,
|
||||
"langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None,
|
||||
}
|
||||
|
||||
# Sub-tables folded into their respective JSON blobs.
|
||||
_MODEL_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = {
|
||||
"temperature": lambda rp: rp.temperature,
|
||||
"top_p": lambda rp: rp.top_p,
|
||||
"max_tokens": lambda rp: rp.max_tokens,
|
||||
"frequency_penalty": lambda rp: rp.frequency_penalty,
|
||||
"presence_penalty": lambda rp: rp.presence_penalty,
|
||||
"seed": lambda rp: rp.seed,
|
||||
}
|
||||
_USAGE_FIELDS: dict[str, Callable[[LLMUsage], AttrValue | None]] = {
|
||||
"input": lambda u: u.input_tokens,
|
||||
"output": lambda u: u.output_tokens,
|
||||
"total": lambda u: u.total_tokens,
|
||||
}
|
||||
|
||||
# JSON-payload attributes: each builder returns the serialized blob or None.
|
||||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"langfuse.observation.model.parameters": lambda d: json_if(
|
||||
collect(LangfuseMapper._MODEL_PARAMS, d.request_params)
|
||||
),
|
||||
"langfuse.observation.input": lambda d: serialize_messages(d.messages_in),
|
||||
"langfuse.observation.output": lambda d: serialize_messages(output_messages(d)),
|
||||
"langfuse.observation.usage_details": lambda d: json_if(
|
||||
collect(LangfuseMapper._USAGE_FIELDS, d.usage)
|
||||
),
|
||||
"langfuse.observation.cost_details": lambda d: (
|
||||
json.dumps({"total": d.response_cost})
|
||||
if d.response_cost is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
return {
|
||||
**collect(cls._LLM_CALL_ATTRS, data),
|
||||
**collect(cls._BLOB_ATTRS, data),
|
||||
}
|
||||
64
litellm/integrations/otel/mappers/langtrace.py
Normal file
64
litellm/integrations/otel/mappers/langtrace.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""Langtrace attribute mapper.
|
||||
|
||||
Produces Langtrace's attribute vocabulary so a span can be ingested by a
|
||||
Langtrace backend. Compose it alongside other mappers like any other
|
||||
vocabulary.
|
||||
|
||||
Scalar attributes are declared as a flat ``key -> extractor`` table (one lambda
|
||||
per mapping operation); the prompt/completion blobs are serialized as a tail.
|
||||
"""
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import (
|
||||
collect,
|
||||
json_or_none,
|
||||
output_messages,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
|
||||
|
||||
class LangtraceMapper:
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"gen_ai.operation.name": lambda d: "chat",
|
||||
"langtrace.service.name": lambda d: d.provider or None,
|
||||
"llm.model": lambda d: d.request_model or None,
|
||||
"gen_ai.response.model": lambda d: d.response_model or None,
|
||||
"gen_ai.response_id": lambda d: d.response_id or None,
|
||||
"gen_ai.system_fingerprint": lambda d: d.system_fingerprint or None,
|
||||
"llm.temperature": lambda d: d.request_params.temperature,
|
||||
"llm.top_p": lambda d: d.request_params.top_p,
|
||||
"llm.top_k": lambda d: d.request_params.top_k,
|
||||
"llm.max_tokens": lambda d: d.request_params.max_tokens,
|
||||
"llm.frequency_penalty": lambda d: d.request_params.frequency_penalty,
|
||||
"llm.presence_penalty": lambda d: d.request_params.presence_penalty,
|
||||
"llm.stream": lambda d: d.is_streaming,
|
||||
"llm.token.counts.prompt": lambda d: d.usage.input_tokens,
|
||||
"llm.token.counts.completion": lambda d: d.usage.output_tokens,
|
||||
"llm.token.counts.total": lambda d: d.usage.total_tokens,
|
||||
}
|
||||
|
||||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"llm.prompts": lambda d: (
|
||||
json_or_none(list(d.messages_in)) if d.messages_in else None
|
||||
),
|
||||
"llm.completions": lambda d: (
|
||||
json_or_none(output_messages(d)) if d.choices_out else None
|
||||
),
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
return {
|
||||
**collect(cls._LLM_CALL_ATTRS, data),
|
||||
**collect(cls._BLOB_ATTRS, data),
|
||||
}
|
||||
97
litellm/integrations/otel/mappers/legacy.py
Normal file
97
litellm/integrations/otel/mappers/legacy.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""Mapper for the older semantic-convention attribute vocabulary.
|
||||
|
||||
Emits attributes under the semconv-ai / Traceloop key names (e.g.
|
||||
``gen_ai.system``, ``gen_ai.usage.prompt_tokens``, ``llm.is_streaming``) plus a
|
||||
few bare, unprefixed service keys (``service``, ``call_type``, ``error``), for
|
||||
backends that consume those names.
|
||||
|
||||
Like ``GenAIMapper``, each span kind declares its schema as a flat
|
||||
``attribute key -> extractor`` table: one lambda per mapping operation.
|
||||
"""
|
||||
|
||||
from typing import Callable, Final
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import collect, drop_none
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
LLMCallSpanData,
|
||||
ServiceSpanData,
|
||||
ToolDefinition,
|
||||
)
|
||||
|
||||
# Attribute keys in the semconv-ai / Traceloop vocabulary.
|
||||
_LEGACY_SYSTEM: Final = "gen_ai.system"
|
||||
_LEGACY_PROMPT_TOKENS: Final = "gen_ai.usage.prompt_tokens"
|
||||
_LEGACY_COMPLETION_TOKENS: Final = "gen_ai.usage.completion_tokens"
|
||||
_LEGACY_TOTAL_TOKENS: Final = "gen_ai.usage.total_tokens"
|
||||
_LEGACY_IS_STREAMING: Final = "llm.is_streaming"
|
||||
_LEGACY_TOP_K: Final = "llm.top_k"
|
||||
_LEGACY_FREQUENCY_PENALTY: Final = "llm.frequency_penalty"
|
||||
_LEGACY_PRESENCE_PENALTY: Final = "llm.presence_penalty"
|
||||
_LEGACY_STOP_SEQUENCES: Final = "llm.chat.stop_sequences"
|
||||
_LEGACY_SERVICE: Final = "service"
|
||||
_LEGACY_CALL_TYPE: Final = "call_type"
|
||||
_LEGACY_ERROR: Final = "error"
|
||||
|
||||
|
||||
class LegacyMapper:
|
||||
"""Emits LLM-call and service attributes under the older key names."""
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
_LEGACY_SYSTEM: lambda d: d.provider or None,
|
||||
_LEGACY_PROMPT_TOKENS: lambda d: d.usage.input_tokens,
|
||||
_LEGACY_COMPLETION_TOKENS: lambda d: d.usage.output_tokens,
|
||||
_LEGACY_TOTAL_TOKENS: lambda d: d.usage.total_tokens,
|
||||
_LEGACY_IS_STREAMING: lambda d: d.is_streaming,
|
||||
_LEGACY_TOP_K: lambda d: d.request_params.top_k,
|
||||
_LEGACY_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty,
|
||||
_LEGACY_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty,
|
||||
_LEGACY_STOP_SEQUENCES: lambda d: (
|
||||
list(d.request_params.stop_sequences)
|
||||
if d.request_params.stop_sequences
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
|
||||
"name": lambda t: t.name,
|
||||
"description": lambda t: t.description or None,
|
||||
"parameters": lambda t: t.parameters_json or None,
|
||||
}
|
||||
|
||||
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {
|
||||
_LEGACY_SERVICE: lambda d: d.service_name,
|
||||
_LEGACY_CALL_TYPE: lambda d: d.call_type,
|
||||
_LEGACY_ERROR: lambda d: (
|
||||
d.error.message if d.error is not None and d.error.message else None
|
||||
),
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case ServiceSpanData():
|
||||
return self._service(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
attrs = collect(cls._LLM_CALL_ATTRS, data)
|
||||
attrs.update(
|
||||
drop_none(
|
||||
{
|
||||
f"llm.request.functions.{idx}.{suffix}": extract(tool)
|
||||
for idx, tool in enumerate(data.tools)
|
||||
for suffix, extract in cls._TOOL_ATTRS.items()
|
||||
}
|
||||
)
|
||||
)
|
||||
return attrs
|
||||
|
||||
@classmethod
|
||||
def _service(cls, data: ServiceSpanData) -> AttributeMap:
|
||||
attrs = collect(cls._SERVICE_ATTRS, data)
|
||||
attrs.update(dict(data.event_metadata))
|
||||
return attrs
|
||||
128
litellm/integrations/otel/mappers/openinference.py
Normal file
128
litellm/integrations/otel/mappers/openinference.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""OpenInference attribute mapper (Arize + Arize-Phoenix shared vocabulary).
|
||||
|
||||
Spec: https://github.com/Arize-ai/openinference/tree/main/spec — the standard
|
||||
both Arize and Phoenix consume. Composing this mapper after ``GenAIMapper``
|
||||
gives the same span both vocabularies, so a single trace lights up Arize +
|
||||
Phoenix + any other OpenInference-aware backend simultaneously.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Callable, Sequence
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import (
|
||||
collect,
|
||||
drop_none,
|
||||
json_if,
|
||||
message_content,
|
||||
output_messages,
|
||||
)
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
LLMCallSpanData,
|
||||
LLMRequestParams,
|
||||
ToolDefinition,
|
||||
)
|
||||
|
||||
|
||||
class OpenInferenceMapper:
|
||||
"""Emits OpenInference attributes for LLM_CALL spans.
|
||||
|
||||
Key families (per the OpenInference spec):
|
||||
- ``openinference.span.kind`` — discriminator (``"LLM"`` here)
|
||||
- ``llm.model_name`` / ``llm.provider`` / ``llm.invocation_parameters``
|
||||
- ``llm.input_messages.{i}.message.role`` / ``...content``
|
||||
- ``llm.output_messages.{i}.message.role`` / ``...content``
|
||||
- ``llm.token_count.prompt`` / ``...completion`` / ``...total``
|
||||
- ``input.value`` / ``output.value`` — JSON-serialized request / response
|
||||
"""
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"openinference.span.kind": lambda d: "LLM",
|
||||
"llm.model_name": lambda d: d.request_model or None,
|
||||
"llm.provider": lambda d: d.provider or None,
|
||||
"llm.token_count.prompt": lambda d: d.usage.input_tokens,
|
||||
"llm.token_count.completion": lambda d: d.usage.output_tokens,
|
||||
"llm.token_count.total": lambda d: d.usage.total_tokens,
|
||||
}
|
||||
|
||||
# Folded into the ``llm.invocation_parameters`` JSON blob.
|
||||
_INVOCATION_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = {
|
||||
"temperature": lambda rp: rp.temperature,
|
||||
"top_p": lambda rp: rp.top_p,
|
||||
"top_k": lambda rp: rp.top_k,
|
||||
"max_tokens": lambda rp: rp.max_tokens,
|
||||
"frequency_penalty": lambda rp: rp.frequency_penalty,
|
||||
"presence_penalty": lambda rp: rp.presence_penalty,
|
||||
"seed": lambda rp: rp.seed,
|
||||
}
|
||||
|
||||
# Per-tool extractors, keyed by the ``llm.tools.{idx}.*`` suffix.
|
||||
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
|
||||
"tool.name": lambda t: t.name,
|
||||
"tool.description": lambda t: t.description or None,
|
||||
"tool.json_schema": lambda t: t.parameters_json or None,
|
||||
}
|
||||
|
||||
# JSON-payload attributes: each builder returns the serialized blob or None.
|
||||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"llm.invocation_parameters": lambda d: json_if(
|
||||
collect(OpenInferenceMapper._INVOCATION_PARAMS, d.request_params)
|
||||
),
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
return {
|
||||
**collect(cls._LLM_CALL_ATTRS, data),
|
||||
**collect(cls._BLOB_ATTRS, data),
|
||||
**cls._messages("llm.input_messages", "input.value", data.messages_in),
|
||||
**cls._messages(
|
||||
"llm.output_messages", "output.value", output_messages(data)
|
||||
),
|
||||
**cls._tools(data),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _messages(
|
||||
prefix: str, value_key: str, messages: Sequence[object]
|
||||
) -> AttributeMap:
|
||||
"""Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob."""
|
||||
parsed = [
|
||||
(m.get("role") if isinstance(m, dict) else None, message_content(m))
|
||||
for m in messages
|
||||
]
|
||||
attrs = drop_none(
|
||||
{
|
||||
key: value
|
||||
for idx, (role, content) in enumerate(parsed)
|
||||
for key, value in (
|
||||
(
|
||||
f"{prefix}.{idx}.message.role",
|
||||
role if isinstance(role, str) else None,
|
||||
),
|
||||
(f"{prefix}.{idx}.message.content", content),
|
||||
)
|
||||
}
|
||||
)
|
||||
if parsed:
|
||||
attrs[value_key] = json.dumps(
|
||||
[{"role": role, "content": content} for role, content in parsed]
|
||||
)
|
||||
return attrs
|
||||
|
||||
@classmethod
|
||||
def _tools(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
return drop_none(
|
||||
{
|
||||
f"llm.tools.{idx}.{suffix}": extract(tool)
|
||||
for idx, tool in enumerate(data.tools)
|
||||
for suffix, extract in cls._TOOL_ATTRS.items()
|
||||
}
|
||||
)
|
||||
76
litellm/integrations/otel/mappers/utils.py
Normal file
76
litellm/integrations/otel/mappers/utils.py
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
"""Shared helpers for the attribute mappers.
|
||||
|
||||
Small, mapper-agnostic utilities — JSON serialization, message extraction, and
|
||||
extractor-table application — pulled out of the individual mapper modules so
|
||||
they live in one place.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Callable, Mapping, Sequence
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
|
||||
|
||||
def drop_none(values: Mapping[str, AttrValue | None]) -> AttributeMap:
|
||||
"""Return ``values`` with ``None``-valued entries removed."""
|
||||
return {k: v for k, v in values.items() if v is not None}
|
||||
|
||||
|
||||
def collect(table: Mapping[str, Callable], source: object) -> AttributeMap:
|
||||
"""Apply an extractor table to ``source``, dropping ``None`` results."""
|
||||
return drop_none({key: extract(source) for key, extract in table.items()})
|
||||
|
||||
|
||||
def json_if(payload: Mapping[str, object]) -> str | None:
|
||||
"""JSON-serialize ``payload`` only when it's non-empty; else ``None``."""
|
||||
return json.dumps(payload) if payload else None
|
||||
|
||||
|
||||
def json_or_none(value: object) -> str | None:
|
||||
"""JSON-serialize ``value`` (falling back to ``str``); ``None`` on failure."""
|
||||
try:
|
||||
return json.dumps(value, default=str)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def stringify_message(message: object) -> str | None:
|
||||
"""JSON-serialize a chat message dict; ``None`` if not a dict or on failure."""
|
||||
if not isinstance(message, dict):
|
||||
return None
|
||||
try:
|
||||
return json.dumps(message, default=str)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def serialize_messages(messages: Sequence[object]) -> str | None:
|
||||
"""Round-trip a sequence of message dicts through ``stringify_message``."""
|
||||
serialized = [
|
||||
json.loads(s) for s in (stringify_message(m) for m in messages) if s is not None
|
||||
]
|
||||
return json.dumps(serialized) if serialized else None
|
||||
|
||||
|
||||
def message_content(message: object) -> str | None:
|
||||
"""Extract the textual ``content`` from a chat message dict."""
|
||||
if not isinstance(message, dict):
|
||||
return None
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
# multimodal: concatenate text parts only
|
||||
parts = [
|
||||
part.get("text", "")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "text"
|
||||
]
|
||||
return "".join(p for p in parts if isinstance(p, str)) or None
|
||||
return None
|
||||
|
||||
|
||||
def output_messages(data: LLMCallSpanData) -> list:
|
||||
"""The ``message`` payload of each response choice."""
|
||||
return [c.get("message") for c in data.choices_out if isinstance(c, dict)]
|
||||
48
litellm/integrations/otel/mappers/weave.py
Normal file
48
litellm/integrations/otel/mappers/weave.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
"""Weave (W&B) attribute mapper.
|
||||
|
||||
Weave consumes OpenInference + a small set of Weave-specific keys (display
|
||||
name, thread id, output value). This mapper layers the latter on top of
|
||||
OpenInference's vocabulary — compose ``["genai", "openinference", "weave"]``
|
||||
to feed a Weave backend.
|
||||
"""
|
||||
|
||||
from typing import Callable
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
|
||||
from litellm.integrations.otel.mappers.utils import collect, json_or_none
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
|
||||
|
||||
class WeaveMapper:
|
||||
"""Maps ``LLMCallSpanData`` to Weave's vendor attributes."""
|
||||
|
||||
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
# ``display_name`` has the form ``"{operation} {model}"``. The span
|
||||
# name already covers that, but Weave reads this attribute too.
|
||||
"weave.display_name": lambda d: (
|
||||
f"{d.operation.value} {d.request_model}" if d.request_model else None
|
||||
),
|
||||
"weave.call_id": lambda d: d.identity.call_id or None,
|
||||
}
|
||||
|
||||
# JSON-payload attributes: each builder returns the serialized blob or None.
|
||||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
# Weave treats the response choices as the "output" payload.
|
||||
"weave.output": lambda d: (
|
||||
json_or_none(list(d.choices_out)) if d.choices_out else None
|
||||
),
|
||||
}
|
||||
|
||||
def map(self, data: SpanData) -> AttributeMap:
|
||||
match data:
|
||||
case LLMCallSpanData():
|
||||
return self._llm_call(data)
|
||||
case _:
|
||||
return {}
|
||||
|
||||
@classmethod
|
||||
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
|
||||
return {
|
||||
**collect(cls._LLM_CALL_ATTRS, data),
|
||||
**collect(cls._BLOB_ATTRS, data),
|
||||
}
|
||||
0
litellm/integrations/otel/model/__init__.py
Normal file
0
litellm/integrations/otel/model/__init__.py
Normal file
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue