scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'upstream/litellm_internal_staging' into atr-guardrail

Browse source 
# Conflicts:
#	litellm/types/guardrails.py
This commit is contained in:
eeee2345 2026-06-05 06:31:42 +08:00
commit aa52a34ce0
1952 changed files with 126902 additions and 36208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

232
.circleci/config.yml
View file

@ -182,7 +182,14 @@ jobs:
- run:
name: Run Windows-specific test
command: |
uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
uv run --no-sync python -m pytest tests/windows_tests/ -v
- run:
name: Guard against MAX_PATH-busting packaged wheel paths
environment:
UV_HTTP_TIMEOUT: "300"
command: |
uv build --wheel --out-dir dist
uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py
local_testing_part1:
docker:
@ -228,7 +235,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--split-by=timings \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov-report=xml \
@ -242,8 +249,15 @@ jobs:
- run:
name: Rename the coverage files
command: |
mv coverage.xml local_testing_part1_coverage.xml
mv .coverage local_testing_part1_coverage
# When CI reruns only the failed tests, a parallel node can receive
# zero tests and pytest never writes coverage. Emit empty placeholders
# so persist_to_workspace and the downstream coverage combine stay green.
if [ -f coverage.xml ]; then
mv coverage.xml local_testing_part1_coverage.xml
mv .coverage local_testing_part1_coverage
else
touch local_testing_part1_coverage.xml local_testing_part1_coverage
fi
# Store test results
- store_test_results:
@ -293,7 +307,7 @@ jobs:
echo "$TEST_FILES" | circleci tests run \
--split-by=timings \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov-report=xml \
@ -307,8 +321,15 @@ jobs:
- run:
name: Rename the coverage files
command: |
mv coverage.xml local_testing_part2_coverage.xml
mv .coverage local_testing_part2_coverage
# When CI reruns only the failed tests, a parallel node can receive
# zero tests and pytest never writes coverage. Emit empty placeholders
# so persist_to_workspace and the downstream coverage combine stay green.
if [ -f coverage.xml ]; then
mv coverage.xml local_testing_part2_coverage.xml
mv .coverage local_testing_part2_coverage
else
touch local_testing_part2_coverage.xml local_testing_part2_coverage
fi
# Store test results
- store_test_results:
@ -356,7 +377,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -409,7 +430,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -431,6 +452,120 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- setup_google_dns
- install_uv
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -457,12 +592,17 @@ jobs:
- run:
name: Run tests
command: |
# On a "rerun failed tests" build a parallel node can receive no
# tests, so the test command never creates test-results. Pre-create it
# so store_test_results doesn't fail the node on a missing path.
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--split-by=timings \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
-k 'router' \
-n 4 \
@ -504,7 +644,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -547,7 +687,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -589,7 +729,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -625,7 +765,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -668,7 +808,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -710,7 +850,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -754,7 +894,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -805,7 +945,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -836,7 +976,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -878,7 +1018,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -922,7 +1062,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
@ -952,7 +1092,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -994,7 +1134,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -1037,7 +1177,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -1080,7 +1220,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1112,7 +1252,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
-n 4 \
@ -1155,7 +1295,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -1206,7 +1346,7 @@ jobs:
tests/local_testing/test_router_utils.py)
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--junitxml=test-results/junit.xml \
@ -1456,7 +1596,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
@ -1539,7 +1679,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -v -x \
--junitxml=test-results/junit.xml \
-n 4 \
@ -1622,7 +1762,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-s -vv \
--junitxml=test-results/junit.xml \
--durations=5"
@ -1698,7 +1838,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v \
--junitxml=test-results/junit.xml \
--durations=5"
@ -1748,7 +1888,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
@ -1824,7 +1964,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
@ -1922,7 +2062,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
@ -1985,7 +2125,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit.xml \
--durations=5"
@ -2065,7 +2205,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--junitxml=test-results/junit-2.xml \
--durations=5"
@ -2209,7 +2349,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit.xml \
--durations=5"
@ -2275,7 +2415,7 @@ jobs:
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--junitxml=test-results/junit.xml \
--durations=5"
@ -2400,6 +2540,11 @@ jobs:
environment:
DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
CI: "true"
# Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can
# assert the redirect. Set at job level so both the proxy boot step and the
# Playwright step (whose skip guard reads this) see the same value. Safe for
# the rest of the suite: nothing else performs a logout.
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- setup_google_dns
@ -2476,7 +2621,8 @@ jobs:
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
SERVER_ROOT_PATH: ""
PROXY_LOGOUT_URL: ""
# PROXY_LOGOUT_URL is inherited from the job-level environment so the
# proxy and proxyLogoutUrl.spec.ts agree on the logout target.
# LITELLM_LICENSE is forwarded from the project env so premium-gated
# UI flows can be exercised. license.spec.ts asserts the resulting
# JWT carries premium_user=true; if it ever stops being passed, that
@ -2611,6 +2757,12 @@ workflows:
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:

3
.git-blame-ignore-revs
View file

@ -8,3 +8,6 @@
# Update pydantic code to fix warnings (GH-3600)
876840e9957bc7e9f7d6a2b58c4d7c53dad16481
# style(ui): run prettier --write across the dashboard (#29622)
7edf3a9cb55548b143df1692f4ed7c4681d7fcf7

4
.github/pull_request_template.md vendored
View file

@ -10,9 +10,9 @@
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] I have added meaningful tests
- [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
- [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
## Delays in PR merge?

40
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,6 +27,11 @@ on:
required: false
type: number
default: 10
dist:
description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
required: false
type: string
default: "loadscope"
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: true
@ -82,18 +87,31 @@ jobs:
MAX_FAILURES: ${{ inputs.max-failures }}
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DIST: ${{ inputs.dist }}
run: |
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
-n "${WORKERS}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist=loadscope \
--durations=20 \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
-n "${WORKERS}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
- name: Save coverage report
if: always()

190
.github/workflows/_test-unit-services-base.yml vendored
View file

@ -1,190 +0,0 @@
name: _Unit Test Services Base (Reusable)
on:
workflow_call:
inputs:
test-path:
description: "Pytest path(s) to run"
required: true
type: string
workers:
description: "Number of pytest-xdist workers (0 = no parallelism)"
required: false
type: number
default: 2
reruns:
description: "Number of reruns for flaky tests"
required: false
type: number
default: 2
timeout-minutes:
description: "Job timeout in minutes"
required: false
type: number
default: 20
max-failures:
description: "Stop after this many failures"
required: false
type: number
default: 10
enable-postgres:
description: "Start a local Postgres service container and run Prisma migrations"
required: false
type: boolean
default: false
dist:
description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
required: false
type: string
default: "loadscope"
artifact-name:
description: "Unique name for the coverage artifact (must be unique per run)"
required: false
type: string
default: "run"
permissions:
contents: read
# The postgres service container below is spawned per-job on localhost and
# destroyed with the job. Nothing outside the runner can reach it. The
# user/password/database here are not secrets — they're bootstrap values
# for a throwaway container — so we hardcode them instead of attaching
# every matrix shard to a GHA environment just to read three "secrets"
# (which also produces a "temporarily deployed to …" notification on the
# PR timeline per shard per push).
jobs:
run:
name: Run tests
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.timeout-minutes }}
services:
postgres:
image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
env:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: litellm
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-services-
- name: Install dependencies
run: |
uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Generate Prisma client
env:
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run Prisma migrations
if: ${{ inputs.enable-postgres }}
env:
DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test"
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
env:
TEST_PATH: ${{ inputs.test-path }}
MAX_FAILURES: ${{ inputs.max-failures }}
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DIST: ${{ inputs.dist }}
DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
uv run --no-sync pytest ${TEST_PATH:?} \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
-n "${WORKERS}" \
--reruns "${RERUNS}" \
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi
- name: Save coverage report
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage.xml
retention-days: 1
upload-coverage:
name: Upload coverage to Codecov
needs: run
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
pull-requests: write
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Download coverage report
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
with:
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
path: coverage-reports
merge-multiple: true
- name: Upload to Codecov
uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
with:
use_oidc: true
directory: coverage-reports
root_dir: ${{ github.workspace }}
flags: ${{ inputs.artifact-name }}
fail_ci_if_error: false

25
.github/workflows/create-release-branch.yml vendored
View file

@ -63,3 +63,28 @@ jobs:
sha: commitHash,
});
core.info(`Created branch ${branchName} at ${commitHash}`);
- name: Create stable line branch
env:
TAG: ${{ inputs.tag }}
COMMIT_HASH: ${{ inputs.commit_hash }}
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
with:
script: |
const tag = process.env.TAG;
const commitHash = process.env.COMMIT_HASH;
const match = tag.match(/^v?(\d+)\.(\d+)\.0$/);
if (!match) {
core.info(`Tag ${tag} is not the X.Y.0 stable opener; skipping stable line branch`);
return;
}
const lineBranch = `stable/${match[1]}.${match[2]}.x`;
await github.rest.git.createRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: `refs/heads/${lineBranch}`,
sha: commitHash,
});
core.info(`Created branch ${lineBranch} at ${commitHash}`);

76
.github/workflows/test-litellm-ui-build.yml vendored
View file

@ -36,3 +36,79 @@ jobs:
- name: Build
run: npm run build
frontend-lint:
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
run:
working-directory: ui/litellm-dashboard
steps:
- name: Checkout repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
fetch-depth: 0
persist-credentials: false
- name: Collect changed files
id: changed
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
: > "$RUNNER_TEMP/prettier_files.txt"
: > "$RUNNER_TEMP/eslint_files.txt"
while IFS= read -r f; do
[ -f "$f" ] || continue
case "$f" in
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
esac
done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .)
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
echo "has_files=true" >> "$GITHUB_OUTPUT"
else
echo "has_files=false" >> "$GITHUB_OUTPUT"
echo "No lintable UI files changed in this PR; nothing to check."
fi
- name: Setup Node.js
if: steps.changed.outputs.has_files == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
with:
node-version: "20"
cache: "npm"
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changed.outputs.has_files == 'true'
run: npm ci
- name: Lint changed files (prettier + eslint)
if: steps.changed.outputs.has_files == 'true'
run: |
prettier_files=()
eslint_files=()
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
status=0
if [ ${#prettier_files[@]} -gt 0 ]; then
echo "::group::Prettier (${#prettier_files[@]} files)"
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
echo "::endgroup::"
fi
if [ ${#eslint_files[@]} -gt 0 ]; then
echo "::group::ESLint (${#eslint_files[@]} files)"
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
echo "::endgroup::"
fi
exit $status
- name: Check lint budgets
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
run: |
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json

25
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -1,9 +1,10 @@
name: "Unit Tests: Proxy DB Operations"
# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
on:
push:
branches: [main, "litellm_**"]
pull_request:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
@ -30,9 +31,6 @@ concurrency:
# xdist balances its 188 parametrized cases across workers instead of
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
# * test_db_schema_migration.py is isolated because one test in it
# (test_aaaasschema_migration_check) takes ~170s — by itself it
# determines the shard's wall-clock floor.
jobs:
# Fast guard — fails the workflow if a test_*.py file under
# tests/proxy_unit_tests/ is not referenced by any matrix entry below.
@ -166,18 +164,6 @@ jobs:
dist: loadscope
timeout: 15
# ---- db-and-spend: isolate the 170s schema-migration test ----
# test_db_schema_migration.py has exactly one test, and that test
# is mostly waiting on `prisma migrate deploy` / `prisma migrate
# diff` subprocesses (~170s). It does no CPU-bound Python work
# inside the test. Running with workers=0 (serial, no xdist)
# skips the 4-worker cold-start cost we'd otherwise pay for a
# single test, saving ~4 minutes of wall-clock.
- test-group: schema-migration
test-path: "tests/proxy_unit_tests/test_db_schema_migration.py"
workers: 0
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: >-
tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
@ -232,12 +218,11 @@ jobs:
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-services-base.yml
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}
enable-postgres: true
dist: ${{ matrix.dist }}
artifact-name: proxy-db-${{ matrix.test-group }}

3
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -33,13 +33,16 @@ jobs:
tests/test_litellm/proxy/image_endpoints
tests/test_litellm/proxy/vector_store_endpoints
tests/test_litellm/proxy/agent_endpoints
tests/test_litellm/proxy/a2a
tests/test_litellm/proxy/discovery_endpoints
tests/test_litellm/proxy/health_endpoints
tests/test_litellm/proxy/shutdown
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/utils
workers: 2
reruns: 2
artifact-name: proxy-endpoints

34
.github/workflows/test-unit-proxy-mgmt-behavior.yml vendored
View file

@ -1,34 +0,0 @@
name: "Unit Tests: Proxy Management-Endpoint Behavior Pinning"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
proxy-mgmt-behavior:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
test-path: tests/proxy_behavior
# workers=0 (no xdist): the world seed is a single shared Postgres
# state — two xdist workers both call seed_world() and race on the
# ``behavior-pin-budget`` row, producing UniqueViolation + cascading
# missing-membership FK failures. The whole suite is ~7s sequentially,
# so the cost of disabling parallelism here is negligible.
workers: 0
reruns: 0
enable-postgres: true
artifact-name: proxy-mgmt-behavior
timeout-minutes: 15

28
.github/workflows/test-unit-security.yml vendored
View file

@ -1,28 +0,0 @@
name: "Unit Tests: Security"
# Kept push-only (was previously required by DATABASE_URL secret scoping;
# now the postgres credentials are ephemeral localhost values but the
# push-trigger stays to match the proxy-db workflow cadence).
on:
push:
branches: [main, "litellm_**"]
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
security:
uses: ./.github/workflows/_test-unit-services-base.yml
with:
test-path: "tests/proxy_security_tests/"
workers: 1
reruns: 2
timeout-minutes: 20
enable-postgres: true
artifact-name: security

5
.gitignore vendored
View file

@ -28,6 +28,8 @@ litellm/tests/config_*.yaml
litellm/tests/langfuse.log
langfuse.log
.langfuse.log
.pin_list.txt
.cov_new.xml
litellm/tests/test_custom_logger.py
litellm/tests/langfuse.log
litellm/tests/dynamo*.log
@ -120,4 +122,5 @@ crash.log
crash.*.log
# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
# and should be committed.
.vscode
.vscode
.pin_list.txt

307
AGENTS.md
View file

@ -1,306 +1 @@
# INSTRUCTIONS FOR LITELLM
This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
## Confidentiality: Customer and Company Names in Code
The codebase is public. Before writing **any** third-party organization name into this repository — in source code, file or directory names, docstrings, comments, tests, fixtures, mock payloads, error messages, log lines, commit messages, or PR descriptions — pause and check:
**Already in the codebase** (OpenAI, Anthropic, Google, Azure, Bedrock, Fireworks, and other established LLM providers / integrations) — fine to use. Quick check: `git grep -i "<name>"` — if it returns hits in real code (not just your current diff), the name is established.
**Anything else** — customers, prospects, partners, new vendor integrations, observability tools, infra vendors, or any organization name that does not already appear in the repo. STOP and surface it to the user. Ask for explicit consent before writing the name into any file, commit message, or PR description. Do not write it speculatively and clean up later. Do not substitute a placeholder and proceed. Do not assume it is safe because it "looks like" a public company. The user must approve first.
**What to do instead of a customer-specific reference:**
- If you find yourself reaching for a customer name — real or fake — step back. The code shouldn't be customer-specific in the first place. Generalize the feature, or capture the customer motivation in internal docs (Notion / Linear / the internal staging PR description), never in the repo.
- Frame changes by the capability they add, not the customer who asked for it ("add per-team Bedrock guardrail routing", not "add routing for $CUSTOMER").
- Standard "fake value" markers (`example.com`, `localhost`, `127.0.0.1`, `test@example.com`) and abstract identifiers (`team_a`, `user_1`, `tenant_x`) are fine — those are not customer stand-ins.
## OVERVIEW
LiteLLM is a unified interface for 100+ LLMs that:
- Translates inputs to provider-specific completion, embedding, and image generation endpoints
- Provides consistent OpenAI-format output across all providers
- Includes retry/fallback logic across multiple deployments (Router)
- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
- Supports advanced features like function calling, streaming, caching, and observability
## REPOSITORY STRUCTURE
### Core Components
- `litellm/` - Main library code
- `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
- `proxy/` - Proxy server implementation (LLM Gateway)
- `router_utils/` - Load balancing and fallback logic
- `types/` - Type definitions and schemas
- `integrations/` - Third-party integrations (observability, caching, etc.)
### Key Directories
- `tests/` - Comprehensive test suites
- `ui/litellm-dashboard/` - Admin dashboard UI
- `enterprise/` - Enterprise-specific features
Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai).
## DEVELOPMENT GUIDELINES
### MAKING CODE CHANGES
1. **Provider Implementations**: When adding/modifying LLM providers:
- Follow existing patterns in `litellm/llms/{provider}/`
- Implement proper transformation classes that inherit from `BaseConfig`
- Support both sync and async operations
- Handle streaming responses appropriately
- Include proper error handling with provider-specific exceptions
2. **Type Safety**:
- Use proper type hints throughout
- Update type definitions in `litellm/types/`
- Ensure compatibility with both Pydantic v1 and v2
3. **Testing**:
- Add tests in appropriate `tests/` subdirectories
- Include both unit tests and integration tests
- Test provider-specific functionality thoroughly
- Consider adding load tests for performance-critical changes
### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
1. **Always use `antd` for new UI components — Tremor is DEPRECATED**
- We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file.
- Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
- The only exception is the Tremor Table component and its required Tremor Table sub components.
2. **Use Common Components as much as possible**:
- These are usually defined in the `common_components` directory
- Use these components as much as possible and avoid building new components unless needed
3. **Testing**:
- The codebase uses **Vitest** and **React Testing Library**
- **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
- **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
- **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
- **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
- **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
- **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
- **Structure tests properly**:
- First test should verify the component renders successfully
- Subsequent tests should focus on functionality and user interactions
- Use `waitFor` for async operations that aren't already awaited
- **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
### IMPORTANT PATTERNS
1. **Function/Tool Calling**:
- LiteLLM standardizes tool calling across providers
- OpenAI format is the standard, with transformations for other providers
- See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
2. **Streaming**:
- All providers should support streaming where possible
- Use consistent chunk formatting across providers
- Handle both sync and async streaming
3. **Error Handling**:
- Use provider-specific exception classes
- Maintain consistent error formats across providers
- Include proper retry logic and fallback mechanisms
4. **Configuration**:
- Support both environment variables and programmatic configuration
- Use `BaseConfig` classes for provider configurations
- Allow dynamic parameter passing
## PROXY SERVER (LLM GATEWAY)
The proxy server is a critical component that provides:
- Authentication and authorization
- Rate limiting and budget management
- Load balancing across multiple models/deployments
- Observability and logging
- Admin dashboard UI
- Enterprise features
Key files:
- `litellm/proxy/proxy_server.py` - Main server implementation
- `litellm/proxy/auth/` - Authentication logic
- `litellm/proxy/management_endpoints/` - Admin API endpoints
**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
LiteLLM supports MCP for agent workflows:
- MCP server integration for tool calling
- Transformation between OpenAI and MCP tool formats
- Support for external MCP servers (Zapier, Jira, Linear, etc.)
- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
## RUNNING SCRIPTS
Use `uv run python script.py` to run Python scripts in the project environment (for non-test files).
## GITHUB TEMPLATES
When opening issues or pull requests, follow these templates:
### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
- Describe what happened vs. expected behavior
- Include relevant log output
- Specify LiteLLM version
- Indicate if you're part of an ML Ops team (helps with prioritization)
### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
- Clearly describe the feature
- Explain motivation and use case with concrete examples
### Pull Requests (`.github/pull_request_template.md`)
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## TESTING CONSIDERATIONS
1. **Provider Tests**: Test against real provider APIs when possible
2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
3. **Performance Tests**: Load testing for high-throughput scenarios
4. **Integration Tests**: End-to-end workflows including tool calling
## DOCUMENTATION
- Keep documentation in sync with code changes
- Update provider documentation when adding new providers
- Include code examples for new features
- Update changelog and release notes
## SECURITY CONSIDERATIONS
- Handle API keys securely
- Validate all inputs, especially for proxy endpoints
- Consider rate limiting and abuse prevention
- Follow security best practices for authentication
## ENTERPRISE FEATURES
- Some features are enterprise-only
- Check `enterprise/` directory for enterprise-specific code
- Maintain compatibility between open-source and enterprise versions
## COMMON PITFALLS TO AVOID
1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
2. **Provider Specifics**: Each provider has unique quirks - handle them properly
3. **Rate Limits**: Respect provider rate limits in tests
4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
5. **Dependencies**: Keep dependencies minimal and well-justified
6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
**Example of BAD** (hardcoded model checks):
```python
@staticmethod
def _is_effort_supported_model(model: str) -> bool:
"""Check if the model supports the output_config.effort parameter..."""
model_lower = model.lower()
if AnthropicConfig._is_claude_4_6_model(model):
return True
return any(
v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
)
```
**Example of GOOD** (config-driven or helper that reads from config):
```python
if (
"claude-3-7-sonnet" in model
or AnthropicConfig._is_claude_4_6_model(model)
or supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
):
...
```
Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
## HELPFUL RESOURCES
- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs))
- Provider-specific docs: https://docs.litellm.ai/docs/providers/
- Admin UI for testing proxy features
## WHEN IN DOUBT
- Follow existing patterns in the codebase
- Check similar provider implementations
- Ensure comprehensive test coverage
- Update documentation appropriately
- Consider backward compatibility impact
## Cursor Cloud specific instructions
### Environment
- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
- Python 3.12, Node 22 are pre-installed.
- The project virtual environment lives under `.venv/`.
### Running the proxy server
Create a minimal config file and start the proxy:
```yaml
# config.yaml
model_list:
- model_name: fake-openai-endpoint
litellm_params:
model: openai/fake-model
api_key: fake-key
api_base: https://fake-api.example.com
general_settings:
master_key: sk-1234
litellm_settings:
drop_params: True
telemetry: False
```
```bash
uv run litellm --config config.yaml --port 4000
```
The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
### Running tests
See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow.
- The `--timeout` pytest flag is NOT available; don't pass it.
- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4`
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry.
### Lint
```bash
cd litellm && uv run ruff check .
```
Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
### UI Dashboard development
- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`
Read @CLAUDE.md for coding guidelines

215
CLAUDE.md
View file

@ -1,194 +1,75 @@
# CLAUDE.md
Do not write comments unless they are absolutely necessary to explain some very complex business logic. Please clean up if there are comments that are not absolutely necessary. Do not remove comments that are unrelated to the addition of the code of this PR
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
Explanation: code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive to the reader, while being both easy to maintain and high performance
## Confidentiality: Customer and Company Names in Code
Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in:
- correct
- secure
- performant
- readable
- easy to maintain/change
- modern
The codebase is public. Before writing **any** third-party organization name into this repository — in source code, file or directory names, docstrings, comments, tests, fixtures, mock payloads, error messages, log lines, commit messages, or PR descriptions — pause and check:
In that order of importance
**Already in the codebase** (OpenAI, Anthropic, Google, Azure, Bedrock, Fireworks, and other established LLM providers / integrations) — fine to use. Quick check: `git grep -i "<name>"` — if it returns hits in real code (not just your current diff), the name is established.
When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate
**Anything else** — customers, prospects, partners, new vendor integrations, observability tools, infra vendors, or any organization name that does not already appear in the repo. STOP and surface it to the user. Ask for explicit consent before writing the name into any file, commit message, or PR description. Do not write it speculatively and clean up later. Do not substitute a placeholder and proceed. Do not assume it is safe because it "looks like" a public company. The user must approve first.
Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
**What to do instead of a customer-specific reference:**
- If you find yourself reaching for a customer name — real or fake — step back. The code shouldn't be customer-specific in the first place. Generalize the feature, or capture the customer motivation in internal docs (Notion / Linear / the internal staging PR description), never in the repo.
- Frame changes by the capability they add, not the customer who asked for it ("add per-team Bedrock guardrail routing", not "add routing for $CUSTOMER").
- Standard "fake value" markers (`example.com`, `localhost`, `127.0.0.1`, `test@example.com`) and abstract identifiers (`team_a`, `user_1`, `tenant_x`) are fine — those are not customer stand-ins.
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
## Documentation
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead.
Always use @.github/pull_request_template.md as a guide for your PR body
## Development Commands
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install the full local test environment and generate the Prisma client
If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
- don't use emojis
- don't use "—". Instead, reach for ";", ".", etc.
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- don't add a trailing "." at the end of paragraphs (just like this file)
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
Run tests, format your code, and lint your code before each commit
### Single Test Files
- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
Ask to commit and push your work when you're done (or if you're confident that your code is good and works, just do it)
### Running Scripts
- `uv run python script.py` - Run Python scripts (use for non-test files)
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
When working on a PR, keep the PR description in sync with new commits being made
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in
## Architecture Overview
Do not put names of customers or customer company names in code, PRs, and issues. The codebase is public
LiteLLM is a unified interface for 100+ LLM providers with two main components:
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
## Think Before Coding
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
**Don't assume. Don't hide confusion. Surface tradeoffs**
## Key Patterns
Before implementing:
- State your assumptions explicitly. If uncertain, ask
- If multiple interpretations exist, present them. Don't pick silently
- If a simpler approach exists, say so. Push back when warranted
- If something is unclear, stop. Name what's confusing. Ask
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
## Simplicity First
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
**Minimum code that solves the problem. Nothing speculative**
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
- No features beyond what was asked
- No abstractions for single-use code
- No "flexibility" or "configurability" that wasn't requested
- No error handling for impossible scenarios
- If you write 200 lines and it could be 50, rewrite it
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
### UI / Backend Consistency
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
### UI Component Library
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
### MCP OAuth / OpenAPI Transport Mapping
- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
### MCP Credential Storage
- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
### Browser Storage Safety (UI)
- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Proxy database access
- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
### Setup Wizard (`litellm/setup_wizard.py`)
- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features
### CI Supply-Chain Safety
- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install.
- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you.
- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest.
- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_<tool>` or `- wait_for_service`.
- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it.
- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions.
### HTTP Client Cache Safety
- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
### Troubleshooting: DB schema out of sync after proxy restart
`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
**Fix options:**
1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify

109
GEMINI.md
View file

@ -1,108 +1 @@
# GEMINI.md
This file provides guidance to Gemini when working with code in this repository.
## Development Commands
### Installation
- `make install-dev` - Install core development dependencies
- `make install-proxy-dev` - Install proxy development dependencies with full feature set
- `make install-test-deps` - Install all test dependencies
### Testing
- `make test` - Run all tests
- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
- `make test-integration` - Run integration tests (excludes unit tests)
- `pytest tests/` - Direct pytest execution
### Code Quality
- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
- `make format` - Apply Black code formatting
- `make lint-ruff` - Run Ruff linting only
- `make lint-mypy` - Run MyPy type checking only
### Single Test Files
- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
### Running Scripts
- `uv run python script.py` - Run Python scripts (use for non-test files)
### GitHub Issue & PR Templates
When contributing to the project, use the appropriate templates:
**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
- Describe what happened vs. what you expected
- Include relevant log output
- Specify your LiteLLM version
**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
- Describe the feature clearly
- Explain the motivation and use case
**Pull Requests** (`.github/pull_request_template.md`):
- Add at least 1 test in `tests/litellm/`
- Ensure `make test-unit` passes
## Architecture Overview
LiteLLM is a unified interface for 100+ LLM providers with two main components:
### Core Library (`litellm/`)
- **Main entry point**: `litellm/main.py` - Contains core completion() function
- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
- **Type definitions**: `litellm/types/` - Pydantic models and type hints
- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
### Proxy Server (`litellm/proxy/`)
- **Main server**: `proxy_server.py` - FastAPI application
- **Authentication**: `auth/` - API key management, JWT, OAuth2
- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
- **Guardrails**: `guardrails/` - Safety and content filtering hooks
- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
## Key Patterns
### Provider Implementation
- Providers inherit from base classes in `litellm/llms/base.py`
- Each provider has transformation functions for input/output formatting
- Support both sync and async operations
- Handle streaming responses and function calling
### Error Handling
- Provider-specific exceptions mapped to OpenAI-compatible errors
- Fallback logic handled by Router system
- Comprehensive logging through `litellm/_logging.py`
### Configuration
- YAML config files for proxy server (see `proxy/example_config_yaml/`)
- Environment variables for API keys and settings
- Database schema managed via Prisma (`proxy/schema.prisma`)
## Development Notes
### Code Style
- Uses Black formatter, Ruff linter, MyPy type checker
- Pydantic v2 for data validation
- Async/await patterns throughout
- Type hints required for all public APIs
### Testing Strategy
- Unit tests in `tests/test_litellm/`
- Integration tests for each provider in `tests/llm_translation/`
- Proxy tests in `tests/proxy_unit_tests/`
- Load tests in `tests/load_tests/`
### Database Migrations
- Prisma handles schema migrations
- Migration files auto-generated with `prisma migrate dev`
- Always test migrations against both PostgreSQL and SQLite
### Enterprise Features
- Enterprise-specific code in `enterprise/` directory
- Optional features enabled via environment variables
- Separate licensing and authentication for enterprise features
Read @CLAUDE.md for coding guidelines

2
Makefile
View file

@ -146,7 +146,7 @@ test-unit-proxy-core: install-test-deps
$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
test-unit-proxy-misc: install-test-deps
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
test-unit-integrations: install-test-deps
$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20

2
README.md
View file

@ -37,7 +37,7 @@
</a>
</h4>
<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
<img alt="LiteLLM AI Gateway" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
---

2
cookbook/gollem_go_agent_framework/go.mod
View file

@ -1,5 +1,5 @@
module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
go 1.25.1
go 1.26.3
require github.com/fugue-labs/gollem v0.1.0

24
deploy/charts/litellm-helm/values.yaml
View file

@ -285,11 +285,31 @@ db:
deployStandalone: true
# Lifecycle hooks for the LiteLLM container
#
# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks
# the pod NotReady and blocks only until in-flight requests actually finish
# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always
# waiting the worst-case duration. The drain runs once (the preStop hook and
# the SIGTERM handler share it), so set terminationGracePeriodSeconds a few
# seconds above GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before
# SIGKILL.
#
# /health/drain is off by default; enable it with
# general_settings.enable_drain_endpoint: true. The kubelet calls preStop
# hooks without proxy credentials, so when the health port is reachable from
# other pods (the common case) also set
# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env
# var) and send the same value on the X-Drain-Token header from the hook.
# Calls missing/wrong the token get a 401 and have no side effect.
# Example:
# lifecycle:
# preStop:
# exec:
# command: ["/bin/sh", "-c", "sleep 10"]
# httpGet:
# path: /health/drain
# port: 4000
# httpHeaders:
# - name: X-Drain-Token
# value: <same value as drain_endpoint_token>
lifecycle: {}
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored

19
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/resend_email.py
View file

@ -19,12 +19,26 @@ RESEND_API_ENDPOINT = "https://api.resend.com/emails"
class ResendEmailLogger(BaseEmailLogger):
"""
Send emails using Resend's API.
Required env vars:
- RESEND_API_KEY
Optional env vars:
- RESEND_FROM_EMAIL: Override the default sender address. Must be on a
domain verified in your Resend account. When unset, falls back to the
`from_email` argument passed by the caller (which defaults to
`notifications@alerts.litellm.ai` and only works on LiteLLM Cloud).
"""
def __init__(self, internal_usage_cache=None, **kwargs):
super().__init__(internal_usage_cache=internal_usage_cache, **kwargs)
self.async_httpx_client = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
self.resend_api_key = os.getenv("RESEND_API_KEY")
self.resend_from_email = os.getenv("RESEND_FROM_EMAIL")
async def send_email(
self,
@ -33,13 +47,14 @@ class ResendEmailLogger(BaseEmailLogger):
subject: str,
html_body: str,
):
sender_email = self.resend_from_email or from_email
verbose_logger.debug(
f"Sending email from {from_email} to {to_email} with subject {subject}"
f"Sending email from {sender_email} to {to_email} with subject {subject}"
)
response = await self.async_httpx_client.post(
url=RESEND_API_ENDPOINT,
json={
"from": from_email,
"from": sender_email,
"to": to_email,
"subject": subject,
"html": html_body,

2
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -658,7 +658,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
if c.get("type") == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object["file"]
file_id = file_object_file_field.get("file_id")

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.41"
version = "0.1.42"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.41"
version = "0.1.42"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

1
gateway/routes/allowlist.py
View file

@ -106,6 +106,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
# Health & ops
"/health",
"/metrics",
"/watsonx"
)
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(

34
helm/litellm/templates/_helpers.tpl
View file

@ -56,16 +56,34 @@ app.kubernetes.io/component: ui
{{- end -}}
{{/*
Shared ServiceAccount name used by all three component Deployments. When
`serviceAccount.create` is true and `serviceAccount.name` is empty, default
to the chart fullname. When `create` is false, fall back to the provided
name or the namespace's `default` SA.
Per-component ServiceAccount name helpers.
Each component (gateway, backend, ui) has its own SA config under
.Values.serviceAccounts.<component>. When `create` is true and `name` is
empty the chart defaults to "<release>-litellm-<component>". When `create`
is false the chart uses the provided name, or the namespace `default` SA.
*/}}
{{- define "litellm.serviceAccountName" -}}
{{- if .Values.serviceAccount.create -}}
{{ default (include "litellm.fullname" .) .Values.serviceAccount.name }}
{{- define "litellm.gateway.serviceAccountName" -}}
{{- if .Values.serviceAccounts.gateway.create -}}
{{ default (include "litellm.gateway.fullname" .) .Values.serviceAccounts.gateway.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.name }}
{{ default "default" .Values.serviceAccounts.gateway.name }}
{{- end -}}
{{- end -}}
{{- define "litellm.backend.serviceAccountName" -}}
{{- if .Values.serviceAccounts.backend.create -}}
{{ default (include "litellm.backend.fullname" .) .Values.serviceAccounts.backend.name }}
{{- else -}}
{{ default "default" .Values.serviceAccounts.backend.name }}
{{- end -}}
{{- end -}}
{{- define "litellm.ui.serviceAccountName" -}}
{{- if .Values.serviceAccounts.ui.create -}}
{{ default (include "litellm.ui.fullname" .) .Values.serviceAccounts.ui.name }}
{{- else -}}
{{ default "default" .Values.serviceAccounts.ui.name }}
{{- end -}}
{{- end -}}

26
helm/litellm/templates/backend/deployment.yaml
View file

@ -12,14 +12,20 @@ spec:
{{- include "litellm.backend.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.backend.podAnnotations }}
{{- if or .Values.gateway.config.create .Values.backend.podAnnotations }}
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.backend.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
labels:
{{- include "litellm.backend.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
@ -34,7 +40,17 @@ spec:
protocol: TCP
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
{{- if .Values.gateway.config.create }}
volumeMounts:
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- with .Values.backend.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
@ -45,6 +61,12 @@ spec:
{{- end }}
resources:
{{- toYaml .Values.backend.resources | nindent 12 }}
{{- if .Values.gateway.config.create }}
volumes:
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- with .Values.backend.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}

3
helm/litellm/templates/gateway/deployment.yaml
View file

@ -22,7 +22,8 @@ spec:
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}

2
helm/litellm/templates/migrations-job.yaml
View file

@ -28,7 +28,7 @@ spec:
app.kubernetes.io/component: migrations
spec:
restartPolicy: Never
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.backend.serviceAccountName" . }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}

46
helm/litellm/templates/serviceaccount.yaml
View file

@ -1,13 +1,51 @@
{{- if .Values.serviceAccount.create -}}
{{- $prev := false -}}
{{- if .Values.serviceAccounts.gateway.create -}}
{{- $prev = true }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.serviceAccountName" . }}
name: {{ include "litellm.gateway.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
app.kubernetes.io/component: gateway
{{- with .Values.serviceAccounts.gateway.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
{{- end }}
{{- if .Values.serviceAccounts.backend.create }}
{{- if $prev }}
---
{{- end }}
{{- $prev = true }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.backend.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
{{- with .Values.serviceAccounts.backend.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }}
{{- end }}
{{- if .Values.serviceAccounts.ui.create }}
{{- if $prev }}
---
{{- end }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.ui.serviceAccountName" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: ui
{{- with .Values.serviceAccounts.ui.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
{{- end }}

3
helm/litellm/templates/ui/deployment.yaml
View file

@ -19,7 +19,8 @@ spec:
labels:
{{- include "litellm.ui.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
serviceAccountName: {{ include "litellm.ui.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.ui.automount }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}

37
helm/litellm/values.yaml
View file

@ -14,16 +14,33 @@ ingress:
host: "" # optional; if set, becomes the rule's host
tls: []
# Shared ServiceAccount used by all three component Deployments. Set
# `create: true` to have the chart provision it (e.g. when wiring an EKS
# Pod Identity association by SA name). Set `name` to use an existing SA
# (chart-created or out-of-band). When both are empty / false, pods run
# with the namespace's `default` SA.
serviceAccount:
create: false
automount: true
annotations: {}
name: ""
# Per-component ServiceAccounts for gateway, backend, and ui.
#
# Each section mirrors the old shared serviceAccount shape. Set `create:
# true` to have the chart provision the SA (useful for EKS Pod Identity /
# GKE Workload Identity annotations). Set `name` to bind an existing SA.
# When both are unset the component pod runs with the namespace `default` SA.
#
# The UI SA deliberately defaults to `automount: false` — the static nginx
# container does not need the K8s API and should not carry a projected
# ServiceAccount token that a compromised container could use to call the
# cloud-provider metadata service or the K8s API.
serviceAccounts:
gateway:
create: false
automount: true
annotations: {}
name: ""
backend:
create: false
automount: true
annotations: {}
name: ""
ui:
create: false
automount: false
annotations: {}
name: ""
# Pre-install / pre-upgrade Helm hook that runs `prisma migrate deploy`
# against the writer database, creating the LiteLLM schema (tables that

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260526120000_add_oauth_passthrough_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "oauth_passthrough" BOOLEAN NOT NULL DEFAULT false;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
allow_all_keys Boolean @default(false)
available_on_public_internet Boolean @default(true)
delegate_auth_to_upstream Boolean @default(false)
oauth_passthrough Boolean @default(false)
is_byok Boolean @default(false)
byok_description String[] @default([])
byok_api_key_help_url String?

22
litellm/__init__.py
View file

@ -240,6 +240,7 @@ api_key: Optional[str] = None
openai_key: Optional[str] = None
groq_key: Optional[str] = None
gigachat_key: Optional[str] = None
xai_key: Optional[str] = None
databricks_key: Optional[str] = None
openai_like_key: Optional[str] = None
azure_key: Optional[str] = None
@ -277,6 +278,7 @@ ovhcloud_key: Optional[str] = None
lemonade_key: Optional[str] = None
sap_service_key: Optional[str] = None
amazon_nova_api_key: Optional[str] = None
inception_key: Optional[str] = None
common_cloud_provider_auth_params: dict = {
"params": ["project", "region_name", "token"],
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
@ -442,6 +444,7 @@ disable_copilot_system_to_assistant: bool = (
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
)
public_mcp_servers: Optional[List[str]] = None
public_mcp_hub_strict_whitelist: bool = True
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
@ -550,6 +553,7 @@ cohere_models: Set = set()
cohere_chat_models: Set = set()
mistral_chat_models: Set = set()
text_completion_codestral_models: Set = set()
text_completion_inception_models: Set = set()
anthropic_models: Set = set()
openrouter_models: Set = set()
datarobot_models: Set = set()
@ -627,6 +631,7 @@ publicai_models: Set = set()
v0_models: Set = set()
morph_models: Set = set()
lambda_ai_models: Set = set()
inception_models: Set = set()
hyperbolic_models: Set = set()
black_forest_labs_models: Set = set()
recraft_models: Set = set()
@ -791,6 +796,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
fireworks_ai_embedding_models.add(key)
elif value.get("litellm_provider") == "text-completion-codestral":
text_completion_codestral_models.add(key)
elif value.get("litellm_provider") == "text-completion-inception":
text_completion_inception_models.add(key)
elif value.get("litellm_provider") == "xai":
xai_models.add(key)
elif value.get("litellm_provider") == "zai":
@ -877,6 +884,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
morph_models.add(key)
elif value.get("litellm_provider") == "lambda_ai":
lambda_ai_models.add(key)
elif value.get("litellm_provider") == "inception":
inception_models.add(key)
elif value.get("litellm_provider") == "hyperbolic":
hyperbolic_models.add(key)
elif value.get("litellm_provider") == "black_forest_labs":
@ -979,6 +988,7 @@ model_list = list(
| watsonx_models
| gemini_models
| text_completion_codestral_models
| text_completion_inception_models
| xai_models
| zai_models
| fal_ai_models
@ -1017,6 +1027,7 @@ model_list = list(
| v0_models
| morph_models
| lambda_ai_models
| inception_models
| black_forest_labs_models
| recraft_models
| cometapi_models
@ -1073,6 +1084,7 @@ models_by_provider: dict = {
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
"aleph_alpha": aleph_alpha_models,
"text-completion-codestral": text_completion_codestral_models,
"text-completion-inception": text_completion_inception_models,
"xai": xai_models,
"zai": zai_models,
"fal_ai": fal_ai_models,
@ -1117,6 +1129,7 @@ models_by_provider: dict = {
"v0": v0_models,
"morph": morph_models,
"lambda_ai": lambda_ai_models,
"inception": inception_models,
"hyperbolic": hyperbolic_models,
"black_forest_labs": black_forest_labs_models,
"recraft": recraft_models,
@ -1727,6 +1740,9 @@ if TYPE_CHECKING:
from .llms.openrouter.responses.transformation import (
OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig,
)
from .llms.bedrock_mantle.responses.transformation import (
BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig,
)
from .llms.gemini.interactions.transformation import (
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
)
@ -1868,6 +1884,9 @@ if TYPE_CHECKING:
from .llms.codestral.completion.transformation import (
CodestralTextCompletionConfig as CodestralTextCompletionConfig,
)
from .llms.inception.completion.transformation import (
InceptionTextCompletionConfig as InceptionTextCompletionConfig,
)
from .llms.azure.azure import (
AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig,
)
@ -1936,6 +1955,9 @@ if TYPE_CHECKING:
from .llms.lambda_ai.chat.transformation import (
LambdaAIChatConfig as LambdaAIChatConfig,
)
from .llms.inception.chat.transformation import (
InceptionChatConfig as InceptionChatConfig,
)
from .llms.hyperbolic.chat.transformation import (
HyperbolicChatConfig as HyperbolicChatConfig,
)

15
litellm/_lazy_imports_registry.py
View file

@ -237,6 +237,7 @@ LLM_CONFIG_NAMES = (
"PerplexityResponsesConfig",
"DatabricksResponsesAPIConfig",
"OpenRouterResponsesAPIConfig",
"BedrockMantleResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
@ -267,6 +268,7 @@ LLM_CONFIG_NAMES = (
"AIMLChatConfig",
"VolcEngineChatConfig",
"CodestralTextCompletionConfig",
"InceptionTextCompletionConfig",
"AzureOpenAIAssistantsAPIConfig",
"HerokuChatConfig",
"CometAPIConfig",
@ -310,6 +312,7 @@ LLM_CONFIG_NAMES = (
"MorphChatConfig",
"RAGFlowConfig",
"LambdaAIChatConfig",
"InceptionChatConfig",
"HyperbolicChatConfig",
"VercelAIGatewayConfig",
"OVHCloudChatConfig",
@ -956,6 +959,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.openrouter.responses.transformation",
"OpenRouterResponsesAPIConfig",
),
"BedrockMantleResponsesAPIConfig": (
".llms.bedrock_mantle.responses.transformation",
"BedrockMantleResponsesAPIConfig",
),
"GoogleAIStudioInteractionsConfig": (
".llms.gemini.interactions.transformation",
"GoogleAIStudioInteractionsConfig",
@ -1040,6 +1047,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
".llms.codestral.completion.transformation",
"CodestralTextCompletionConfig",
),
"InceptionTextCompletionConfig": (
".llms.inception.completion.transformation",
"InceptionTextCompletionConfig",
),
"AzureOpenAIAssistantsAPIConfig": (
".llms.azure.azure",
"AzureOpenAIAssistantsAPIConfig",
@ -1154,6 +1165,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
"MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"),
"RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"),
"LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"),
"InceptionChatConfig": (
".llms.inception.chat.transformation",
"InceptionChatConfig",
),
"HyperbolicChatConfig": (
".llms.hyperbolic.chat.transformation",
"HyperbolicChatConfig",

105
litellm/_service_logger.py
View file

@ -24,6 +24,22 @@ else:
UserAPIKeyAuth = Any
def _get_otel_v2_class() -> Optional[type]:
"""Return the ``OpenTelemetryV2`` class, or ``None`` if the OTel SDK is absent.
Imported lazily: ``litellm.integrations.otel.logger`` imports the OpenTelemetry
SDK at module scope, so importing it eagerly would break installs without the
SDK. The V2 logger only exists when ``LITELLM_OTEL_V2`` is enabled (which
requires the SDK), so a failed import simply means "no V2 logger in play".
"""
try:
from litellm.integrations.otel.logger import OpenTelemetryV2
return OpenTelemetryV2
except Exception:
return None
class ServiceLogging(CustomLogger):
"""
Separate class used for monitoring health of litellm-adjacent services (redis/postgres).
@ -38,6 +54,37 @@ class ServiceLogging(CustomLogger):
if "prometheus_system" in litellm.service_callback:
self.prometheusServicesLogger = PrometheusServicesLogger()
def _resolve_otel_service_logger(self, callback: Any) -> Optional[Any]:
"""Resolve the OTel logger (legacy or V2) to emit a service span on.
Returns the logger instance whose ``async_service_*_hook`` should fire for
this ``callback``, or ``None`` when ``callback`` is not an OTel callback.
The V2 ``OpenTelemetryV2`` logger is a plain ``CustomLogger`` and is NOT a
subclass of the legacy ``OpenTelemetry``, so the legacy ``isinstance``
check alone misses it — which is why redis/postgres service spans never
showed up under ``LITELLM_OTEL_V2``. Match both the legacy and V2 types,
whether the callback is the logger instance itself or the ``"otel"`` string
(which routes to the proxy's registered ``open_telemetry_logger``).
"""
otel_v2_cls = _get_otel_v2_class()
def _is_otel_logger(obj: Any) -> bool:
if isinstance(obj, OpenTelemetry):
return True
return otel_v2_cls is not None and isinstance(obj, otel_v2_cls)
if _is_otel_logger(callback):
return callback
if callback == "otel":
from litellm.proxy.proxy_server import open_telemetry_logger
if open_telemetry_logger is not None and _is_otel_logger(
open_telemetry_logger
):
return open_telemetry_logger
return None
def service_success_hook(
self,
service: ServiceTypes,
@ -129,6 +176,13 @@ class ServiceLogging(CustomLogger):
event_metadata=event_metadata,
)
# OTel loggers already fired this event. ``service_callback`` can hold more
# than one reference that resolves to the *same* logger — the ``"otel"``
# string AND the registered instance both map to ``open_telemetry_logger``
# (the V2 logger self-registers its instance even when the string is
# present, unlike V1). Without this guard each such reference emits its own
# span, so a single DB call shows up as duplicate ``postgres ...`` spans.
emitted_otel_logger_ids: set = set()
for callback in litellm.service_callback:
if callback == "prometheus_system":
await self.init_prometheus_services_logger_if_none()
@ -144,19 +198,18 @@ class ServiceLogging(CustomLogger):
end_time=end_time,
event_metadata=event_metadata,
)
elif callback == "otel" or isinstance(callback, OpenTelemetry):
_otel_logger_to_use: Optional[OpenTelemetry] = None
if isinstance(callback, OpenTelemetry):
_otel_logger_to_use = callback
else:
from litellm.proxy.proxy_server import open_telemetry_logger
if open_telemetry_logger is not None and isinstance(
open_telemetry_logger, OpenTelemetry
):
_otel_logger_to_use = open_telemetry_logger
if _otel_logger_to_use is not None and parent_otel_span is not None:
else:
_otel_logger_to_use = self._resolve_otel_service_logger(callback)
# No ``parent_otel_span is not None`` gate: a background service
# call (no request on the stack) has no parent, and dropping it
# here is what hid those calls from traces entirely. The OTel
# logger decides what to do with a missing parent — legacy V1
# no-ops, V2 emits a root span (and skips metrics-only pings).
if (
_otel_logger_to_use is not None
and id(_otel_logger_to_use) not in emitted_otel_logger_ids
):
emitted_otel_logger_ids.add(id(_otel_logger_to_use))
await _otel_logger_to_use.async_service_success_hook(
payload=payload,
parent_otel_span=parent_otel_span,
@ -238,6 +291,9 @@ class ServiceLogging(CustomLogger):
event_metadata=event_metadata,
)
# Dedupe OTel loggers per event — see ``async_service_success_hook`` for why
# the same logger can be referenced twice in ``service_callback``.
emitted_otel_logger_ids: set = set()
for callback in litellm.service_callback:
if callback == "prometheus_system":
await self.init_prometheus_services_logger_if_none()
@ -255,22 +311,19 @@ class ServiceLogging(CustomLogger):
end_time=end_time,
event_metadata=event_metadata,
)
elif callback == "otel" or isinstance(callback, OpenTelemetry):
_otel_logger_to_use: Optional[OpenTelemetry] = None
if isinstance(callback, OpenTelemetry):
_otel_logger_to_use = callback
else:
from litellm.proxy.proxy_server import open_telemetry_logger
if open_telemetry_logger is not None and isinstance(
open_telemetry_logger, OpenTelemetry
):
_otel_logger_to_use = open_telemetry_logger
else:
_otel_logger_to_use = self._resolve_otel_service_logger(callback)
if not isinstance(error, str):
error = str(error)
if _otel_logger_to_use is not None and parent_otel_span is not None:
# See the success hook: no parent gate, so background failures
# are traced too. V1 no-ops without a parent; V2 emits a root.
if (
_otel_logger_to_use is not None
and id(_otel_logger_to_use) not in emitted_otel_logger_ids
):
emitted_otel_logger_ids.add(id(_otel_logger_to_use))
await _otel_logger_to_use.async_service_failure_hook(
payload=payload,
error=error,
@ -318,6 +371,8 @@ class ServiceLogging(CustomLogger):
service=ServiceTypes.LITELLM,
duration=_duration,
call_type=kwargs.get("call_type", "unknown"),
start_time=start_time,
end_time=end_time,
)
except Exception as e:
raise e

99
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -20,9 +20,20 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
)
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
# litellm_params key carrying the authenticated principal (hashed virtual key) so
# A2A provider configs can scope provider-side state (e.g. LangFlow session memory)
# per key instead of trusting the client-supplied A2A contextId.
A2A_USER_API_KEY_HASH_PARAM = "litellm_a2a_user_api_key_hash"
# Agent metadata fields stored in litellm_params that are not valid litellm.acompletion() kwargs
_AGENT_ONLY_PARAMS = frozenset(
{"is_public", "agent_name", "agent_id", "agent_card_params"}
{
"is_public",
"agent_name",
"agent_id",
"agent_card_params",
A2A_USER_API_KEY_HASH_PARAM,
}
)
@ -37,6 +48,8 @@ class A2ACompletionBridgeHandler:
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
*,
_skip_a2a_provider_routing: bool = False,
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request via litellm.acompletion.
@ -50,25 +63,24 @@ class A2ACompletionBridgeHandler:
Returns:
A2A SendMessageResponse dict
"""
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}")
response_data = await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
if not _skip_a2a_provider_routing:
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
return response_data
if a2a_provider_config is not None:
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider}"
)
return await a2a_provider_config.handle_non_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
)
# Extract message from params
message = params.get("message", {})
@ -107,6 +119,14 @@ class A2ACompletionBridgeHandler:
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
}
completion_params.update(litellm_params_to_add)
# Apply forward metadata AFTER the litellm_params merge so the helper
# sees any agent-owner-configured ``extra_body.metadata`` and can keep
# those keys authoritative over the client-supplied A2A metadata.
A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
completion_params=completion_params,
a2a_message=message,
params=params,
)
# Call litellm.acompletion
response = await litellm.acompletion(**completion_params)
@ -129,6 +149,8 @@ class A2ACompletionBridgeHandler:
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
*,
_skip_a2a_provider_routing: bool = False,
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request via litellm.acompletion with stream=True.
@ -148,28 +170,27 @@ class A2ACompletionBridgeHandler:
Yields:
A2A streaming response events
"""
# Get provider config for custom_llm_provider
custom_llm_provider = litellm_params.get("custom_llm_provider")
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
# If provider config exists, use it
if a2a_provider_config is not None:
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
if not _skip_a2a_provider_routing:
a2a_provider_config = A2AProviderConfigManager.get_provider_config(
custom_llm_provider=custom_llm_provider,
model=litellm_params.get("model"),
)
async for chunk in a2a_provider_config.handle_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk
if a2a_provider_config is not None:
verbose_logger.info(
f"A2A: Using provider config for {custom_llm_provider} (streaming)"
)
return
async for chunk in a2a_provider_config.handle_streaming(
request_id=request_id,
params=params,
api_base=api_base,
litellm_params=litellm_params,
):
yield chunk
return
# Extract message from params
message = params.get("message", {})
@ -214,6 +235,14 @@ class A2ACompletionBridgeHandler:
if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
}
completion_params.update(litellm_params_to_add)
# Apply forward metadata AFTER the litellm_params merge so the helper
# sees any agent-owner-configured ``extra_body.metadata`` and can keep
# those keys authoritative over the client-supplied A2A metadata.
A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
completion_params=completion_params,
a2a_message=message,
params=params,
)
# 1. Emit initial task event (kind: "task", status: "submitted")
task_event = A2ACompletionBridgeTransformation.create_task_event(ctx)

141
litellm/a2a_protocol/litellm_completion_bridge/transformation.py
View file

@ -45,10 +45,80 @@ class A2ACompletionBridgeTransformation:
Static methods for transforming between A2A and OpenAI message formats.
"""
@staticmethod
def _extract_text_from_a2a_parts(parts: List[Dict[str, Any]]) -> str:
"""Extract text from A2A parts (with or without explicit ``kind``)."""
content_parts: List[str] = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
text = part.get("text")
if text is None:
continue
if kind in (None, "", "text"):
content_parts.append(str(text))
return "\n".join(content_parts)
@staticmethod
def get_forward_metadata(
a2a_message: Dict[str, Any],
params: Optional[Dict[str, Any]] = None,
) -> Optional[Dict[str, Any]]:
"""
Merge A2A metadata from MessageSendParams and the message for downstream providers.
Forwarded once on the LangGraph run payload (``metadata``), not duplicated on
each input message — see ``apply_forward_metadata_to_completion_params``.
"""
merged: Dict[str, Any] = {}
if params and isinstance(params.get("metadata"), dict):
merged.update(params["metadata"])
message_metadata = a2a_message.get("metadata")
if isinstance(message_metadata, dict):
merged.update(message_metadata)
return merged or None
@staticmethod
def apply_forward_metadata_to_completion_params(
completion_params: Dict[str, Any],
a2a_message: Dict[str, Any],
params: Optional[Dict[str, Any]] = None,
) -> None:
"""
Attach A2A metadata to completion kwargs for provider bridges (e.g. LangGraph).
Uses ``extra_body`` so we do not collide with LiteLLM's spend-log ``metadata`` kwarg.
"""
forward_metadata = A2ACompletionBridgeTransformation.get_forward_metadata(
a2a_message=a2a_message,
params=params,
)
if not forward_metadata:
return
extra_body = completion_params.get("extra_body")
if not isinstance(extra_body, dict):
extra_body = {}
# Layer client-supplied A2A metadata under any agent-owner-configured
# ``extra_body.metadata`` so the configured keys remain authoritative
# and an A2A caller cannot overwrite server-set run metadata.
existing_metadata = extra_body.get("metadata")
existing_dict: Dict[str, Any] = (
existing_metadata if isinstance(existing_metadata, dict) else {}
)
merged_metadata: Dict[str, Any] = {**forward_metadata, **existing_dict}
extra_body = {**extra_body, "metadata": merged_metadata}
completion_params["extra_body"] = extra_body
verbose_logger.debug(
f"A2A -> completion forward metadata keys={list(forward_metadata.keys())}"
)
@staticmethod
def a2a_message_to_openai_messages(
a2a_message: Dict[str, Any],
) -> List[Dict[str, str]]:
) -> List[Dict[str, Any]]:
"""
Transform an A2A message to OpenAI message format.
@ -70,21 +140,20 @@ class A2ACompletionBridgeTransformation:
elif role == "system":
openai_role = "system"
# Extract text content from parts
content_parts = []
for part in parts:
kind = part.get("kind", "")
if kind == "text":
text = part.get("text", "")
content_parts.append(text)
if not isinstance(parts, list):
parts = []
content = "\n".join(content_parts) if content_parts else ""
content = A2ACompletionBridgeTransformation._extract_text_from_a2a_parts(parts)
# Do not attach A2A message.metadata here — the completion bridge forwards it
# once at run level via extra_body.metadata (LangGraph POST /runs/wait shape).
openai_message: Dict[str, Any] = {"role": openai_role, "content": content}
verbose_logger.debug(
f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}"
)
return [{"role": openai_role, "content": content}]
return [openai_message]
@staticmethod
def openai_response_to_a2a_response(
@ -110,6 +179,7 @@ class A2ACompletionBridgeTransformation:
# Build A2A message
a2a_message = {
"kind": "message",
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
@ -119,9 +189,7 @@ class A2ACompletionBridgeTransformation:
a2a_response = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": a2a_message,
},
"result": a2a_message,
}
verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}")
@ -235,50 +303,3 @@ class A2ACompletionBridgeTransformation:
"taskId": ctx.task_id,
},
}
@staticmethod
def openai_chunk_to_a2a_chunk(
chunk: Any,
request_id: Optional[str] = None,
is_final: bool = False,
) -> Optional[Dict[str, Any]]:
"""
Transform a LiteLLM streaming chunk to A2A streaming format.
NOTE: This method is deprecated for streaming. Use the event-based
methods (create_task_event, create_status_update_event,
create_artifact_update_event) instead for proper A2A streaming.
Args:
chunk: LiteLLM ModelResponse chunk
request_id: Original A2A request ID
is_final: Whether this is the final chunk
Returns:
A2A streaming chunk dict or None if no content
"""
# Extract delta content
content = ""
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
choice = chunk.choices[0]
if hasattr(choice, "delta") and choice.delta:
content = choice.delta.content or ""
if not content and not is_final:
return None
# Build A2A streaming chunk (legacy format)
a2a_chunk = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": {
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
},
"final": is_final,
},
}
return a2a_chunk

17
litellm/a2a_protocol/main.py
View file

@ -159,7 +159,9 @@ async def _send_message_via_completion_bridge(
api_base=api_base,
)
return LiteLLMSendMessageResponse.from_dict(response_dict)
return LiteLLMSendMessageResponse.from_dict(
response_dict, request_id=str(request.id)
)
async def _execute_a2a_send_with_retry(
@ -317,15 +319,6 @@ async def asend_message(
)
card_url = getattr(agent_card, "url", None) if agent_card else None
context_id = trace_id or str(uuid.uuid4())
message = request.params.message
if isinstance(message, dict):
if message.get("context_id") is None:
message["context_id"] = context_id
else:
if getattr(message, "context_id", None) is None:
message.context_id = context_id
a2a_response = await _execute_a2a_send_with_retry(
a2a_client=a2a_client,
request=request,
@ -338,7 +331,9 @@ async def asend_message(
verbose_logger.info(f"A2A send_message completed, request_id={request.id}")
# Wrap in LiteLLM response type for _hidden_params support
response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response)
response = LiteLLMSendMessageResponse.from_a2a_response(
a2a_response, request_id=str(request.id)
)
# Calculate token usage from request and response
response_dict = a2a_response.model_dump(mode="json", exclude_none=True)

12
litellm/a2a_protocol/providers/config_manager.py
View file

@ -48,4 +48,16 @@ class A2AProviderConfigManager:
return BedrockAgentCoreA2AConfig()
if custom_llm_provider == "langflow":
from litellm.a2a_protocol.providers.langflow.config import LangFlowA2AConfig
return LangFlowA2AConfig()
if custom_llm_provider == "watsonx_orchestrate":
from litellm.a2a_protocol.providers.watsonx_orchestrate.config import (
WatsonxOrchestrateA2AConfig,
)
return WatsonxOrchestrateA2AConfig()
return None

0
litellm/a2a_protocol/providers/langflow/__init__.py Normal file
View file

62
litellm/a2a_protocol/providers/langflow/config.py Normal file
View file

@ -0,0 +1,62 @@
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.litellm_completion_bridge.handler import (
A2A_USER_API_KEY_HASH_PARAM,
A2ACompletionBridgeHandler,
)
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.llms.langflow.a2a import merge_a2a_session_into_litellm_params
class LangFlowA2AConfig(BaseA2AProviderConfig):
"""A2A bridge for LangFlow: scopes contextId to the authenticated key as the
LangFlow session_id, then uses completion."""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> Dict[str, Any]:
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for LangFlowA2AConfig "
"(must contain custom_llm_provider and model)"
)
litellm_params = merge_a2a_session_into_litellm_params(
litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
)
return await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
_skip_a2a_provider_routing=True,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs,
) -> AsyncIterator[Dict[str, Any]]:
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for LangFlowA2AConfig "
"(must contain custom_llm_provider and model)"
)
litellm_params = merge_a2a_session_into_litellm_params(
litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
)
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
_skip_a2a_provider_routing=True,
):
yield chunk

74
litellm/a2a_protocol/providers/litellm_completion/README.md
View file

@ -1,74 +0,0 @@
# A2A to LiteLLM Completion Bridge
Routes A2A protocol requests through `litellm.acompletion`, enabling any LiteLLM-supported provider to be invoked via A2A.
## Flow
```
A2A Request → Transform → litellm.acompletion → Transform → A2A Response
```
## SDK Usage
Use the existing `asend_message` and `asend_message_streaming` functions with `litellm_params`:
```python
from litellm.a2a_protocol import asend_message, asend_message_streaming
from a2a.types import SendMessageRequest, SendStreamingMessageRequest, MessageSendParams
from uuid import uuid4
# Non-streaming
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
)
)
response = await asend_message(
request=request,
api_base="http://localhost:2024",
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
)
# Streaming
stream_request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={"role": "user", "parts": [{"kind": "text", "text": "Hello!"}], "messageId": uuid4().hex}
)
)
async for chunk in asend_message_streaming(
request=stream_request,
api_base="http://localhost:2024",
litellm_params={"custom_llm_provider": "langgraph", "model": "agent"},
):
print(chunk)
```
## Proxy Usage
Configure an agent with `custom_llm_provider` in `litellm_params`:
```yaml
agents:
- agent_name: my-langgraph-agent
agent_card_params:
name: "LangGraph Agent"
url: "http://localhost:2024" # Used as api_base
litellm_params:
custom_llm_provider: langgraph
model: agent
```
When an A2A request hits `/a2a/{agent_id}/message/send`, the bridge:
1. Detects `custom_llm_provider` in agent's `litellm_params`
2. Transforms A2A message → OpenAI messages
3. Calls `litellm.acompletion(model="langgraph/agent", api_base="http://localhost:2024")`
4. Transforms response → A2A format
## Classes
- `A2ACompletionBridgeTransformation` - Static methods for message format conversion
- `A2ACompletionBridgeHandler` - Static methods for handling requests (streaming/non-streaming)

5
litellm/a2a_protocol/providers/litellm_completion/__init__.py
View file

@ -1,5 +0,0 @@
"""
LiteLLM Completion bridge provider for A2A protocol.
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
"""

301
litellm/a2a_protocol/providers/litellm_completion/handler.py
View file

@ -1,301 +0,0 @@
"""
Handler for A2A to LiteLLM completion bridge.
Routes A2A requests through litellm.acompletion based on custom_llm_provider.
A2A Streaming Events (in order):
1. Task event (kind: "task") - Initial task creation with status "submitted"
2. Status update (kind: "status-update") - Status change to "working"
3. Artifact update (kind: "artifact-update") - Content/artifact delivery
4. Status update (kind: "status-update") - Final status "completed" with final=true
"""
from typing import Any, AsyncIterator, Dict, Optional
import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.litellm_completion_bridge.pydantic_ai_transformation import (
PydanticAITransformation,
)
from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2ACompletionBridgeTransformation,
A2AStreamingContext,
)
class A2ACompletionBridgeHandler:
"""
Static methods for handling A2A requests via LiteLLM completion.
"""
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> Dict[str, Any]:
"""
Handle non-streaming A2A request via litellm.acompletion.
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
Returns:
A2A SendMessageResponse dict
"""
# Check if this is a Pydantic AI agent request
custom_llm_provider = litellm_params.get("custom_llm_provider")
if custom_llm_provider == "pydantic_ai_agents":
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Routing to Pydantic AI agent at {api_base}"
)
# Send request directly to Pydantic AI agent
response_data = await PydanticAITransformation.send_non_streaming_request(
api_base=api_base,
request_id=request_id,
params=params,
)
return response_data
# Extract message from params
message = params.get("message", {})
# Transform A2A message to OpenAI format
openai_messages = (
A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
model = litellm_params.get("model", "agent")
# Build full model string if provider specified
# Skip prepending if model already starts with the provider prefix
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
full_model = f"{custom_llm_provider}/{model}"
else:
full_model = model
verbose_logger.info(
f"A2A completion bridge: model={full_model}, api_base={api_base}"
)
# Build completion params dict
completion_params = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": False,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
completion_params.update(litellm_params_to_add)
# Call litellm.acompletion
response = await litellm.acompletion(**completion_params)
# Transform response to A2A format
a2a_response = (
A2ACompletionBridgeTransformation.openai_response_to_a2a_response(
response=response,
request_id=request_id,
)
)
verbose_logger.info(f"A2A completion bridge completed: request_id={request_id}")
return a2a_response
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> AsyncIterator[Dict[str, Any]]:
"""
Handle streaming A2A request via litellm.acompletion with stream=True.
Emits proper A2A streaming events:
1. Task event (kind: "task") - Initial task with status "submitted"
2. Status update (kind: "status-update") - Status "working"
3. Artifact update (kind: "artifact-update") - Content delivery
4. Status update (kind: "status-update") - Final "completed" status
Args:
request_id: A2A JSON-RPC request ID
params: A2A MessageSendParams containing the message
litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
api_base: API base URL from agent_card_params
Yields:
A2A streaming response events
"""
# Check if this is a Pydantic AI agent request
custom_llm_provider = litellm_params.get("custom_llm_provider")
if custom_llm_provider == "pydantic_ai_agents":
if api_base is None:
raise ValueError("api_base is required for Pydantic AI agents")
verbose_logger.info(
f"Pydantic AI: Faking streaming for Pydantic AI agent at {api_base}"
)
# Get non-streaming response first
response_data = await PydanticAITransformation.send_non_streaming_request(
api_base=api_base,
request_id=request_id,
params=params,
)
# Convert to fake streaming
async for chunk in PydanticAITransformation.fake_streaming_from_response(
response_data=response_data,
request_id=request_id,
):
yield chunk
return
# Extract message from params
message = params.get("message", {})
# Create streaming context
ctx = A2AStreamingContext(
request_id=request_id,
input_message=message,
)
# Transform A2A message to OpenAI format
openai_messages = (
A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
)
# Get completion params
custom_llm_provider = litellm_params.get("custom_llm_provider")
model = litellm_params.get("model", "agent")
# Build full model string if provider specified
# Skip prepending if model already starts with the provider prefix
if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
full_model = f"{custom_llm_provider}/{model}"
else:
full_model = model
verbose_logger.info(
f"A2A completion bridge streaming: model={full_model}, api_base={api_base}"
)
# Build completion params dict
completion_params = {
"model": full_model,
"messages": openai_messages,
"api_base": api_base,
"stream": True,
}
# Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
litellm_params_to_add = {
k: v
for k, v in litellm_params.items()
if k not in ("model", "custom_llm_provider")
}
completion_params.update(litellm_params_to_add)
# 1. Emit initial task event (kind: "task", status: "submitted")
task_event = A2ACompletionBridgeTransformation.create_task_event(ctx)
yield task_event
# 2. Emit status update (kind: "status-update", status: "working")
working_event = A2ACompletionBridgeTransformation.create_status_update_event(
ctx=ctx,
state="working",
final=False,
message_text="Processing request...",
)
yield working_event
# Call litellm.acompletion with streaming
response = await litellm.acompletion(**completion_params)
# 3. Accumulate content and emit artifact update
accumulated_text = ""
chunk_count = 0
async for chunk in response: # type: ignore[union-attr]
chunk_count += 1
# Extract delta content
content = ""
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
choice = chunk.choices[0]
if hasattr(choice, "delta") and choice.delta:
content = choice.delta.content or ""
if content:
accumulated_text += content
# Emit artifact update with accumulated content
if accumulated_text:
artifact_event = (
A2ACompletionBridgeTransformation.create_artifact_update_event(
ctx=ctx,
text=accumulated_text,
)
)
yield artifact_event
# 4. Emit final status update (kind: "status-update", status: "completed", final: true)
completed_event = A2ACompletionBridgeTransformation.create_status_update_event(
ctx=ctx,
state="completed",
final=True,
)
yield completed_event
verbose_logger.info(
f"A2A completion bridge streaming completed: request_id={request_id}, chunks={chunk_count}"
)
# Convenience functions that delegate to the class methods
async def handle_a2a_completion(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> Dict[str, Any]:
"""Convenience function for non-streaming A2A completion."""
return await A2ACompletionBridgeHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
)
async def handle_a2a_completion_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
api_base: Optional[str] = None,
) -> AsyncIterator[Dict[str, Any]]:
"""Convenience function for streaming A2A completion."""
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
api_base=api_base,
):
yield chunk

284
litellm/a2a_protocol/providers/litellm_completion/transformation.py
View file

@ -1,284 +0,0 @@
"""
Transformation utilities for A2A <-> OpenAI message format conversion.
A2A Message Format:
{
"role": "user",
"parts": [{"kind": "text", "text": "Hello!"}],
"messageId": "abc123"
}
OpenAI Message Format:
{"role": "user", "content": "Hello!"}
A2A Streaming Events:
- Task event (kind: "task") - Initial task creation with status "submitted"
- Status update (kind: "status-update") - Status changes (working, completed)
- Artifact update (kind: "artifact-update") - Content/artifact delivery
"""
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from uuid import uuid4
from litellm._logging import verbose_logger
class A2AStreamingContext:
"""
Context holder for A2A streaming state.
Tracks task_id, context_id, and message accumulation.
"""
def __init__(self, request_id: str, input_message: Dict[str, Any]):
self.request_id = request_id
self.task_id = str(uuid4())
self.context_id = str(uuid4())
self.input_message = input_message
self.accumulated_text = ""
self.has_emitted_task = False
self.has_emitted_working = False
class A2ACompletionBridgeTransformation:
"""
Static methods for transforming between A2A and OpenAI message formats.
"""
@staticmethod
def a2a_message_to_openai_messages(
a2a_message: Dict[str, Any],
) -> List[Dict[str, str]]:
"""
Transform an A2A message to OpenAI message format.
Args:
a2a_message: A2A message with role, parts, and messageId
Returns:
List of OpenAI-format messages
"""
role = a2a_message.get("role", "user")
parts = a2a_message.get("parts", [])
# Map A2A roles to OpenAI roles
openai_role = role
if role == "user":
openai_role = "user"
elif role == "assistant":
openai_role = "assistant"
elif role == "system":
openai_role = "system"
# Extract text content from parts
content_parts = []
for part in parts:
kind = part.get("kind", "")
if kind == "text":
text = part.get("text", "")
content_parts.append(text)
content = "\n".join(content_parts) if content_parts else ""
verbose_logger.debug(
f"A2A -> OpenAI transform: role={role} -> {openai_role}, content_length={len(content)}"
)
return [{"role": openai_role, "content": content}]
@staticmethod
def openai_response_to_a2a_response(
response: Any,
request_id: Optional[str] = None,
) -> Dict[str, Any]:
"""
Transform a LiteLLM ModelResponse to A2A SendMessageResponse format.
Args:
response: LiteLLM ModelResponse object
request_id: Original A2A request ID
Returns:
A2A SendMessageResponse dict
"""
# Extract content from response
content = ""
if hasattr(response, "choices") and response.choices:
choice = response.choices[0]
if hasattr(choice, "message") and choice.message:
content = choice.message.content or ""
# Build A2A message
a2a_message = {
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
}
# Build A2A response
a2a_response = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": a2a_message,
},
}
verbose_logger.debug(f"OpenAI -> A2A transform: content_length={len(content)}")
return a2a_response
@staticmethod
def _get_timestamp() -> str:
"""Get current timestamp in ISO format with timezone."""
return datetime.now(timezone.utc).isoformat()
@staticmethod
def create_task_event(
ctx: A2AStreamingContext,
) -> Dict[str, Any]:
"""
Create the initial task event with status 'submitted'.
This is the first event emitted in an A2A streaming response.
"""
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"contextId": ctx.context_id,
"history": [
{
"contextId": ctx.context_id,
"kind": "message",
"messageId": ctx.input_message.get("messageId", uuid4().hex),
"parts": ctx.input_message.get("parts", []),
"role": ctx.input_message.get("role", "user"),
"taskId": ctx.task_id,
}
],
"id": ctx.task_id,
"kind": "task",
"status": {
"state": "submitted",
},
},
}
@staticmethod
def create_status_update_event(
ctx: A2AStreamingContext,
state: str,
final: bool = False,
message_text: Optional[str] = None,
) -> Dict[str, Any]:
"""
Create a status update event.
Args:
ctx: Streaming context
state: Status state ('working', 'completed')
final: Whether this is the final event
message_text: Optional message text for 'working' status
"""
status: Dict[str, Any] = {
"state": state,
"timestamp": A2ACompletionBridgeTransformation._get_timestamp(),
}
# Add message for 'working' status
if state == "working" and message_text:
status["message"] = {
"contextId": ctx.context_id,
"kind": "message",
"messageId": str(uuid4()),
"parts": [{"kind": "text", "text": message_text}],
"role": "agent",
"taskId": ctx.task_id,
}
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"contextId": ctx.context_id,
"final": final,
"kind": "status-update",
"status": status,
"taskId": ctx.task_id,
},
}
@staticmethod
def create_artifact_update_event(
ctx: A2AStreamingContext,
text: str,
) -> Dict[str, Any]:
"""
Create an artifact update event with content.
Args:
ctx: Streaming context
text: The text content for the artifact
"""
return {
"id": ctx.request_id,
"jsonrpc": "2.0",
"result": {
"artifact": {
"artifactId": str(uuid4()),
"name": "response",
"parts": [{"kind": "text", "text": text}],
},
"contextId": ctx.context_id,
"kind": "artifact-update",
"taskId": ctx.task_id,
},
}
@staticmethod
def openai_chunk_to_a2a_chunk(
chunk: Any,
request_id: Optional[str] = None,
is_final: bool = False,
) -> Optional[Dict[str, Any]]:
"""
Transform a LiteLLM streaming chunk to A2A streaming format.
NOTE: This method is deprecated for streaming. Use the event-based
methods (create_task_event, create_status_update_event,
create_artifact_update_event) instead for proper A2A streaming.
Args:
chunk: LiteLLM ModelResponse chunk
request_id: Original A2A request ID
is_final: Whether this is the final chunk
Returns:
A2A streaming chunk dict or None if no content
"""
# Extract delta content
content = ""
if chunk is not None and hasattr(chunk, "choices") and chunk.choices:
choice = chunk.choices[0]
if hasattr(choice, "delta") and choice.delta:
content = choice.delta.content or ""
if not content and not is_final:
return None
# Build A2A streaming chunk (legacy format)
a2a_chunk = {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": {
"role": "agent",
"parts": [{"kind": "text", "text": content}],
"messageId": uuid4().hex,
},
"final": is_final,
},
}
return a2a_chunk

17
litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py
View file

@ -289,16 +289,16 @@ class PydanticAITransformation:
Transform Pydantic AI task response to standard A2A non-streaming format.
Pydantic AI returns a task with history/artifacts, but the standard A2A
non-streaming format expects:
non-streaming format expects ``result`` to be the Message directly
(``kind="message"``), per the A2A spec / ``SendMessageResponse``:
{
"jsonrpc": "2.0",
"id": "...",
"result": {
"message": {
"role": "agent",
"parts": [{"kind": "text", "text": "..."}],
"messageId": "..."
}
"kind": "message",
"role": "agent",
"parts": [{"kind": "text", "text": "..."}],
"messageId": "..."
}
}
@ -316,6 +316,7 @@ class PydanticAITransformation:
# Build standard A2A message
a2a_message = {
"kind": "message",
"role": "agent",
"parts": parts if parts else [{"kind": "text", "text": full_text}],
"messageId": message_id,
@ -325,9 +326,7 @@ class PydanticAITransformation:
return {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"message": a2a_message,
},
"result": a2a_message,
}
@staticmethod

3
litellm/a2a_protocol/providers/watsonx_orchestrate/__init__.py Normal file
View file

@ -0,0 +1,3 @@
"""
IBM watsonx Orchestrate (WXO) A2A provider.
"""

55
litellm/a2a_protocol/providers/watsonx_orchestrate/config.py Normal file
View file

@ -0,0 +1,55 @@
"""
A2A provider configuration for IBM watsonx Orchestrate (WXO).
"""
from typing import Any, AsyncIterator, Dict, Optional
from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig
from litellm.a2a_protocol.providers.watsonx_orchestrate.handler import (
WatsonxOrchestrateHandler,
)
class WatsonxOrchestrateA2AConfig(BaseA2AProviderConfig):
"""A2A bridge for IBM watsonx Orchestrate (REST runs API + poll/SSE)."""
async def handle_non_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs: Any,
) -> Dict[str, Any]:
"""Handle a non-streaming A2A request via WXO runs API."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for WatsonxOrchestrateA2AConfig "
"(must contain cp4d_host, instance_id, wxo_agent_id, api_key)"
)
return await WatsonxOrchestrateHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
async def handle_streaming(
self,
request_id: str,
params: Dict[str, Any],
api_base: Optional[str] = None,
**kwargs: Any,
) -> AsyncIterator[Dict[str, Any]]:
"""Handle a streaming A2A request via WXO streaming runs API."""
litellm_params = kwargs.get("litellm_params")
if not litellm_params:
raise ValueError(
"litellm_params is required for WatsonxOrchestrateA2AConfig "
"(must contain cp4d_host, instance_id, wxo_agent_id, api_key)"
)
async for chunk in WatsonxOrchestrateHandler.handle_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
):
yield chunk

373
litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py Normal file
View file

@ -0,0 +1,373 @@
"""
Handler for IBM watsonx Orchestrate (WXO) agent provider.
"""
import asyncio
import hashlib
import json
import time
from typing import Any, AsyncIterator, Dict, NamedTuple, Optional, Tuple, cast
import httpx
from litellm._logging import verbose_logger
from litellm.a2a_protocol.providers.watsonx_orchestrate.transformation import (
WatsonxOrchestrateTransformation,
)
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
get_async_httpx_client,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
_IBM_CLOUD_IAM_URL = "https://iam.cloud.ibm.com/identity/token"
_POLL_INTERVAL_S = 2.0
_MAX_POLL_ATTEMPTS = 90
_TOKEN_CACHE_TTL_BUFFER_S = 60
_token_cache: Dict[str, Tuple[str, float]] = {}
class WXORequestParams(NamedTuple):
cp4d_host: str
instance_id: str
wxo_agent_id: str
api_key: str
username: Optional[str]
auth_mode: str
thread_id: Optional[str]
class WatsonxOrchestrateHandler:
@staticmethod
def _http_client(timeout: float = 90.0) -> AsyncHTTPHandler:
return get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
params={"timeout": timeout},
)
@staticmethod
def _token_cache_key(
auth_mode: str,
cp4d_host: str,
api_key: str,
username: Optional[str],
) -> str:
material = f"{auth_mode}:{cp4d_host}:{username or ''}:{api_key}"
return hashlib.sha256(material.encode()).hexdigest()
@staticmethod
def _cp4d_token_ttl_seconds(
expiration: Any, now_wall: Optional[float] = None
) -> int:
# CP4D returns expiration as absolute Unix epoch seconds, not a duration.
expires_at = int(expiration)
wall = now_wall if now_wall is not None else time.time()
return max(expires_at - int(wall), 0)
@staticmethod
async def _get_bearer_token(
cp4d_host: str,
auth_mode: str,
api_key: str,
username: Optional[str] = None,
client: Optional[AsyncHTTPHandler] = None,
) -> str:
cache_key = WatsonxOrchestrateHandler._token_cache_key(
auth_mode, cp4d_host, api_key, username
)
now = time.monotonic()
cached = _token_cache.get(cache_key)
if cached and cached[1] > now:
return cached[0]
if client is None:
client = WatsonxOrchestrateHandler._http_client(timeout=30.0)
if auth_mode == "ibm_cloud":
response = await client.post(
_IBM_CLOUD_IAM_URL,
data={
"grant_type": "urn:ibm:params:oauth:grant-type:apikey",
"apikey": api_key,
},
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
response.raise_for_status()
payload = response.json()
token = str(payload["access_token"])
ttl_s = int(payload.get("expires_in", 3600))
else:
if not username:
raise ValueError(
"'username' is required in litellm_params when auth_mode='cp4d'"
)
token_url = f"{cp4d_host.rstrip('/')}/icp4d-api/v1/authorize"
response = await client.post(
token_url,
json={"username": username, "api_key": api_key},
headers={"Content-Type": "application/json"},
)
response.raise_for_status()
payload = response.json()
token = str(payload["token"])
expiration = payload.get("expiration")
if expiration is None:
ttl_s = 3600
else:
ttl_s = WatsonxOrchestrateHandler._cp4d_token_ttl_seconds(expiration)
expires_at = now + max(ttl_s - _TOKEN_CACHE_TTL_BUFFER_S, 0)
_token_cache[cache_key] = (token, expires_at)
for stale_key, (_, stale_expires_at) in list(_token_cache.items()):
if stale_expires_at <= now:
del _token_cache[stale_key]
return token
@staticmethod
async def _poll_run(
base_url: str,
run_id: str,
auth_headers: Dict[str, str],
client: AsyncHTTPHandler,
max_attempts: int = _MAX_POLL_ATTEMPTS,
interval_s: float = _POLL_INTERVAL_S,
) -> Dict[str, Any]:
url = f"{base_url}/v1/orchestrate/runs/{run_id}"
for attempt in range(max_attempts):
await asyncio.sleep(interval_s)
response = await client.get(url, headers=auth_headers)
response.raise_for_status()
result: Dict[str, Any] = response.json()
status = result.get("status", "")
verbose_logger.debug(
f"WXO: Poll {attempt + 1}/{max_attempts} run='{run_id}' status='{status}'"
)
if status in WatsonxOrchestrateTransformation.TERMINAL_STATES:
return result
raise asyncio.TimeoutError(
f"WXO run '{run_id}' did not reach a terminal state after "
f"{max_attempts * interval_s:.0f}s"
)
@staticmethod
async def _get_successful_run_data(
run_data: Dict[str, Any],
base_url: str,
auth_headers: Dict[str, str],
client: AsyncHTTPHandler,
) -> Dict[str, Any]:
status = run_data.get("status", "")
if status not in WatsonxOrchestrateTransformation.TERMINAL_STATES:
run_id = run_data.get("run_id") or run_data.get("id") or ""
if not run_id:
raise ValueError(f"WXO: No run_id in response: {run_data}")
run_data = await WatsonxOrchestrateHandler._poll_run(
base_url=base_url,
run_id=run_id,
auth_headers=auth_headers,
client=client,
)
status = run_data.get("status", "")
if status not in WatsonxOrchestrateTransformation.SUCCESS_STATES:
raise RuntimeError(
f"WXO run ended with non-success status '{status}': {run_data}"
)
return run_data
@staticmethod
async def _accumulate_wxo_sse_text(response: Any) -> str:
accumulated_text = ""
async for line in response.aiter_lines():
if not line.startswith("data:"):
continue
data_str = line[5:].strip()
if not data_str or data_str == "[DONE]":
continue
try:
event = json.loads(data_str)
except json.JSONDecodeError:
continue
chunk_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result(
event
)
if chunk_text:
accumulated_text += chunk_text
return accumulated_text
@staticmethod
def _extract_litellm_params(litellm_params: Dict[str, Any]) -> WXORequestParams:
cp4d_host = litellm_params.get("cp4d_host") or ""
instance_id = litellm_params.get("instance_id") or ""
wxo_agent_id = litellm_params.get("wxo_agent_id") or ""
api_key = litellm_params.get("api_key") or ""
if not cp4d_host:
raise ValueError("'cp4d_host' is required in litellm_params for WXO agents")
if not instance_id:
raise ValueError(
"'instance_id' is required in litellm_params for WXO agents"
)
if not wxo_agent_id:
raise ValueError(
"'wxo_agent_id' is required in litellm_params for WXO agents"
)
if not api_key:
raise ValueError("'api_key' is required in litellm_params for WXO agents")
return WXORequestParams(
cp4d_host=cp4d_host,
instance_id=instance_id,
wxo_agent_id=wxo_agent_id,
api_key=api_key,
username=litellm_params.get("username") or None,
auth_mode=litellm_params.get("auth_mode") or "cp4d",
thread_id=litellm_params.get("thread_id") or None,
)
@staticmethod
async def handle_non_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
) -> Dict[str, Any]:
wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
client = WatsonxOrchestrateHandler._http_client(timeout=90.0)
token = await WatsonxOrchestrateHandler._get_bearer_token(
cp4d_host=wxo.cp4d_host,
auth_mode=wxo.auth_mode,
api_key=wxo.api_key,
username=wxo.username,
client=client,
)
base_url = WatsonxOrchestrateTransformation.get_api_base_url(
wxo.cp4d_host, wxo.instance_id
)
auth_headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"Accept": "application/json",
}
text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params)
body = WatsonxOrchestrateTransformation.build_wxo_run_body(
wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id
)
run_response = await client.post(
f"{base_url}/v1/orchestrate/runs",
json=body,
headers=auth_headers,
)
run_response.raise_for_status()
run_data: Dict[str, Any] = run_response.json()
run_data = await WatsonxOrchestrateHandler._get_successful_run_data(
run_data=run_data,
base_url=base_url,
auth_headers=auth_headers,
client=client,
)
response_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result(
run_data
)
return WatsonxOrchestrateTransformation.build_a2a_message_response(
request_id=request_id, text=response_text
)
@staticmethod
async def handle_streaming(
request_id: str,
params: Dict[str, Any],
litellm_params: Dict[str, Any],
chunk_size: int = 50,
delay_ms: int = 10,
) -> AsyncIterator[Dict[str, Any]]:
wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
client = WatsonxOrchestrateHandler._http_client(timeout=120.0)
token = await WatsonxOrchestrateHandler._get_bearer_token(
cp4d_host=wxo.cp4d_host,
auth_mode=wxo.auth_mode,
api_key=wxo.api_key,
username=wxo.username,
client=client,
)
base_url = WatsonxOrchestrateTransformation.get_api_base_url(
wxo.cp4d_host, wxo.instance_id
)
auth_headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
"Accept": "text/event-stream, application/json",
}
text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params)
body = WatsonxOrchestrateTransformation.build_wxo_run_body(
wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id
)
try:
response = await client.post(
f"{base_url}/v1/orchestrate/runs/stream",
json=body,
headers=auth_headers,
stream=True,
)
response.raise_for_status()
except httpx.TransportError as exc:
verbose_logger.warning(
f"WXO: Streaming request failed before a run was submitted "
f"({exc!r}), falling back to non-streaming + fake streaming",
exc_info=True,
)
result = await WatsonxOrchestrateHandler.handle_non_streaming(
request_id=request_id,
params=params,
litellm_params=litellm_params,
)
response_text = (
WatsonxOrchestrateTransformation.extract_text_from_a2a_message_response(
result
)
)
async for (
chunk
) in WatsonxOrchestrateTransformation.fake_streaming_from_text(
text=response_text,
request_id=request_id,
chunk_size=chunk_size,
delay_ms=delay_ms,
):
yield chunk
return
content_type = response.headers.get("content-type", "").lower()
if "text/event-stream" not in content_type:
response_body = await response.aread()
result = json.loads(response_body)
result = await WatsonxOrchestrateHandler._get_successful_run_data(
run_data=result,
base_url=base_url,
auth_headers=auth_headers,
client=client,
)
accumulated_text = (
WatsonxOrchestrateTransformation.extract_text_from_wxo_result(result)
)
else:
accumulated_text = await WatsonxOrchestrateHandler._accumulate_wxo_sse_text(
response
)
async for chunk in WatsonxOrchestrateTransformation.fake_streaming_from_text(
text=accumulated_text,
request_id=request_id,
chunk_size=chunk_size,
delay_ms=delay_ms,
):
yield chunk

224
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py Normal file
View file

@ -0,0 +1,224 @@
"""
Transformation layer for IBM watsonx Orchestrate (WXO) agent provider.
WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
POST /v1/orchestrate/runs → submit run, get run_id
GET /v1/orchestrate/runs/{id} → poll until terminal state
POST /v1/orchestrate/runs/stream → native SSE streaming
"""
import asyncio
from typing import Any, AsyncIterator, Dict, Optional
from uuid import uuid4
from litellm._logging import verbose_logger
class WatsonxOrchestrateTransformation:
"""
Handles request/response transformation between A2A and the WXO REST API.
"""
TERMINAL_STATES = frozenset(
{"completed", "succeeded", "failed", "error", "cancelled"}
)
SUCCESS_STATES = frozenset({"completed", "succeeded"})
@staticmethod
def get_api_base_url(cp4d_host: str, instance_id: str) -> str:
"""Build the WXO API base URL from host and instance ID."""
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
@staticmethod
def extract_text_from_a2a_params(params: Dict[str, Any]) -> str:
"""
Extract user message text from A2A MessageSendParams.
A2A format: params.message.parts[*] where part.kind == "text"
"""
message = params.get("message", {})
parts = message.get("parts", [])
texts = []
for part in parts:
if not isinstance(part, dict):
continue
kind = part.get("kind")
if kind in (None, "", "text") and part.get("text"):
texts.append(part["text"])
return " ".join(texts) or ""
@staticmethod
def build_wxo_run_body(
wxo_agent_id: str,
text: str,
thread_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Build the WXO POST /v1/orchestrate/runs request body."""
body: Dict[str, Any] = {
"agent_id": wxo_agent_id,
"message": {
"role": "user",
"content": [
{
"response_type": "text",
"text": text,
}
],
},
}
if thread_id:
body["thread_id"] = thread_id
return body
@staticmethod
def extract_text_from_wxo_result(result: Any) -> str:
"""
Extract response text from a WXO run result.
WXO can return text in several locations; checks in priority order per the API spec.
"""
if not isinstance(result, dict):
return ""
# Primary: last_message.content[0].text
try:
text = result["last_message"]["content"][0]["text"]
if text:
return str(text)
except (KeyError, IndexError, TypeError):
pass
# Secondary: result.data.message.content[0].text
try:
text = result["result"]["data"]["message"]["content"][0]["text"]
if text:
return str(text)
except (KeyError, IndexError, TypeError):
pass
# Tertiary: results as a raw string
results = result.get("results")
if results and isinstance(results, str):
return results
return ""
@staticmethod
def extract_text_from_a2a_message_response(a2a_response: Dict[str, Any]) -> str:
result = a2a_response.get("result")
if not isinstance(result, dict):
verbose_logger.warning("WXO: A2A response missing result object")
return ""
parts = result.get("parts")
if not isinstance(parts, list):
verbose_logger.warning("WXO: A2A result has no parts list")
return ""
for part in parts:
if (
isinstance(part, dict)
and part.get("kind") == "text"
and part.get("text")
):
return str(part["text"])
verbose_logger.warning("WXO: A2A result parts contained no text")
return ""
@staticmethod
def build_a2a_message_response(request_id: str, text: str) -> Dict[str, Any]:
"""
Build a standard A2A non-streaming SendMessageResponse (kind=message).
"""
return {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"kind": "message",
"role": "agent",
"parts": [{"kind": "text", "text": text}],
"messageId": str(uuid4()),
},
}
@staticmethod
async def fake_streaming_from_text(
text: str,
request_id: str,
chunk_size: int = 50,
delay_ms: int = 10,
) -> AsyncIterator[Dict[str, Any]]:
"""
Emit standard A2A streaming events from a completed text response.
Event sequence:
1. task (kind="task", state="submitted")
2. status-update (kind="status-update", state="working")
3. artifact-update chunks
4. status-update (kind="status-update", state="completed", final=True)
"""
task_id = str(uuid4())
context_id = str(uuid4())
artifact_id = str(uuid4())
# 1. Task submitted
yield {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"id": task_id,
"kind": "task",
"status": {"state": "submitted"},
},
}
# 2. Working
yield {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"final": False,
"kind": "status-update",
"status": {"state": "working"},
"taskId": task_id,
},
}
await asyncio.sleep(delay_ms / 1000.0)
# 3. Artifact chunks (always emit at least one chunk, even for empty text)
text_to_chunk = text or ""
for i in range(0, max(len(text_to_chunk), 1), chunk_size):
chunk_text = text_to_chunk[i : i + chunk_size]
is_last = (i + chunk_size) >= max(len(text_to_chunk), 1)
yield {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"kind": "artifact-update",
"taskId": task_id,
"artifact": {
"artifactId": artifact_id,
"parts": [{"kind": "text", "text": chunk_text}],
},
},
}
if not is_last:
await asyncio.sleep(delay_ms / 1000.0)
# 4. Completed
yield {
"jsonrpc": "2.0",
"id": request_id,
"result": {
"contextId": context_id,
"final": True,
"kind": "status-update",
"status": {"state": "completed"},
"taskId": task_id,
},
}
verbose_logger.debug(
f"WXO: Fake streaming completed for request_id={request_id}"
)

6
litellm/a2a_protocol/utils.py
View file

@ -60,6 +60,12 @@ class A2ARequestUtils:
if not isinstance(result, dict):
return ""
# Direct message format (A2A spec): detect by explicit kind tag only.
# The "parts" heuristic is too broad and would match any future result
# type that happens to include a "parts" field.
if result.get("kind") == "message":
return A2ARequestUtils.extract_text_from_message(result)
message = result.get("message", {})
return A2ARequestUtils.extract_text_from_message(message)

7
litellm/anthropic_interface/messages/__init__.py
View file

@ -10,7 +10,7 @@ This is an __init__.py file to allow the following interface
"""
from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union
from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional, Union
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages as _async_anthropic_messages,
@ -100,8 +100,11 @@ def create(
**kwargs,
) -> Union[
AnthropicMessagesResponse,
Iterator[bytes],
AsyncIterator[Any],
Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]],
Coroutine[
Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any], Iterator[bytes]]
],
]:
"""
Async wrapper for Anthropic's messages API

15
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -182,6 +182,14 @@ class ResponsesToCompletionBridgeHandler:
client=kwargs.get("client"),
)
# Pin the resolved provider so `responses()` doesn't re-run
# `get_llm_provider()` on the model string and strip a second
# provider prefix (see GitHub issue #28505). request_data already
# carries `custom_llm_provider` via the spread of
# `sanitized_litellm_params`; overwriting it on the dict (rather
# than adding an explicit kwarg) avoids the duplicate-keyword
# TypeError that would otherwise fire on the real bridge path.
request_data["custom_llm_provider"] = custom_llm_provider
result = responses(
**request_data,
)
@ -268,6 +276,13 @@ class ResponsesToCompletionBridgeHandler:
except Exception as e:
raise e
# Pin the resolved provider so `aresponses()` doesn't re-run
# `get_llm_provider()` on the model string and strip a second
# provider prefix (see GitHub issue #28505). Set on request_data
# rather than passed as a separate kwarg to avoid the duplicate-
# keyword TypeError when `sanitized_litellm_params` already
# carries `custom_llm_provider`.
request_data["custom_llm_provider"] = custom_llm_provider
result = await aresponses(
**request_data,
aresponses=True,

18
litellm/constants.py
View file

@ -585,6 +585,7 @@ LITELLM_CHAT_PROVIDERS = [
"volcengine",
"codestral",
"text-completion-codestral",
"text-completion-inception",
"deepseek",
"sambanova",
"maritalk",
@ -620,6 +621,7 @@ LITELLM_CHAT_PROVIDERS = [
"oci",
"morph",
"lambda_ai",
"inception",
"vercel_ai_gateway",
"wandb",
"ovhcloud",
@ -676,6 +678,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [
"extra_headers",
"thinking",
"web_search_options",
"include_server_side_tool_invocations",
"service_tier",
"prompt_cache_key",
"prompt_cache_retention",
@ -737,6 +740,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
"verbosity": None,
"thinking": None,
"web_search_options": None,
"include_server_side_tool_invocations": None,
"service_tier": None,
"safety_identifier": None,
"prompt_cache_key": None,
@ -771,6 +775,7 @@ openai_compatible_endpoints: List = [
"https://api.moonshot.ai/v1",
"https://api.publicai.co/v1",
"https://api.synthetic.new/openai/v1",
"https://serverless.tensormesh.ai/v1",
"https://api.stima.tech/v1",
"https://nano-gpt.com/api/v1",
"https://api.poe.com/v1",
@ -778,6 +783,7 @@ openai_compatible_endpoints: List = [
"https://api.v0.dev/v1",
"https://api.morphllm.com/v1",
"https://api.lambda.ai/v1",
"https://api.inceptionlabs.ai/v1",
"https://api.hyperbolic.xyz/v1",
"https://ai-gateway.helicone.ai/",
"https://ai-gateway.vercel.sh/v1",
@ -820,6 +826,7 @@ openai_compatible_providers: List = [
"meta_llama",
"publicai", # PublicAI - JSON-configured provider
"synthetic", # Synthetic - JSON-configured provider
"tensormesh", # Tensormesh - JSON-configured provider
"apertis", # Apertis - JSON-configured provider
"nano-gpt", # Nano-GPT - JSON-configured provider
"poe", # Poe - JSON-configured provider
@ -833,6 +840,7 @@ openai_compatible_providers: List = [
"helicone",
"morph",
"lambda_ai",
"inception",
"hyperbolic",
"vercel_ai_gateway",
"aiml",
@ -855,6 +863,7 @@ openai_text_completion_compatible_providers: List = (
"moonshot",
"publicai",
"synthetic",
"tensormesh",
"apertis",
"nano-gpt",
"poe",
@ -868,6 +877,7 @@ openai_text_completion_compatible_providers: List = (
_openai_like_providers: List = [
"predibase",
"databricks",
"lemonade",
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
@ -1147,6 +1157,7 @@ BEDROCK_CONVERSE_MODELS = [
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-opus-4-8",
"anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-6-v1:0",
"anthropic.claude-opus-4-6-v1",
@ -1408,6 +1419,13 @@ LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
# Prometheus metrics, audit trails, or any other downstream consumer.
LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
# Marker placed in ``model_call_details`` on a synthetic ``Logging`` object that
# records a proxy-gate error (auth/rate-limit rejection) for a request that never
# reached an upstream provider. Tracing callbacks key off it to avoid fabricating
# an LLM-call span for a call that did not happen. See
# ``ProxyLogging._handle_logging_proxy_only_error``.
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL = "litellm_no_upstream_llm_call"
# Key Rotation Constants
LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int(

2
litellm/cost_calculator.py
View file

@ -133,6 +133,8 @@ _VIDEO_CALL_TYPES = frozenset(
{
CallTypes.create_video.value,
CallTypes.acreate_video.value,
CallTypes.video_edit.value,
CallTypes.avideo_edit.value,
CallTypes.video_remix.value,
CallTypes.avideo_remix.value,
}

129
litellm/experimental_mcp_client/client.py
View file

@ -4,6 +4,7 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
import asyncio
import base64
import os
from typing import (
Any,
Awaitable,
@ -16,7 +17,6 @@ from typing import (
TypeVar,
Union,
)
import httpx
from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
from mcp.client.sse import sse_client
@ -42,9 +42,8 @@ from mcp.types import (
)
from mcp.types import Tool as MCPTool
from pydantic import AnyUrl
from litellm._logging import verbose_logger
from litellm.constants import MCP_CLIENT_TIMEOUT
from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR
from litellm.llms.custom_httpx.http_handler import get_ssl_configuration
from litellm.types.llms.custom_http import VerifyTypes
from litellm.types.mcp import (
@ -67,7 +66,6 @@ TSessionResult = TypeVar("TSessionResult")
class MCPSigV4Auth(httpx.Auth):
"""
httpx Auth class that signs each request with AWS SigV4.
This is used for MCP servers that require AWS SigV4 authentication,
such as AWS Bedrock AgentCore MCP servers. httpx calls auth_flow()
for every outgoing request, enabling per-request signature computation.
@ -92,10 +90,8 @@ class MCPSigV4Auth(httpx.Auth):
"Missing botocore to use AWS SigV4 authentication. "
"Run 'pip install boto3'."
)
self.service_name = aws_service_name or "bedrock-agentcore"
self.region_name = aws_region_name or "us-east-1"
# Note: os.environ/ prefixed values are already resolved by
# ProxyConfig._check_for_os_environ_vars() at config load time.
# Values arrive here as plain strings.
@ -143,20 +139,17 @@ class MCPSigV4Auth(httpx.Auth):
session_name = (
aws_session_name or f"litellm-mcp-{int(__import__('time').time())}"
)
sts_kwargs: dict = {"region_name": aws_region_name}
if aws_access_key_id and aws_secret_access_key:
sts_kwargs["aws_access_key_id"] = aws_access_key_id
sts_kwargs["aws_secret_access_key"] = aws_secret_access_key
if aws_session_token:
sts_kwargs["aws_session_token"] = aws_session_token
sts_client = boto3.client("sts", **sts_kwargs)
sts_response = sts_client.assume_role(
RoleArn=aws_role_name,
RoleSessionName=session_name,
)
sts_creds = sts_response["Credentials"]
return Credentials(
access_key=sts_creds["AccessKeyId"],
@ -178,17 +171,14 @@ class MCPSigV4Auth(httpx.Auth):
data=request.content,
headers=dict(request.headers),
)
# Sign the request — SigV4Auth.add_auth() adds Authorization,
# X-Amz-Date, and X-Amz-Security-Token (if session token present).
# Host header is derived automatically from the URL.
sigv4 = SigV4Auth(self.credentials, self.service_name, self.region_name)
sigv4.add_auth(aws_request)
# Copy SigV4 headers back to the httpx request
for header_name, header_value in aws_request.headers.items():
request.headers[header_name] = header_value
yield request
@ -198,6 +188,8 @@ class MCPClient:
SSE and HTTP transports
Authentication via Bearer token, Basic Auth, or API Key
Tool calling with error handling and result parsing
Sampling callbacks for upstream server LLM requests
Elicitation callbacks for upstream server user-input requests
"""
def __init__(
@ -211,6 +203,9 @@ class MCPClient:
extra_headers: Optional[Dict[str, str]] = None,
ssl_verify: Optional[VerifyTypes] = None,
aws_auth: Optional[httpx.Auth] = None,
sampling_callback: Optional[Callable] = None,
elicitation_callback: Optional[Callable] = None,
logging_callback: Optional[Callable] = None,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@ -222,6 +217,9 @@ class MCPClient:
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
self._aws_auth: Optional[httpx.Auth] = aws_auth
self._last_initialize_instructions: Optional[str] = None
self._sampling_callback: Optional[Callable] = sampling_callback
self._elicitation_callback: Optional[Callable] = elicitation_callback
self._logging_callback: Optional[Callable] = logging_callback
# handle the basic auth value if provided
if auth_value:
self.update_auth_value(auth_value)
@ -231,23 +229,20 @@ class MCPClient:
) -> Tuple[Any, Optional[httpx.AsyncClient]]:
"""
Create the appropriate transport context based on transport type.
Returns:
Tuple of (transport_context, http_client).
http_client is only set for HTTP transport and needs cleanup.
"""
http_client: Optional[httpx.AsyncClient] = None
if self.transport_type == MCPTransport.stdio:
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {}),
env=self._get_safe_stdio_env(self.stdio_config.get("env")),
)
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
@ -260,14 +255,12 @@ class MCPClient:
),
None,
)
# HTTP transport (default)
if streamable_http_client is None:
raise ImportError(
"streamable_http_client is not available. "
"Please install mcp with HTTP support."
)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
@ -281,6 +274,54 @@ class MCPClient:
)
return transport_ctx, http_client
def _get_safe_stdio_env(
self, provided_env: Optional[Dict[str, str]]
) -> Optional[Dict[str, str]]:
"""
Return a safe environment for the stdio subprocess.
If provided_env is set, we use it as-is.
If provided_env is None, we return a minimal allowlist from the parent environment
to avoid leaking sensitive LiteLLM keys (OPENAI_API_KEY, etc.) to sub-processes.
"""
if provided_env is not None:
return provided_env
# Minimal allowlist of safe/standard environment variables
safe_keys = {
"PATH",
"HOME",
"USER",
"LOGNAME",
"TMPDIR",
"TMP",
"TEMP",
"SHELL",
"LANG",
"LC_ALL",
# Node/Package manager caches
"NPM_CONFIG_CACHE",
"PNPM_HOME",
"XDG_CACHE_HOME",
"XDG_CONFIG_HOME",
"XDG_DATA_HOME",
# System info
"SYSTEMROOT",
"COMSPEC",
"PATHEXT",
"WINDIR",
}
safe_env = {}
for key in safe_keys:
if key in os.environ:
safe_env[key] = os.environ[key]
if "NPM_CONFIG_CACHE" not in safe_env:
safe_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR
return safe_env
async def _execute_session_operation(
self,
transport_ctx: Any,
@ -288,13 +329,23 @@ class MCPClient:
) -> TSessionResult:
"""
Execute an operation within a transport and session context.
Handles entering/exiting contexts and running the operation.
Passes sampling/elicitation/logging callbacks to the ClientSession
so that upstream MCP servers can request LLM inference (sampling),
user input (elicitation), or send log messages.
"""
transport = await transport_ctx.__aenter__()
try:
read_stream, write_stream = transport[0], transport[1]
session_ctx = ClientSession(read_stream, write_stream)
# Build session kwargs with optional callbacks
session_kwargs: Dict[str, Any] = {}
if self._sampling_callback is not None:
session_kwargs["sampling_callback"] = self._sampling_callback
if self._elicitation_callback is not None:
session_kwargs["elicitation_callback"] = self._elicitation_callback
if self._logging_callback is not None:
session_kwargs["logging_callback"] = self._logging_callback
session_ctx = ClientSession(read_stream, write_stream, **session_kwargs)
session = await session_ctx.__aenter__()
try:
init_result = await session.initialize()
@ -351,7 +402,6 @@ class MCPClient:
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers = {}
if self._mcp_auth_value:
if isinstance(self._mcp_auth_value, str):
if self.auth_type == MCPAuth.bearer_token:
@ -373,17 +423,14 @@ class MCPClient:
# Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
# signing (including the body hash), so it uses httpx.Auth flow instead
# of static headers. See MCPSigV4Auth and _create_httpx_client_factory().
# update the headers with the extra headers
if self.extra_headers:
headers.update(self.extra_headers)
return headers
def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]:
"""
Create a custom httpx client factory that uses LiteLLM's SSL configuration.
This factory follows the same CA bundle path logic as http_handler.py:
1. Check ssl_verify parameter (can be SSLContext, bool, or path to CA bundle)
2. Check SSL_VERIFY environment variable
@ -400,17 +447,14 @@ class MCPClient:
"""Create an httpx.AsyncClient with LiteLLM's SSL configuration."""
# Get unified SSL configuration using the same logic as http_handler.py
ssl_config = get_ssl_configuration(self.ssl_verify)
verbose_logger.debug(
f"MCP client using SSL configuration: {type(ssl_config).__name__}"
)
# Use SigV4 auth if configured and no explicit auth provided.
# The MCP SDK's sse_client and streamable_http_client call this
# factory without passing auth=, so self._aws_auth is used.
# For non-SigV4 clients, self._aws_auth is None — no behavior change.
effective_auth = auth if auth is not None else self._aws_auth
return httpx.AsyncClient(
headers=headers,
timeout=timeout,
@ -421,8 +465,16 @@ class MCPClient:
return factory
async def list_tools(self) -> List[MCPTool]:
"""List available tools from the server."""
async def list_tools(self, raise_on_error: bool = False) -> List[MCPTool]:
"""List available tools from the server.
Args:
raise_on_error: When True, re-raise exceptions instead of returning
an empty list. Used by the proxy's pass-through MCP flow so it
can surface upstream HTTP 401 responses as a proper 401 to the
MCP client (triggering the upstream OAuth flow) rather than
masking them as "connected, no tools".
"""
verbose_logger.debug(
f"MCP client listing tools from {self.server_url or 'stdio'}"
)
@ -450,7 +502,6 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
@ -458,6 +509,8 @@ class MCPClient:
"the MCP server may have crashed, disconnected, or timed out"
)
if raise_on_error:
raise
# Return empty list instead of raising to allow graceful degradation
return []
@ -481,7 +534,6 @@ class MCPClient:
f"MCP Tool '{call_tool_request_params.name}' progress: "
f"{progress}/{total} ({percentage:.0f}%) - {message or ''}"
)
# Forward to Host if callback provided
if host_progress_callback:
try:
@ -511,7 +563,6 @@ class MCPClient:
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
@ -522,14 +573,12 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream - "
"the MCP server may have crashed, disconnected, or timed out."
)
# Return a default error result instead of raising
return MCPCallToolResult(
content=[
@ -567,14 +616,12 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_tools - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
@ -607,7 +654,6 @@ class MCPClient:
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
@ -618,14 +664,12 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during get_prompt - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise
async def list_resources(self) -> list[Resource]:
@ -657,14 +701,12 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resources - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
@ -699,14 +741,12 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during list_resource_templates - "
"the MCP server may have crashed, disconnected, or timed out"
)
# Return empty list instead of raising to allow graceful degradation
return []
@ -732,7 +772,6 @@ class MCPClient:
error_trace = traceback.format_exc()
verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}")
# Log detailed error information
error_type = type(e).__name__
verbose_logger.error(
@ -743,12 +782,10 @@ class MCPClient:
f"Server: {self.server_url or 'stdio'}, "
f"Transport: {self.transport_type}"
)
# Check if it's a stream/connection error
if "BrokenResourceError" in error_type or "Broken" in error_type:
verbose_logger.error(
"MCP client detected broken connection/stream during read_resource - "
"the MCP server may have crashed, disconnected, or timed out."
)
raise

702
litellm/integrations/arize/_utils.py
View file

@ -8,18 +8,23 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.types.utils import StandardLoggingPayload
if TYPE_CHECKING:
from opentelemetry.trace import Span
from litellm.integrations._types.open_inference import (
MessageAttributes,
ImageAttributes,
SpanAttributes,
AudioAttributes,
EmbeddingAttributes,
ImageAttributes,
MessageAttributes,
MessageContentAttributes,
OpenInferenceSpanKindValues,
SpanAttributes,
ToolCallAttributes,
)
@ -53,40 +58,24 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes):
msg.get("content", ""),
)
@staticmethod
@override
def set_response_output_messages(span: "Span", response_obj):
"""
Sets output message attributes on the span from the LLM response.
Args:
span: The OpenTelemetry span to set attributes on
response_obj: The response object containing choices with messages
"""
from litellm.integrations._types.open_inference import (
MessageAttributes,
SpanAttributes,
)
# Additive: emit structured tool_calls / multimodal content
# so Arize/Phoenix can render tool-using and image-bearing
# turns. These set NEW attribute keys (MESSAGE_TOOL_CALLS /
# MESSAGE_NAME / MESSAGE_TOOL_CALL_ID / MESSAGE_CONTENTS.*) —
# never replace the MESSAGE_CONTENT write above.
_safe_emit(
f"input message extras (idx={idx})",
_emit_input_message_extras,
span,
prefix,
msg,
)
for idx, choice in enumerate(response_obj.get("choices", [])):
response_message = choice.get("message", {})
safe_set_attribute(
span,
SpanAttributes.OUTPUT_VALUE,
response_message.get("content", ""),
)
# This shows up under `output_messages` tab on the span page.
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{idx}"
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
response_message.get("role"),
)
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
response_message.get("content", ""),
)
# Note: `BaseLLMObsOTELAttributes.set_response_output_messages` is not
# overridden here. The live code path uses `_set_choice_outputs` (called
# via `_set_response_attributes` from `set_attributes`) which handles
# tool_calls, multimodal output, embeddings, audio, images, and structured
# outputs in a single place.
def _set_response_attributes(span: "Span", response_obj):
@ -106,11 +95,17 @@ def _set_response_attributes(span: "Span", response_obj):
def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
for idx, choice in enumerate(response_obj.get("choices", [])):
response_message = choice.get("message", {})
safe_set_attribute(
span,
span_attrs.OUTPUT_VALUE,
response_message.get("content", ""),
)
content = response_message.get("content", "")
# Tool-only assistant responses have empty content; serialize the
# tool_calls into OUTPUT_VALUE so Arize's "Output" pane isn't blank.
output_value = content
if not output_value:
tool_calls = _get_tool_calls(response_message)
if tool_calls:
output_value = _summarize_tool_calls_for_output(tool_calls)
safe_set_attribute(span, span_attrs.OUTPUT_VALUE, output_value)
prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{idx}"
safe_set_attribute(
span,
@ -120,7 +115,18 @@ def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
safe_set_attribute(
span,
f"{prefix}.{msg_attrs.MESSAGE_CONTENT}",
response_message.get("content", ""),
content,
)
# Additive: emit assistant tool_calls so tool-using turns render in
# Arize/Phoenix. Sets new MESSAGE_TOOL_CALLS keys only — does not
# change MESSAGE_CONTENT/MESSAGE_ROLE writes above.
_safe_emit(
f"output tool_calls (idx={idx})",
_emit_message_tool_calls,
span,
prefix,
response_message,
)
@ -278,6 +284,43 @@ def _set_usage_outputs(span: "Span", response_obj, span_attrs):
reasoning_tokens,
)
# Additive: cache token breakdown so prompt-caching savings render in
# Arize. Sources covered:
# - OpenAI Chat Completions: `prompt_tokens_details.cached_tokens`
# - Anthropic / Bedrock-Anthropic: `cache_read_input_tokens`,
# `cache_creation_input_tokens`
# All emits are conditional, so when none of these fields exist (the
# situation in the existing test fixtures) no extra attributes are set.
prompt_token_details = _safe_get(usage, "prompt_tokens_details") or _safe_get(
usage, "input_tokens_details"
)
cache_read = _safe_get(prompt_token_details, "cached_tokens") or _safe_get(
usage, "cache_read_input_tokens"
)
if cache_read:
safe_set_attribute(
span,
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ,
cache_read,
)
# Anthropic / Bedrock-Anthropic only — OpenAI's `prompt_tokens_details`
# does not expose a cache-write count, so we read straight off `usage`.
cache_write = _safe_get(usage, "cache_creation_input_tokens")
if cache_write:
safe_set_attribute(
span,
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE,
cache_write,
)
audio_prompt_tokens = _safe_get(prompt_token_details, "audio_tokens")
if audio_prompt_tokens:
safe_set_attribute(
span,
span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_AUDIO,
audio_prompt_tokens,
)
def _infer_open_inference_span_kind(call_type: Optional[str]) -> str:
"""
@ -321,6 +364,10 @@ def _infer_open_inference_span_kind(call_type: Optional[str]) -> str:
"videos",
"realtime",
"pass_through",
# `passthrough` (no underscore) is what real call_types use:
# `allm_passthrough_route`, `llm_passthrough_route`. Without
# this they fell through to UNKNOWN, blanking span.kind.
"passthrough",
"anthropic_messages",
"ocr",
)
@ -396,6 +443,18 @@ def set_attributes(
"""
Populates span with OpenInference-compliant LLM attributes for Arize and Phoenix tracing.
"""
# Coerce non-dict response objects (e.g. httpx.Response from passthrough
# routes) into a dict so downstream `.get()` calls don't crash. Existing
# dict / `.get()`-bearing objects (incl. Pydantic OpenAI Responses API
# models) are returned unchanged, preserving the existing test behavior.
response_obj_for_attrs = _coerce_response_obj_for_attrs(response_obj)
# Set span.kind defensively before anything else. If a downstream step
# throws, the span still has a kind so Arize can render it correctly
# (an LLM call instead of UNKNOWN). This is the single source of truth
# for span.kind — no late re-write happens below.
_safe_emit("early span kind", _set_early_span_kind, span, kwargs)
try:
optional_params = _sanitize_optional_params(kwargs.get("optional_params"))
litellm_params = kwargs.get("litellm_params", {}) or {}
@ -415,25 +474,22 @@ def set_attributes(
metadata_tools = _extract_metadata_tools(metadata)
optional_tools = _extract_optional_tools(optional_params)
call_type = standard_logging_payload.get("call_type")
_set_request_attributes(
span=span,
kwargs=kwargs,
standard_logging_payload=standard_logging_payload,
optional_params=optional_params,
litellm_params=litellm_params,
response_obj=response_obj,
response_obj=response_obj_for_attrs,
span_attrs=SpanAttributes,
)
span_kind = _infer_open_inference_span_kind(call_type=call_type)
# span.kind was already set above by `_set_early_span_kind`. We do
# NOT re-write it here based on tool presence: a chat completion
# that passes `tools=[...]` (or returns `tool_calls`) is still an
# LLM call per the OpenInference spec — TOOL is reserved for actual
# tool execution spans, not LLM calls that request tools.
_set_tool_attributes(span, optional_tools, metadata_tools)
if (
optional_tools or metadata_tools
) and span_kind != OpenInferenceSpanKindValues.TOOL.value:
span_kind = OpenInferenceSpanKindValues.TOOL.value
safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, span_kind)
attributes.set_messages(span, kwargs)
model_params = (
@ -443,7 +499,7 @@ def set_attributes(
)
_set_model_params(span, model_params, SpanAttributes)
_set_response_attributes(span=span, response_obj=response_obj)
_set_response_attributes(span=span, response_obj=response_obj_for_attrs)
except Exception as e:
verbose_logger.error(
@ -452,6 +508,22 @@ def set_attributes(
if hasattr(span, "record_exception"):
span.record_exception(e)
# Additive emitters. Each is independently guarded so a failure can never
# blank the attributes set by the main try-block above. New attributes are
# written under new keys; existing attributes are not overwritten.
slp = kwargs.get("standard_logging_object")
_safe_emit("session/user attrs", _set_session_and_user_attrs, span, kwargs, slp)
_safe_emit("response cost", _set_response_cost_attr, span, slp)
_safe_emit(
"passthrough normalization",
_maybe_normalize_passthrough,
span,
kwargs,
response_obj,
response_obj_for_attrs,
slp,
)
def _sanitize_optional_params(optional_params: Optional[dict]) -> dict:
if not isinstance(optional_params, dict):
@ -534,3 +606,529 @@ def _set_model_params(span: "Span", model_params: Optional[dict], span_attrs) ->
user_id = model_params.get("user")
if user_id is not None:
safe_set_attribute(span, span_attrs.USER_ID, user_id)
# ---------------------------------------------------------------------------
# Additive rendering helpers (introduced to enhance Arize/Phoenix rendering
# without changing any previously-emitted attribute keys or values).
# ---------------------------------------------------------------------------
def _safe_emit(label: str, fn, *args, **kwargs) -> None:
"""Run an additive attribute emitter, swallowing any error so it cannot
blank attributes set elsewhere on the span. Failures are logged at debug.
"""
try:
fn(*args, **kwargs)
except Exception as e:
verbose_logger.debug("[Arize] %s skipped: %s", label, e)
def _set_early_span_kind(span: "Span", kwargs: dict) -> None:
"""Defensively set OPENINFERENCE_SPAN_KIND before any other logic runs."""
slp = kwargs.get("standard_logging_object")
call_type = slp.get("call_type") if isinstance(slp, dict) else None
safe_set_attribute(
span,
SpanAttributes.OPENINFERENCE_SPAN_KIND,
_infer_open_inference_span_kind(call_type=call_type),
)
def _coerce_response_obj_for_attrs(response_obj):
"""Return a `.get`-compatible view of `response_obj` when possible.
- dicts and Pydantic models that already expose `.get` are returned
unchanged (preserves all current behavior, including the Responses API
flow which relies on Pydantic attribute access).
- `httpx.Response` and other text-only responses (passthrough routes)
are JSON-decoded so the standard extraction paths can read fields like
`id`, `model`, and `usage`. On failure the original object is returned
so behavior is no worse than today.
"""
if response_obj is None or hasattr(response_obj, "get"):
return response_obj
text = getattr(response_obj, "text", None)
if isinstance(text, str) and text:
try:
parsed = json.loads(text)
if isinstance(parsed, dict):
return parsed
except Exception:
pass
return response_obj
def _coerce_text(value) -> Optional[str]:
"""Best-effort text extraction from a message-content value.
Returns None when no textual portion can be derived. Handles:
- plain strings
- lists of OpenAI-style content parts (`{"type": "text", "text": ...}`)
- lists of Anthropic-style content parts (`{"type": "text", "text": ...}`
or `{"type": "input_text", "text": ...}`)
"""
if value is None:
return None
if isinstance(value, str):
return value
if isinstance(value, list):
parts = []
for part in value:
if isinstance(part, str):
parts.append(part)
elif isinstance(part, dict):
text = part.get("text") or part.get("input_text")
if isinstance(text, str):
parts.append(text)
if parts:
return "\n".join(parts)
return None
def _to_plain_dict(value):
"""Best-effort: coerce a value (Pydantic model / dict / None) to a dict.
Returns the original value when no safe conversion exists. Used to bridge
OpenAI Pydantic message/tool_call objects into the dict-based helpers.
"""
if value is None or isinstance(value, dict):
return value
model_dump = getattr(value, "model_dump", None)
if callable(model_dump):
try:
return model_dump()
except Exception:
pass
return value
def _get_tool_calls(message) -> Optional[list]:
"""Return ``message.tool_calls`` only when it's a non-empty list.
Works for dicts and Pydantic message objects via ``_safe_get``.
"""
tool_calls = _safe_get(message, "tool_calls")
return tool_calls if isinstance(tool_calls, list) and tool_calls else None
def _normalize_tool_call(raw_tc) -> Optional[Dict[str, Any]]:
"""Normalize a single tool_call (dict or Pydantic) into a stable shape:
{"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}}
Arguments are coerced to a JSON string per OpenInference convention.
Returns ``None`` when ``raw_tc`` cannot be coerced to a dict.
"""
tc = _to_plain_dict(raw_tc)
if not isinstance(tc, dict):
return None
function = _to_plain_dict(tc.get("function"))
name = function.get("name") if isinstance(function, dict) else None
args = function.get("arguments") if isinstance(function, dict) else None
if args is not None and not isinstance(args, str):
try:
args = json.dumps(args)
except Exception:
args = str(args)
return {
"id": tc.get("id"),
"type": tc.get("type", "function"),
"function": {"name": name, "arguments": args},
}
def _summarize_tool_calls_for_output(tool_calls) -> str:
"""Render a tool_calls list as a compact JSON string for OUTPUT_VALUE.
Best-effort: returns ``str(tool_calls)`` if anything unexpected happens
so OUTPUT_VALUE is never blanked on a malformed payload.
"""
try:
normalized = [n for n in (_normalize_tool_call(tc) for tc in tool_calls) if n]
return json.dumps({"tool_calls": normalized})
except Exception:
return str(tool_calls)
def _emit_message_tool_calls(span: "Span", prefix: str, message) -> None:
"""Emit ``MESSAGE_TOOL_CALLS.*`` for an assistant message that requested
tool calls. Pure addition: only writes when ``tool_calls`` is non-empty.
Accepts dicts or Pydantic message objects (e.g. ``litellm.Message``); the
same applies to each tool_call entry.
"""
tool_calls = _get_tool_calls(message)
if not tool_calls:
return
for tc_idx, raw_tc in enumerate(tool_calls):
tc = _normalize_tool_call(raw_tc)
if tc is None:
continue
tc_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tc_idx}"
if tc["id"]:
safe_set_attribute(
span, f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tc["id"]
)
fn = tc["function"]
if fn["name"]:
safe_set_attribute(
span,
f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}",
fn["name"],
)
if fn["arguments"] is not None:
safe_set_attribute(
span,
f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}",
fn["arguments"],
)
def _emit_input_message_extras(span: "Span", prefix: str, message: dict) -> None:
"""Emit additive attributes for an input message:
- `MESSAGE_NAME` and `MESSAGE_TOOL_CALL_ID` (commonly set on tool-result
messages so traces show which tool produced which result).
- `MESSAGE_TOOL_CALLS.*` when an assistant message requested tools.
- `MESSAGE_CONTENTS.*` structured content for list-shaped content
(multimodal text + image parts). The plain `MESSAGE_CONTENT` write is
still performed by the caller, so renderers that only read the legacy
key continue to work.
"""
if not isinstance(message, dict):
return
name = message.get("name")
if name:
safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_NAME}", name)
tool_call_id = message.get("tool_call_id")
if tool_call_id:
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALL_ID}",
tool_call_id,
)
_emit_message_tool_calls(span, prefix, message)
content = message.get("content")
if isinstance(content, list):
contents_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}"
for part_idx, part in enumerate(content):
if not isinstance(part, dict):
continue
part_prefix = f"{contents_prefix}.{part_idx}"
part_type = part.get("type")
if part_type in ("text", "input_text"):
text = part.get("text")
if isinstance(text, str):
safe_set_attribute(
span,
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}",
"text",
)
safe_set_attribute(
span,
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}",
text,
)
elif part_type in ("image_url", "image", "input_image"):
url = None
image = part.get("image_url")
if isinstance(image, dict):
url = image.get("url")
elif isinstance(image, str):
url = image
if not url:
# Anthropic-style source.{type=base64,media_type,data}
source = part.get("source")
if isinstance(source, dict) and source.get("data"):
media_type = source.get("media_type", "image/jpeg")
url = f"data:{media_type};base64,{source['data']}"
elif isinstance(part.get("url"), str):
url = part["url"]
if url:
safe_set_attribute(
span,
f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}",
"image",
)
safe_set_attribute(
span,
f"{part_prefix}.message_content.image.image.url",
url,
)
def _set_session_and_user_attrs(
span: "Span", kwargs: dict, standard_logging_payload
) -> None:
"""Emit `SESSION_ID` / `USER_ID` / team metadata when source data exists.
`SESSION_ID` is emitted only when an explicit end-user identifier exists
(`metadata.user_api_key_end_user_id`). We deliberately do NOT fall back
to `trace_id`, because that would create a distinct "session" for every
single request and distort Arize's Session-grouping analytics. The
`trace_id` is still emitted under its own `litellm.trace_id` key so
spans remain filterable by trace.
USER_ID is *only* emitted when no upstream path (model_params.user or
optional_params.user) has already set it, to avoid overwriting an
existing value with a possibly-different one from API-key metadata.
"""
if not isinstance(standard_logging_payload, dict):
return
metadata = standard_logging_payload.get("metadata") or {}
if not isinstance(metadata, dict):
return
session_id = metadata.get("user_api_key_end_user_id")
if session_id:
safe_set_attribute(span, SpanAttributes.SESSION_ID, str(session_id))
trace_id = standard_logging_payload.get("trace_id")
if trace_id:
safe_set_attribute(span, "litellm.trace_id", str(trace_id))
optional_params = kwargs.get("optional_params") or {}
model_params = standard_logging_payload.get("model_parameters") or {}
has_user_already = bool(
(isinstance(optional_params, dict) and optional_params.get("user"))
or (isinstance(model_params, dict) and model_params.get("user"))
)
if not has_user_already:
user_id = metadata.get("user_api_key_user_id")
if user_id:
safe_set_attribute(span, SpanAttributes.USER_ID, str(user_id))
team_id = metadata.get("user_api_key_team_id")
if team_id:
safe_set_attribute(span, "litellm.team_id", str(team_id))
team_alias = metadata.get("user_api_key_team_alias")
if team_alias:
safe_set_attribute(span, "litellm.team_alias", str(team_alias))
key_alias = metadata.get("user_api_key_alias")
if key_alias:
safe_set_attribute(span, "litellm.key_alias", str(key_alias))
def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None:
"""Emit cost attributes from the StandardLoggingPayload when present.
Uses the OpenInference `llm.cost.total` key so Arize / Phoenix can
surface the cost in their "Total Cost" column. LiteLLM only tracks a
single total in `StandardLoggingPayload.response_cost`, so we cannot
split it into prompt/completion. We also keep the legacy
`llm.response.cost` key for back-compat with any consumer querying it.
"""
if not isinstance(standard_logging_payload, dict):
return
cost = standard_logging_payload.get("response_cost")
if cost is None:
return
try:
cost_value = float(cost)
except (TypeError, ValueError):
return
safe_set_attribute(span, "llm.cost.total", cost_value)
safe_set_attribute(span, "llm.response.cost", cost_value)
def _is_passthrough_call_type(call_type: Optional[str]) -> bool:
if not call_type:
return False
lowered = str(call_type).lower()
return "passthrough" in lowered or "pass_through" in lowered
def _maybe_normalize_passthrough(
span: "Span",
kwargs: dict,
raw_response_obj,
coerced_response_obj,
standard_logging_payload,
) -> None:
"""Surface input/output text for passthrough routes (e.g. Bedrock
InvokeModel) so the parent span renders as more than `usage` numbers.
Only runs when `call_type` is a passthrough variant. Reads from:
- `kwargs["additional_args"]["complete_input_dict"]` for input
- the coerced response (or `kwargs["original_response"]`) for output
All emits are best-effort: if the provider shape isn't recognized the
helper exits silently. Existing chat/completion paths never enter this
helper because their call_type doesn't contain "passthrough".
TEMPORARY BRIDGE: passthrough handlers don't populate the
StandardLoggingPayload `messages` field today (they call
`transform_response(messages=[])`), so the input is only available via
`additional_args.complete_input_dict`. The proper fix is upstream in
`base_passthrough_logging_handler._create_response_logging_payload()`:
once that populates SLP `messages`/`response`, every callback gets
passthrough I/O (with central redaction) for free and this helper's
`complete_input_dict` fallback can be deleted. See follow-up issue.
"""
call_type = (
standard_logging_payload.get("call_type")
if isinstance(standard_logging_payload, dict)
else None
)
if not _is_passthrough_call_type(call_type):
return
# Respect LiteLLM's central message-redaction contract. The normal
# chat/completion path is redacted by `perform_redaction` before
# callbacks run, but `complete_input_dict` (read below) is NOT covered by
# that layer — so without this gate, an operator who enabled redaction
# would still see raw passthrough prompts in Arize. Skip entirely when
# redaction is on so neither input nor output leaks through this bridge.
if should_redact_message_logging(kwargs):
return
# --- INPUT --------------------------------------------------------------
additional_args = kwargs.get("additional_args") or {}
complete_input_dict = (
additional_args.get("complete_input_dict")
if isinstance(additional_args, dict)
else None
)
if isinstance(complete_input_dict, dict):
_set_passthrough_input_attributes(span, complete_input_dict.get("messages"))
# --- OUTPUT -------------------------------------------------------------
parsed_response = _parse_passthrough_response(
raw_response_obj, coerced_response_obj, kwargs
)
if not isinstance(parsed_response, dict):
return
_set_passthrough_output_attributes(span, parsed_response)
def _set_passthrough_input_attributes(span: "Span", messages) -> None:
"""Render passthrough request messages into INPUT_VALUE + LLM_INPUT_MESSAGES."""
if not (isinstance(messages, list) and messages):
return
# Set INPUT_VALUE from the last user message text if discoverable.
last_text = None
for msg in reversed(messages):
if isinstance(msg, dict):
last_text = _coerce_text(msg.get("content"))
if last_text:
break
if last_text:
safe_set_attribute(span, SpanAttributes.INPUT_VALUE, last_text)
# Mirror messages into LLM_INPUT_MESSAGES so the input pane renders.
for idx, msg in enumerate(messages):
if not isinstance(msg, dict):
continue
prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}"
role = msg.get("role")
if role:
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
role,
)
text = _coerce_text(msg.get("content"))
if text is not None:
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
text,
)
def _set_passthrough_output_attributes(span: "Span", parsed_response: dict) -> None:
"""Render passthrough response into OUTPUT_VALUE + LLM_OUTPUT_MESSAGES."""
# Anthropic / Bedrock-Anthropic: `content` is a list of typed parts.
content_list = parsed_response.get("content")
if isinstance(content_list, list) and content_list:
texts = []
for part in content_list:
if isinstance(part, dict) and isinstance(part.get("text"), str):
texts.append(part["text"])
joined = "\n\n".join(t for t in texts if t)
if joined:
safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, joined)
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0"
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
parsed_response.get("role", "assistant"),
)
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
joined,
)
# OpenAI-style passthrough: `choices[0].message.content`
choices = parsed_response.get("choices")
if isinstance(choices, list) and choices:
first = choices[0]
if isinstance(first, dict):
msg = first.get("message")
if isinstance(msg, dict):
text = _coerce_text(msg.get("content"))
if text:
safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, text)
prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0"
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
msg.get("role", "assistant"),
)
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
text,
)
def _parse_passthrough_response(raw_response_obj, coerced_response_obj, kwargs):
"""Return a dict view of the provider response for passthrough routes."""
# Prefer the coerced view (already JSON-parsed for httpx.Response).
candidates = []
if isinstance(coerced_response_obj, dict):
candidates.append(coerced_response_obj)
if (
isinstance(raw_response_obj, dict)
and raw_response_obj is not coerced_response_obj
):
candidates.append(raw_response_obj)
for candidate in candidates:
# StandardPassThroughResponseObject wrapper: {"response": "..."}.
if (
"response" in candidate
and "content" not in candidate
and "choices" not in candidate
):
inner = candidate.get("response")
if isinstance(inner, str):
try:
parsed = json.loads(inner)
if isinstance(parsed, dict):
return parsed
except Exception:
continue
if isinstance(inner, dict):
return inner
else:
return candidate
# Fallback: kwargs["original_response"] from the OTel base path.
original = kwargs.get("original_response") if isinstance(kwargs, dict) else None
if isinstance(original, dict):
return original
if isinstance(original, str):
try:
parsed = json.loads(original)
if isinstance(parsed, dict):
return parsed
except Exception:
return None
return None

10
litellm/integrations/custom_guardrail.py
View file

@ -662,6 +662,16 @@ class CustomGuardrail(CustomLogger):
request_data["metadata"] = {}
_append_guardrail_info(request_data["metadata"])
# Emit the otel guardrail span here, where every guardrail execution lands,
# rather than relying on a post-call hook that does not fire on every path
# (e.g. a pass-through request that passes its guardrails).
try:
from litellm.integrations.otel.logger import emit_guardrail_span
emit_guardrail_span(slg)
except Exception:
pass
async def apply_guardrail(
self,
inputs: GenericGuardrailAPIInputs,

97
litellm/integrations/datadog/datadog.py
View file

@ -41,6 +41,7 @@ from litellm.integrations.datadog.datadog_handler import (
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.llms.custom_httpx.http_handler import (
MaskedHTTPStatusError,
_get_httpx_client,
get_async_httpx_client,
httpxSpecialProvider,
@ -68,6 +69,22 @@ DD_LOGGED_SUCCESS_SERVICE_TYPES = [
]
def _resolve_dd_batch_size() -> int:
raw = os.getenv("DD_BATCH_SIZE")
if raw is None:
return DD_MAX_BATCH_SIZE
try:
value = int(raw)
except ValueError:
verbose_logger.warning(
"Datadog: ignoring invalid DD_BATCH_SIZE=%r, using %s",
raw,
DD_MAX_BATCH_SIZE,
)
return DD_MAX_BATCH_SIZE
return max(1, min(value, DD_MAX_BATCH_SIZE))
class DataDogLogger(
CustomBatchLogger,
AdditionalLoggingUtils,
@ -128,7 +145,9 @@ class DataDogLogger(
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
super().__init__(
**kwargs, flush_lock=self.flush_lock, batch_size=DD_MAX_BATCH_SIZE
**kwargs,
flush_lock=self.flush_lock,
batch_size=_resolve_dd_batch_size(),
)
except Exception as e:
verbose_logger.exception(
@ -339,28 +358,14 @@ class DataDogLogger(
"[DATADOG MOCK] Mock mode enabled - API calls will be intercepted"
)
response = await self.async_send_compressed_data(batch_to_send)
if response.status_code == 413:
verbose_logger.exception(DD_ERRORS.DATADOG_413_ERROR.value)
self.log_queue = batch_to_send + self.log_queue
return
response.raise_for_status()
if response.status_code != 202:
raise Exception(
f"Response from datadog API status_code: {response.status_code}, text: {response.text}"
)
undelivered = await self._send_with_413_split(batch_to_send)
if undelivered:
self.log_queue = undelivered + self.log_queue
if self.is_mock_mode:
verbose_logger.debug(
f"[DATADOG MOCK] Batch of {len(batch_to_send)} events successfully mocked"
)
else:
verbose_logger.debug(
"Datadog: Response from datadog API status_code: %s, text: %s",
response.status_code,
response.text,
)
except Exception as e:
self.log_queue = batch_to_send + self.log_queue
@ -368,6 +373,62 @@ class DataDogLogger(
f"Datadog Error sending batch API - {str(e)}\n{traceback.format_exc()}"
)
async def _send_with_413_split(self, batch: List) -> List:
"""
Send a batch, halving any sub-batch that 413s (payload too large) and retrying the
halves, since Datadog enforces a 5MB uncompressed limit per request.
A 413 surfaces as a raised MaskedHTTPStatusError (httpx raise_for_status), not a
returned response, so both paths are handled. A lone event that still 413s is
dropped to avoid wedging the queue on an undeliverable payload. Returns the events
that could not be delivered because of a non-413 (transient) error, so the caller
re-queues only those and never the events already accepted by Datadog.
"""
pending: List[List] = [batch]
while pending:
chunk = pending.pop()
if not chunk:
continue
try:
response = await self.async_send_compressed_data(chunk)
except Exception as e:
if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413:
response = e.response
else:
verbose_logger.exception(
f"Datadog Error sending batch API - {str(e)}"
)
return self._undelivered(chunk, pending)
if response.status_code == 413:
if len(chunk) == 1:
verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value)
continue
mid = len(chunk) // 2
pending.append(chunk[mid:])
pending.append(chunk[:mid])
continue
if response.status_code != 202:
verbose_logger.error(
"Datadog: unexpected response status_code=%s, text=%s",
response.status_code,
response.text,
)
return self._undelivered(chunk, pending)
verbose_logger.debug(
"Datadog: delivered %s events, status_code=%s, text=%s",
len(chunk),
response.status_code,
response.text,
)
return []
@staticmethod
def _undelivered(chunk: List, pending: List[List]) -> List:
return chunk + [event for remaining in reversed(pending) for event in remaining]
async def flush_queue(self):
if self.flush_lock is None:
return

145
litellm/integrations/datadog/datadog_cost_management.py
View file

@ -2,10 +2,17 @@ import asyncio
import os
import time
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple, cast
from litellm._logging import verbose_logger
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.datadog.datadog_handler import (
get_datadog_env,
get_datadog_hostname,
get_datadog_pod_name,
get_datadog_service,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -15,9 +22,30 @@ from litellm.types.integrations.datadog_cost_management import (
)
from litellm.types.utils import StandardLoggingPayload
# Reserved tag keys whose values come from trusted sources (infra env, LiteLLM
# core payload fields, or proxy-controlled auth metadata). User-supplied
# request_tags / metadata cannot overwrite these, even when the key is
# allowlisted via cost_tag_keys, because that would let an authenticated caller
# spoof cost attribution (e.g. request_tags=["team:victim-team"]).
_RESERVED_TAG_KEYS: frozenset = frozenset(
{
"env",
"service",
"host",
"pod_name",
"provider",
"model",
"model_id",
"team",
"user",
"model_group",
}
)
class DatadogCostManagementLogger(CustomBatchLogger):
def __init__(self, **kwargs):
def __init__(self, cost_tag_keys: Optional[List[str]] = None, **kwargs):
self.cost_tag_keys: List[str] = list(cost_tag_keys) if cost_tag_keys else []
self.dd_api_key = os.getenv("DD_API_KEY")
self.dd_app_key = os.getenv("DD_APP_KEY")
self.dd_site = os.getenv("DD_SITE", "datadoghq.com")
@ -68,20 +96,21 @@ class DatadogCostManagementLogger(CustomBatchLogger):
if not self.log_queue:
return
batch_to_send = self.log_queue[:]
self.log_queue = []
try:
# Aggregate costs from the batch
aggregated_entries = self._aggregate_costs(self.log_queue)
aggregated_entries = self._aggregate_costs(batch_to_send)
if not aggregated_entries:
verbose_logger.debug(
"Datadog Cost Management: batch produced no aggregable entries; "
"dropping %d log(s) from queue.",
len(batch_to_send),
)
return
# Send to Datadog
await self._upload_to_datadog(aggregated_entries)
# Clear queue only on success (or if we decide to drop on failure)
# CustomBatchLogger clears queue in flush_queue, so we just process here
except Exception as e:
self.log_queue = batch_to_send + self.log_queue
verbose_logger.exception(
f"Datadog Cost Management: Error in async_send_batch: {str(e)}"
)
@ -151,45 +180,81 @@ class DatadogCostManagementLogger(CustomBatchLogger):
return list(aggregator.values())
def _extract_tags(self, log: StandardLoggingPayload) -> Dict[str, str]:
from litellm.integrations.datadog.datadog_handler import (
get_datadog_env,
get_datadog_hostname,
get_datadog_pod_name,
get_datadog_service,
)
tags = {
tags: Dict[str, str] = {
"env": get_datadog_env(),
"service": get_datadog_service(),
"host": get_datadog_hostname(),
"pod_name": get_datadog_pod_name(),
}
# Add metadata as tags
metadata = log.get("metadata", {})
if metadata:
# Add user info
# Add user info
if metadata.get("user_api_key_alias"):
tags["user"] = str(metadata["user_api_key_alias"])
# Always-on canonical FOCUS dimensions from top-level payload fields.
# Non-sensitive and required for Datadog Custom Costs per-model attribution.
self._add_tag(tags, "provider", log.get("custom_llm_provider"))
self._add_tag(tags, "model", log.get("model"))
self._add_tag(tags, "model_id", log.get("model_id"))
# Add Team Tag
team_tag = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias") # type: ignore
or metadata.get("user_api_key_team_id")
or metadata.get("team_id") # type: ignore
)
# cast because StandardLoggingMetadata is a TypedDict; we iterate it
# as a generic mapping below.
metadata: Dict[str, Any] = cast(Dict[str, Any], log.get("metadata") or {})
if team_tag:
tags["team"] = str(team_tag)
# model_group is not in StandardLoggingMetadata TypedDict, so we need to access it via dict.get()
model_group = metadata.get("model_group") # type: ignore[misc]
if model_group:
tags["model_group"] = str(model_group)
# Backwards-compat: team/user/model_group preserved regardless of allowlist.
if metadata.get("user_api_key_alias"):
tags["user"] = str(metadata["user_api_key_alias"])
team_tag = (
metadata.get("user_api_key_team_alias")
or metadata.get("team_alias")
or metadata.get("user_api_key_team_id")
or metadata.get("team_id")
)
if team_tag:
tags["team"] = str(team_tag)
if metadata.get("model_group"):
tags["model_group"] = str(metadata["model_group"])
# Allowlist-gated: request_tags (split on `:`) and arbitrary metadata.*.
# Reserved keys are hard-blocked here regardless of allowlist membership —
# see _RESERVED_TAG_KEYS for the rationale.
if self.cost_tag_keys:
allow = set(self.cost_tag_keys)
for rt in log.get("request_tags") or []:
if not isinstance(rt, str) or ":" not in rt:
continue
k, _, v = rt.partition(":")
if k in allow and v:
self._set_custom_tag(tags, k, v)
for k, v in metadata.items():
if k in allow and v is not None and not isinstance(v, (dict, list)):
self._set_custom_tag(tags, k, str(v))
for nested_key in ("spend_logs_metadata", "requester_metadata"):
nested = metadata.get(nested_key)
if isinstance(nested, dict):
for k, v in nested.items():
if (
k in allow
and v is not None
and not isinstance(v, (dict, list))
):
self._set_custom_tag(tags, k, str(v))
return tags
@staticmethod
def _set_custom_tag(tags: Dict[str, str], key: str, value: str) -> None:
if key in _RESERVED_TAG_KEYS:
verbose_logger.debug(
"Datadog Cost Management: dropping user-supplied tag %r=%r — "
"key is reserved for trusted cost attribution.",
key,
value,
)
return
tags[key] = value
@staticmethod
def _add_tag(tags: Dict[str, str], key: str, value: Any) -> None:
if value:
tags[key] = str(value)
async def _upload_to_datadog(self, payload: List[Dict]):
if not self.dd_api_key or not self.dd_app_key:
return
@ -201,8 +266,6 @@ class DatadogCostManagementLogger(CustomBatchLogger):
}
# The API endpoint expects a list of objects directly in the body (file content behavior)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
data_json = safe_dumps(payload)
response = await self.async_client.put(

8
litellm/integrations/focus/transformer.py
View file

@ -95,7 +95,9 @@ class FocusTransformer:
pl.lit("Usage-Based").alias("ChargeFrequency"),
fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"),
fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"),
dec(pl.lit(1.0)).alias("ConsumedQuantity"),
dec(
pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0)
).alias("ConsumedQuantity"),
pl.lit("Requests").alias("ConsumedUnit"),
dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"),
none_str.alias("ContractedUnitPrice"),
@ -107,7 +109,9 @@ class FocusTransformer:
none_str.alias("AvailabilityZone"),
pl.lit("USD").alias("PricingCurrency"),
none_str.alias("PricingCategory"),
dec(pl.lit(1.0)).alias("PricingQuantity"),
dec(
pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0)
).alias("PricingQuantity"),
none_dec.alias("PricingCurrencyContractedUnitPrice"),
dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"),
none_dec.alias("PricingCurrencyListUnitPrice"),

148
litellm/integrations/opentelemetry.py
View file

@ -64,6 +64,9 @@ HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE = "http.response.status_code"
HTTP_ROUTE_ATTRIBUTE = "http.route"
URL_PATH_ATTRIBUTE = "url.path"
PREPROCESSING_DURATION_MS_ATTRIBUTE = "litellm.preprocessing.duration_ms"
TEAM_METADATA_ATTRIBUTE = "litellm.team.metadata"
MODEL_GROUP_ATTRIBUTE = "litellm.model_group"
PROVIDER_MODEL_ATTRIBUTE = "litellm.provider.model"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -80,6 +83,19 @@ _VALID_CAPTURE_MODES = {
}
def _normalize_team_metadata_keys(value: Any) -> List[str]:
"""Coerce a team-metadata allowlist from a list or comma-separated string.
config.yaml passes a YAML list; an env var passes a comma-separated string.
Both collapse to a list of stripped, non-empty keys.
"""
if value is None:
return []
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
return [str(item).strip() for item in value if str(item).strip()]
@dataclass
class OpenTelemetryConfig:
exporter: Union[str, SpanExporter] = "console"
@ -97,6 +113,10 @@ class OpenTelemetryConfig:
# One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
capture_message_content: Optional[str] = None
semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set)
# Sub-keys of the team's free-form metadata stamped onto the inference span
# under ``litellm.team.metadata``. Empty by default so none of a team's
# metadata leaves the process until explicitly allowlisted.
baggage_team_metadata_keys: List[str] = field(default_factory=list)
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@ -127,6 +147,11 @@ class OpenTelemetryConfig:
self.semconv_stability_opt_in |= parse_semconv_opt_in(
os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV)
)
self.baggage_team_metadata_keys = _normalize_team_metadata_keys(
self.baggage_team_metadata_keys
) or _normalize_team_metadata_keys(
os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS")
)
@classmethod
def from_env(cls):
@ -185,8 +210,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
meter_provider: Optional[Any] = None,
**kwargs,
):
team_metadata_keys_override = kwargs.pop("baggage_team_metadata_keys", None)
if config is None:
config = OpenTelemetryConfig.from_env()
if team_metadata_keys_override is not None:
config.baggage_team_metadata_keys = _normalize_team_metadata_keys(
team_metadata_keys_override
)
self.config = config
self.callback_name = callback_name
@ -982,6 +1012,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
litellm_params = kwargs.get("litellm_params", {}) or {}
_metadata = litellm_params.get("metadata", {}) or {}
proxy_span = _metadata.get("litellm_parent_otel_span", None)
# Fallback: check litellm_metadata (used by /v1/messages and other
# LITELLM_METADATA_ROUTES).
if proxy_span is None:
_litellm_metadata = litellm_params.get("litellm_metadata", {}) or {}
proxy_span = _litellm_metadata.get("litellm_parent_otel_span", None)
if (
proxy_span is not None
and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME
@ -1213,6 +1250,74 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
):
self._set_team_attributes_from_kwargs(proxy_span, kwargs)
def _set_inference_identity_attributes(
self,
span: Span,
standard_logging_payload: StandardLoggingPayload,
litellm_params: dict,
) -> None:
"""Stamp request-identity attributes onto an inference span so every
LLM-call span is filterable by the route it came in on, the team's
metadata, and both the user-facing (model_group alias) and the
dispatched (provider) model names. Empty/absent values are skipped.
"""
metadata = standard_logging_payload.get("metadata") or {}
http_route = metadata.get("user_api_key_request_route")
if http_route:
self.safe_set_attribute(
span=span, key=HTTP_ROUTE_ATTRIBUTE, value=http_route
)
# ``user_api_key_team_metadata`` is dropped from the standard logging
# payload metadata, so read it from the raw request metadata in kwargs.
# ``metadata`` and ``litellm_metadata`` are alternate names for the same
# full metadata dict (the name varies by endpoint), so first-truthy wins.
raw_metadata = (
litellm_params.get("metadata")
or litellm_params.get("litellm_metadata")
or {}
)
team_metadata = self._team_metadata_json(
raw_metadata.get("user_api_key_team_metadata"),
self.config.baggage_team_metadata_keys,
)
if team_metadata:
self.safe_set_attribute(
span=span, key=TEAM_METADATA_ATTRIBUTE, value=team_metadata
)
model_group = standard_logging_payload.get("model_group")
if model_group:
self.safe_set_attribute(
span=span, key=MODEL_GROUP_ATTRIBUTE, value=model_group
)
hidden_params = standard_logging_payload.get("hidden_params") or {}
provider_model = hidden_params.get(
"litellm_model_name"
) or standard_logging_payload.get("model")
if provider_model:
self.safe_set_attribute(
span=span, key=PROVIDER_MODEL_ATTRIBUTE, value=provider_model
)
@staticmethod
def _team_metadata_json(value: Any, allowed_keys: List[str]) -> Optional[str]:
"""JSON-serialize only the allowlisted sub-keys of a team's metadata.
Returns ``None`` when nothing is allowlisted or no allowlisted key is
present, so the empty case is dropped rather than stamping a useless
``"{}"`` (and so a team's metadata never leaves the process until an
operator opts each sub-key in via ``baggage_team_metadata_keys``).
"""
if not isinstance(value, dict) or not value or not allowed_keys:
return None
filtered = {key: value[key] for key in allowed_keys if key in value}
if not filtered:
return None
return safe_dumps(filtered)
def _record_metrics(self, kwargs, response_obj, start_time, end_time):
duration_s = (end_time - start_time).total_seconds()
params = kwargs.get("litellm_params") or {}
@ -2023,6 +2128,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key="hidden_params",
value=safe_dumps(hidden_params),
)
self._set_inference_identity_attributes(
span=span,
standard_logging_payload=standard_logging_payload,
litellm_params=litellm_params,
)
# Cost breakdown tracking
cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get(
"cost_breakdown"
@ -2564,6 +2675,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
def _to_ns(self, dt):
if dt is None:
return int(datetime.now().timestamp() * 1e9)
if isinstance(dt, (int, float)):
return int(dt * 1e9)
return int(dt.timestamp() * 1e9)
def _get_span_name(self, kwargs):
@ -2610,6 +2725,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
_metadata = litellm_params.get("metadata", {}) or {}
parent_otel_span = _metadata.get("litellm_parent_otel_span", None)
# Fallback: check litellm_metadata (used by /v1/messages and other
# LITELLM_METADATA_ROUTES that store proxy-internal metadata
# separately from the provider's native "metadata" field).
if parent_otel_span is None:
_litellm_metadata = litellm_params.get("litellm_metadata", {}) or {}
parent_otel_span = _litellm_metadata.get("litellm_parent_otel_span", None)
# Priority 1: Explicit parent span from metadata
if parent_otel_span is not None:
verbose_logger.debug(
@ -3183,6 +3305,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=int(status_code),
)
def record_error_attributes_on_span(
self,
span: Optional[Span],
exception: Optional[Exception],
status_code: int,
) -> None:
"""Stamp structured ``error.*`` attributes on the SERVER span from the
exception returned to the client, with ``error.code`` pinned to the real
response status. Idempotent (overwrites); emits no exception event."""
if span is None or exception is None:
return
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
error_information = StandardLoggingPayloadSetup.get_error_information(
original_exception=exception
)
error_information["error_code"] = str(status_code)
self._record_exception_on_span(
span=span,
kwargs={
"standard_logging_object": {"error_information": error_information}
},
)
def set_preprocessing_duration_attribute(
self, span: Optional[Span], container: Any
) -> None:

24
litellm/integrations/opik/opik_payload_builder/extractors.py
View file

@ -39,20 +39,32 @@ def extract_opik_metadata(
standard_logging_metadata: Dict[str, Any],
) -> Dict[str, Any]:
"""
Extract and merge Opik metadata from request and requester.
Merge Opik metadata from three sources in increasing priority order:
1. user_api_key_auth_metadata– lowest priority (operator-level defaults)
2. litellm_metadata (request)– overrides auth-key defaults
3. requester_metadata – highest priority (e.g. proxy header overrides)
Args:
litellm_metadata: Metadata from litellm_params
standard_logging_metadata: Metadata from standard_logging_object
litellm_metadata: Metadata from litellm_params.mak
standard_logging_metadata: Metadata from standard_logging_object.
Returns:
Merged Opik metadata dictionary
Merged Opik metadata dictionary.
"""
opik_meta = litellm_metadata.get("opik", {}).copy()
# Start with auth-key defaults (lowest priority).
auth_meta = standard_logging_metadata.get("user_api_key_auth_metadata") or {}
opik_meta = (auth_meta.get("opik") or {}).copy()
# Request-level values override auth-key defaults.
request_opik = litellm_metadata.get("opik") or {}
opik_meta.update(request_opik)
# Requester-level values win over everything else.
requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {}
requester_opik = requester_metadata.get("opik", {}) or {}
opik_meta.update(requester_opik)
if requester_opik:
opik_meta.update(requester_opik)
_logging.verbose_logger.debug(
f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}"

261
litellm/integrations/otel/README.md Normal file
View file

@ -0,0 +1,261 @@
# OpenTelemetry instrumentation
This package produces OpenTelemetry traces for LiteLLM. It is enabled by the
`LITELLM_OTEL_V2` environment variable (`is_otel_v2_enabled()` in
[`config.py`](./model/config.py)); when unset, nothing in this package runs.
## What gets traced
A traced proxy request produces one trace with two kinds of spans:
```
SERVER span "POST /v1/chat/completions" ← FastAPI instrumentation
├── INTERNAL span "auth /v1/chat/completions" ← auth phase ┐
│ ├── CLIENT span "postgres get_key_object" ← datastore call │
│ └── CLIENT span "postgres get_team_membership" │
├── INTERNAL span "execute_guardrail …" ← guardrail │ this package
├── CLIENT span "chat gpt-4o" ← LLM call │
└── CLIENT span "batch_write_to_db …" ← spend write ┘
```
The gen-ai spans are siblings under the server span. In particular the guardrail
span is a sibling of the LLM call, not a child of it: pre/during/post-call
guardrail hooks are part of the request lifecycle (a pre-call guardrail runs
before the LLM call even starts), so they belong directly under the server span,
alongside the LLM call.
Request-level spans (LLM call, guardrail) parent to the server span via an
**explicit anchor** — `context.set_request_root_span` captures the server span
once at request entry, and `resolve_request_span_context` reads it — rather than
to whatever span is momentarily active. Ambient-only parenting was wrong at two
boundaries: inside the live `auth` phase span the active span is `auth` (so the
span would nest under auth), and a pass-through request closes its span from a
detached `asyncio.create_task` where the server span is no longer active (so the
span orphaned into its own trace). The anchor — a contextvar inherited by those
child tasks — gives a stable parent in both cases. DB/service spans keep ambient
parenting so an auth DB lookup still nests under `auth`.
**Which service calls become spans (`spans.span_role_for_service`).** LiteLLM's
service-logging layer instruments many internal functions, but only some are
traceable units of work:
- **`DB_CALL` (CLIENT)** — outbound datastore calls (redis, postgres,
`batch_write_to_db`), carrying `db.system.name` / `db.operation.name` semconv.
- **`SERVICE` (INTERNAL)** — genuine internal work worth a span (background
budget/reset jobs, pod-lock manager).
- **metrics-only (no span)** — `self` (the `track_llm_api_timing` wrapper, which
duplicates the LLM-call span), `router` (duplicates the request), and
`proxy_pre_call` (a guardrail's real span is `execute_guardrail …`). These
still feed Prometheus/Datadog through their own hooks; they just never enter
the trace. `auth` is also excluded here because it gets a **live phase span**
instead (see below).
Spans are named `"{service} {call_type}"` (e.g. `"redis set"`) so repeated calls
to one service stay distinguishable. Like every other span they parent to the
**ambient** context, falling back to the threaded `litellm_parent_otel_span` only
when ambient has no live span; a background job with neither starts its own root
trace. Caller-supplied `event_metadata` is **sanitized** before it reaches a span
(primitives only, no live objects, no secrets/headers, bounded) — see
`payloads.sanitize_event_metadata`.
**Live phase spans.** `auth` is wrapped in a real, active span
(`logger.phase_span`) for the duration of authentication, so the DB lookups it
triggers nest **under** it instead of flattening onto the server span. Identity
Baggage (team/key/user) is seeded once the key resolves, so every post-auth span
inherits it; auth-internal DB lookups that run before the key is known stay
unlabeled, which is correct.
**Status.** On success a span's status is left `UNSET` (the semconv default,
matching the FastAPI server span); only a genuine error sets `ERROR`.
- **Server spans** (one per HTTP route) are created by the
`opentelemetry-instrumentation-fastapi` package. It stamps `http.*` attributes
and extracts inbound `traceparent` headers. This package does **not** create
or modify server spans — request routes never touch spans.
- **Gen-AI spans** (LLM calls, guardrails, internal service calls) are created
by this package from LiteLLM's logging callbacks. Request-level spans parent to
the server span via the captured anchor; DB/service spans parent to the active
span (ambient) so they nest under the request phase that triggered them.
Both kinds share a single `TracerProvider`, so they belong to the same trace
and export through the same configured exporters. FastAPI middleware can only be
added before the app starts serving, so the app is instrumented at
import time **without** a provider — it binds to the OTel global
`ProxyTracerProvider`. Once config (and the callbacks) is loaded, the proxy
publishes the chosen logger's `TracerProvider` as the global via
`trace.set_tracer_provider(...)`, and the server spans delegate to it. When a
preset callback (`arize`, `langfuse_otel`, …) is configured, its provider
becomes the global, so server spans export to that backend too.
## How a request flows
1. **App creation** (`proxy_server` import): when the gate is on,
`mount.instrument_fastapi_app(app)` calls `FastAPIInstrumentor.instrument_app`
with no provider (the middleware stack is frozen once the app serves, so this
can't wait for startup). It binds to the OTel global `ProxyTracerProvider`. Noisy
non-LLM routes are excluded by default (`mount._DEFAULT_EXCLUDED_ROUTES`): health
checks (`/health*`), the Prometheus scrape (`/metrics`), and static UI/docs assets
(`/litellm-asset-prefix`, `/_next`, `/ui`, `/swagger`, `/docs`, `/redoc`,
`/openapi.json`, favicons, `/.well-known`) — so load-balancer polling, metric
scrapes, and asset fetches don't flood traces. Entries are substring-matched, so
`/metrics` also drops the `/model/metrics` admin-analytics spans. Set
`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` to override the whole set (e.g. `""` to trace
everything, or your own comma-separated path list).
2. **Startup** (`proxy_server.proxy_startup_event`): after the config (and
callbacks) is loaded, the already-registered preset `OpenTelemetryV2` logger
is reused — or a generic one reading `OTEL_*` envs is built when no preset is
configured — and its `TracerProvider` is published as the OTel global with
`trace.set_tracer_provider(...)`. The proxy tracer then delegates to it, so
server spans and gen-ai spans share one provider and the same trace.
3. **Request**: the FastAPI instrumentation starts the server span and makes it
the active context for the request task. The proxy's first call into the V2
logger (`create_litellm_proxy_request_started_span`, at the auth boundary)
**captures it as the request anchor** (`set_request_root_span`), so every later
request-level span has a stable explicit parent regardless of what is active
when it emits.
4. **LLM call span (born at the boundary)**: `OpenTelemetryV2.log_pre_api_call`
runs synchronously in the request task, just before the upstream call, and
**opens** the LLM-call span there, parented to the anchored server span
(`resolve_request_span_context`). The open span is held in a bounded cache keyed
by `litellm_call_id` (a primitive the callback kwargs carry at both `pre_call`
and close), so no live `Span` ever travels through a `litellm_params` metadata
dict. For the boundary hook to fire at all, the logger is registered into
`litellm.input_callback` — the list `Logging.pre_call` iterates. The async
success/failure callback later
**closes** it: it builds an `LLMCallSpanData` from the typed
`standard_logging_object` (token usage and cost are computed only by then),
stamps the attributes, sets status, and ends the span. The sync callback is a
no-op (closing is async-only). When `pre_call` runs off the request task — a
sync-only provider driven through a thread pool, where contextvars (and so the
anchor) don't follow — no parent is visible there, so creation is **deferred**
to the async callback, whose worker context was copied from the request task at
enqueue and so still carries the anchor. **Pass-through** endpoints call
`logging_obj.pre_call` in the request task too, then close from a detached
`asyncio.create_task`; the anchor (not the by-then-inactive server span) keeps
their LLM-call span in the request's trace. `pre_call` is litellm's generic
"log the attempt" hook, so it also fires for synthetic proxy-gate error logs
(auth/rate-limit rejections); those carry `LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL`
and are skipped, so a request rejected before reaching a provider never produces
a phantom CLIENT span.
5. **Guardrails / services**: the post-call and service hooks emit guardrail and
service spans the same way — typed data → engine → span. Service spans
(Redis/Postgres) are dispatched by `litellm/_service_logger.py`, which
recognizes the V2 `OpenTelemetryV2` logger (a plain `CustomLogger`, not a
subclass of the legacy `OpenTelemetry`). It hands every service call to the
logger — including calls with no parent span — and the V2 adapter decides the
role (`DB_CALL` vs `SERVICE`), the parent (ambient → threaded → root), and
whether the call is a traceable operation or a metrics-only ping. Guardrail
span data is built from the typed, provider-agnostic
`StandardLoggingGuardrailInformation` — no single provider's field shape is
assumed.
6. **Export**: each span ends and is handed to the provider's span processors,
which export to the configured backends (OTLP, console, in-memory, …).
## Components
### Sources of truth (`model/`, no OpenTelemetry import)
These define the shape of a span without depending on the OTel SDK, so they can
be imported anywhere. They live in [`model/`](./model) and form a closed set —
nothing here imports outside it:
- [`semconv.py`](./model/semconv.py) — attribute-key constants (`gen_ai.*`, `http.*`,
`litellm.*`), the GenAI operation/provider enums, and the functions that map
LiteLLM provider/call-type strings onto convention values.
- [`spans.py`](./model/spans.py) — the span registry: every span role, its OTel span
kind, its place in the hierarchy, and its name builder.
- [`payloads.py`](./model/payloads.py) — frozen dataclasses (`LLMCallSpanData`,
`GuardrailSpanData`, `ServiceSpanData`, …) built from heterogeneous logging
payloads via `from_*` classmethods.
- [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings
model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate.
`capture_span_content` gates whether prompt/response bodies may be written as
span attributes; it defaults **off** (`no_content`). The Baggage allowlists are
configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` /
`LITELLM_OTEL_BAGGAGE_METADATA_KEYS` /
`LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or
`baggage_promoted_keys` / `baggage_metadata_keys` /
`baggage_team_metadata_keys` (YAML lists) under `callback_settings.otel` in
`config.yaml` — the latter reach the config through the logger's constructor
kwargs. `baggage_team_metadata_keys` is empty by default, so none of a team's
free-form metadata is promoted until each sub-key is explicitly allowlisted.
- [`baggage.py`](./model/baggage.py) — the single definition of which request-identity
values are promoted into Baggage (so child spans inherit them) and under which
attribute keys.
- [`utils.py`](./model/utils.py) — value coercion, JSON serialization, and
extractor-table application, shared across the package.
### Engine
- [`emitter.py`](./emitter.py) — `SpanEmitter.emit(role, data)`: dedupe → start
the span → run the mapper chain to stamp attributes → set status → end. It
owns no attribute keys. The dedupe set (which coalesces the sync+async firing
of one request) is a bounded LRU so it can't grow without limit.
- [`mappers/`](./mappers) — each mapper turns typed span data into a flat
`{attribute key: value}` dict. They compose: listing several mapper names in
the config layers multiple attribute vocabularies onto the same span.
- `genai` — the canonical OpenTelemetry GenAI vocabulary, always present.
- `legacy` — an additional vocabulary using the older semconv-ai / Traceloop
attribute key names, for backends that read those.
- `openinference`, `langfuse`, `weave`, `langtrace` — vendor vocabularies.
- `resolve_mappers(names)` turns config names into mapper instances.
### Plumbing (`plumbing/`)
The OTel-SDK wiring. Everything here imports only `model/` and each other; it
lives in [`plumbing/`](./plumbing):
- [`providers.py`](./plumbing/providers.py) — builds the `TracerProvider`, its exporters
(from `ExporterSpec`s), and the span processor that copies allowlisted Baggage
entries onto every span. `register_exporter_factory(kind, factory)` lets a
preset contribute a custom exporter `kind` (e.g. one that fetches an auth
token lazily) without coupling this module to any vendor.
- [`context.py`](./plumbing/context.py) — trace-context and Baggage read/write helpers.
- [`routing.py`](./plumbing/routing.py) — `TenantTracerCache`: when a request carries
team/key-scoped vendor credentials, route its spans through a credential-keyed
`TracerProvider` so one logger serves many tenants. The cache is a bounded LRU
that flushes + shuts down evicted providers, since the key derives from
request-supplied credentials and must not grow (or leak threads) without limit.
- [`metrics.py`](./plumbing/metrics.py) — GenAI client metric instruments.
### Adapter
- [`logger.py`](./logger.py) — `OpenTelemetryV2`, a `CustomLogger` that
translates LiteLLM's logging callbacks into typed span data and hands them to
the engine. The LLM-call span is opened at the `log_pre_api_call` boundary
(parented to the live server span via ambient context) and closed at the async
success/failure callback; the open span is held in a bounded cache keyed by
`litellm_call_id`, never threaded through a metadata dict. The logger registers
itself into `litellm.input_callback` so `Logging.pre_call` fires the boundary
hook.
- [`mount.py`](./mount.py) — `instrument_fastapi_app(app)`, the single call site
that attaches `opentelemetry-instrumentation-fastapi` for SERVER spans. It owns
the health-check exclusion default (`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`) and the
passthrough span-naming hook (`PASSTHROUGH_PREFIXES`) so `proxy_server` carries
no OTel detail. A safe no-op when the gate is off or the instrumentation package
is absent; must be called at app-creation time (the middleware stack freezes
once the app serves).
### Presets
- [`presets/`](./presets) — each preset reads one integration's env vars and
returns an `OpenTelemetryV2Config` (exporter destination + mapper vocabularies
+ resource attributes). `PRESET_BY_CALLBACK` maps a callback name (`"arize"`,
`"langfuse_otel"`, …) to its preset. Integrations that support team/key-scoped
credentials also provide a per-request OTLP header builder
(`DYNAMIC_HEADERS_BY_CALLBACK`). Presets do **no** network I/O at build time:
AgentOps, for example, mints its JWT lazily inside a custom exporter on the
first export (in the `BatchSpanProcessor` worker thread), never on the event
loop.
## Extending
- **A new attribute vocabulary for a backend**: add a mapper in `mappers/`
(a class with a `map(data) -> AttributeMap` method, typically built from
`key -> extractor` tables) and register it in `mappers/__init__._MAPPER_BY_NAME`.
- **A new integration**: add a preset in `presets/` that returns an
`OpenTelemetryV2Config`, and register it in `presets/__init__.PRESET_BY_CALLBACK`.
If it supports dynamic credentials, add a header builder to
`DYNAMIC_HEADERS_BY_CALLBACK`.
- **A new span kind**: add a role to `spans.py` (registry entry + name builder),
a payload dataclass in `payloads.py`, and a branch in the relevant mapper(s).

118
litellm/integrations/otel/__init__.py Normal file
View file

@ -0,0 +1,118 @@
"""Typed, semconv-aligned OpenTelemetry instrumentation for LiteLLM.
The three sources of truth — attribute keys (:mod:`semconv`), the span and
hierarchy registry (:mod:`spans`), and the typed span-data inputs
(:mod:`payloads`) — plus :mod:`config` are exported here and are free of any
``opentelemetry`` import. The engine layer (``emitter``, ``providers``,
``context``, ``metrics``) and the ``CustomLogger`` adapter (``logger``) are
reached via their submodule paths so that importing this package never
requires the OTel SDK.
The ``LITELLM_OTEL_V2`` env var gates whether the factory in
``litellm_core_utils.litellm_logging`` constructs the ``OpenTelemetryV2``
class (from :mod:`logger`).
"""
from litellm.integrations.otel.model.config import (
OTEL_V2_ENV,
OpenTelemetryV2Config,
is_otel_v2_enabled,
)
from litellm.integrations.otel.model.baggage import (
BAGGAGE_PROMOTED_KEYS,
DEFAULT_BAGGAGE_METADATA_KEYS,
promoted_baggage,
)
from litellm.integrations.otel.model.metadata import (
RequestContext,
RequestIdentity,
)
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
LLMRequestParams,
LLMUsage,
MCPToolCallSpanData,
ProxyRequestSpanData,
ServerInfo,
ServiceSpanData,
SpanError,
is_mcp_tool_call,
)
from litellm.integrations.otel.model.semconv import (
DB,
HTTP,
MCP,
Client,
Error,
GenAI,
GenAIOperation,
GenAIProvider,
JsonRpc,
LiteLLM,
MCPMethod,
Metric,
Network,
NetworkTransport,
Server,
resolve_operation,
resolve_provider,
)
from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY,
LiteLLMSpanKind,
SpanRole,
SpanSpec,
db_system,
span_role_for_service,
validate_registry,
)
__all__ = [
# config
"OTEL_V2_ENV",
"OpenTelemetryV2Config",
"is_otel_v2_enabled",
# semconv
"BAGGAGE_PROMOTED_KEYS",
"DB",
"DEFAULT_BAGGAGE_METADATA_KEYS",
"Client",
"Error",
"GenAI",
"GenAIOperation",
"GenAIProvider",
"HTTP",
"JsonRpc",
"LiteLLM",
"MCP",
"MCPMethod",
"Metric",
"Network",
"NetworkTransport",
"Server",
"resolve_operation",
"resolve_provider",
# spans
"SPAN_REGISTRY",
"LiteLLMSpanKind",
"SpanRole",
"SpanSpec",
"db_system",
"span_role_for_service",
"validate_registry",
# payloads
"GuardrailSpanData",
"LLMCallSpanData",
"LLMRequestParams",
"LLMUsage",
"MCPToolCallSpanData",
"ProxyRequestSpanData",
"RequestContext",
"RequestIdentity",
"ServerInfo",
"ServiceSpanData",
"SpanError",
"is_mcp_tool_call",
"promoted_baggage",
]

190
litellm/integrations/otel/emitter.py Normal file
View file

@ -0,0 +1,190 @@
"""The span engine: dedup, start, run the mapper chain, set status, end."""
from collections import OrderedDict
from typing import Callable, Sequence
from opentelemetry.context import Context
from opentelemetry.trace import Span, Tracer
from opentelemetry.trace.status import Status, StatusCode
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
MCPToolCallSpanData,
ServiceSpanData,
)
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
from litellm.integrations.otel.model.semconv import Error
from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY,
SpanRole,
guardrail_span_name,
llm_call_span_name,
mcp_tool_call_span_name,
service_span_name,
)
# Roles emit() knows how to name and emit. PROXY_REQUEST and the management
# routes are SERVER spans owned by the mounted FastAPI instrumentor, so they
# have no builder here.
_NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = {
SpanRole.LLM_CALL: llm_call_span_name,
SpanRole.MCP_TOOL_CALL: mcp_tool_call_span_name,
SpanRole.GUARDRAIL: guardrail_span_name,
# DB_CALL and SERVICE are both built from ServiceSpanData; they differ only in
# span kind (CLIENT vs INTERNAL) and attribute vocabulary, not in naming.
SpanRole.DB_CALL: service_span_name,
SpanRole.SERVICE: service_span_name,
}
# Cap on the dedup cache. It only needs to coalesce the sync+async firing window
# of a single in-flight request, so a bounded LRU keeps memory flat on a
# long-running proxy while still covering every concurrently-open call.
_DEDUP_CACHE_MAX = 10_000
class SpanEmitter:
def __init__(
self,
tracer: Tracer,
config: OpenTelemetryV2Config,
mappers: Sequence[AttributeMapper] | None = None,
) -> None:
self._tracer = tracer
self._config = config
# The mapper chain is the sole source of span attributes. When not
# passed in, resolve it from the config so there's one source of truth.
self._mappers: list[AttributeMapper] = (
list(mappers)
if mappers is not None
else resolve_mappers(config.mapper_names)
)
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
# only — the value is unused — so it behaves like a capped set.
self._emitted: "OrderedDict[tuple[str, SpanRole], None]" = OrderedDict()
# -- low-level helpers --------------------------------------------------- #
def start_span(
self,
role: SpanRole,
name: str,
parent_context: Context | None = None,
start_time_ns: int | None = None,
*,
tracer: Tracer | None = None,
) -> Span:
"""Start a span for ``role`` without dedup or attribute mapping.
For callers that own and manage their own span lifecycle. ``tracer``
overrides the bound tracer for this span only, used for per-request
multi-tenant credential routing.
"""
return (tracer or self._tracer).start_span(
name,
context=parent_context,
kind=to_otel_span_kind(SPAN_REGISTRY[role].kind),
start_time=start_time_ns,
)
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
"""Return True once a ``(dedup_key, role)`` pair has been emitted.
Guards against emitting the same span twice when a streaming call
fires both a sync and an async logging callback.
"""
if not dedup_key:
return False
marker = (dedup_key, role)
if marker in self._emitted:
self._emitted.move_to_end(marker)
return True
self._emitted[marker] = None
if len(self._emitted) > _DEDUP_CACHE_MAX:
self._emitted.popitem(last=False) # evict least-recently-used
return False
# -- the engine ---------------------------------------------------------- #
def emit(
self,
role: SpanRole,
data: SpanData,
parent_context: Context | None = None,
*,
start_time_ns: int | None = None,
end_time_ns: int | None = None,
tracer: Tracer | None = None,
) -> Span | None:
"""Emit one complete span: dedup, start, map attributes, status, end.
Return the span, or ``None`` if it was deduplicated away. ``tracer``
overrides the bound tracer for this span, used for per-request routing.
"""
# LLM-call and MCP tool-call spans carry a dedup key (their request's
# call id), so a sync+async double-firing coalesces. ``isinstance`` narrows
# the type for mypy and keeps the engine free of duck-typed attribute reads.
dedup_key = (
data.identity.call_id
if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData))
else None
)
if self._seen(dedup_key, role):
return None
span = self.start_span(
role,
_NAME_BUILDERS[role](data),
parent_context=parent_context,
start_time_ns=start_time_ns,
tracer=tracer,
)
self.finish_span(role, span, data, end_time_ns=end_time_ns)
return span
def finish_span(
self,
role: SpanRole,
span: Span,
data: SpanData,
*,
end_time_ns: int | None = None,
) -> None:
"""Stamp attributes + status on an already-started ``span`` and end it.
The counterpart to :meth:`start_span` for callers that own a span's
lifecycle — the LLM-call span is opened at the request's ``pre_call``
boundary (so it parents to the live server span via real ambient context,
never a span threaded through a metadata dict) and closed here once the
typed payload is available. The span name is (re)built from the now-known
data, since the boundary opener only has a provisional name.
"""
span.update_name(_NAME_BUILDERS[role](data))
for mapper in self._mappers:
for key, value in mapper.map(data).items():
span.set_attribute(key, value)
error = (
data.error
if isinstance(
data,
(
LLMCallSpanData,
MCPToolCallSpanData,
ServiceSpanData,
GuardrailSpanData,
),
)
else None
)
if error and (error.error_type or error.message):
span.set_attribute(Error.TYPE, error.error_type or "error")
span.set_status(
Status(StatusCode.ERROR, error.message or error.error_type or "error")
)
# On success leave the status UNSET (the semconv default) rather than
# forcing OK — that matches the FastAPI server span and avoids implying a
# span-level health signal litellm doesn't actually evaluate. Only a
# genuine error sets a status.
span.end(end_time=end_time_ns)

544
litellm/integrations/otel/logger.py Normal file
View file

@ -0,0 +1,544 @@
"""``CustomLogger`` adapter on the OpenTelemetry span engine."""
from collections import OrderedDict
from contextlib import contextmanager
from datetime import datetime
from typing import TYPE_CHECKING, Any, Iterator, Mapping, cast
from opentelemetry.context import attach, get_current
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Span, Tracer, get_current_span, use_span
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.model.baggage import promoted_baggage
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.context import (
is_recordable_span,
resolve_parent_context,
resolve_request_span_context,
set_request_baggage,
set_request_root_span,
)
from litellm.integrations.otel.emitter import SpanEmitter
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.model.metadata import (
LLMCallEvent,
RequestIdentity,
model_from_request_data,
)
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
MCPToolCallSpanData,
ServiceSpanData,
SpanError,
is_mcp_tool_call,
)
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
get_tracer,
)
from litellm.integrations.otel.plumbing.routing import TenantTracerCache
from litellm.integrations.otel.model.spans import SpanRole, span_role_for_service
from litellm.integrations.otel.model.utils import to_ns
if TYPE_CHECKING:
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
)
LITELLM_TRACER_NAME = "litellm"
# Any callback whose class belongs to one of these modules is "the OTel
# callback" for proxy-global-registration purposes.
_OTEL_MODULES = (
"litellm.integrations.otel",
"litellm.integrations.opentelemetry",
)
# Cap on the open-call carrier map. A span opened at ``pre_call`` that never
# reaches a success/failure callback (e.g. a stream that only fires stream
# events) would otherwise linger; bounding the map evicts the oldest so memory
# stays flat on a long-running proxy while covering every concurrent in-flight
# call.
_OPEN_CALLS_MAX = 10_000
class _LLMCallSpan:
"""The state carried from the ``pre_call`` boundary to span close.
``span`` is the live span when it could be opened at the boundary (the server
span was ambient), or ``None`` when creation was deferred because no ambient
parent was visible — in which case the async callback creates it against its
own (worker-copied) ambient context using ``start_time_ns``. The presence of
a carrier for a call at all is the proof that ``pre_call`` ran, i.e. that an
upstream call was actually attempted.
"""
__slots__ = ("span", "start_time_ns")
def __init__(self, span: "Span | None", start_time_ns: int | None) -> None:
self.span = span
self.start_time_ns = start_time_ns
class OpenTelemetryV2(CustomLogger):
"""The ``CustomLogger`` for OpenTelemetry."""
def __init__(
self,
config: OpenTelemetryV2Config | None = None,
callback_name: str | None = None,
tracer_provider: TracerProvider | None = None,
logger_provider: Any | None = None, # reserved for OTel logs
meter_provider: Any | None = None, # reserved for metrics
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)
self.callback_name = callback_name
self._tracer_provider: TracerProvider = (
tracer_provider
if tracer_provider is not None
else build_tracer_provider(self.config)
)
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
self._emitter = SpanEmitter(
self.tracer, self.config, mappers=resolve_mappers(self.config.mapper_names)
)
self._tenant_tracers = TenantTracerCache(
self.config, callback_name, LITELLM_TRACER_NAME
)
self._open_llm_calls: "OrderedDict[str, _LLMCallSpan]" = OrderedDict()
self._init_otel_logger_on_litellm_proxy()
# ====================================================================== #
# Proxy global registration
# ====================================================================== #
def _register_in_callback_list(self, callbacks: list) -> None:
already_otel = any(
cb.__class__.__module__.startswith(_OTEL_MODULES)
for cb in callbacks
if hasattr(cb, "__class__")
)
if not already_otel:
callbacks.append(self)
def _init_otel_logger_on_litellm_proxy(self) -> None:
try:
from litellm.proxy import proxy_server
except Exception:
return
try:
self._register_in_callback_list(litellm.service_callback)
self._register_in_callback_list(litellm.input_callback)
self._register_in_callback_list(litellm._async_success_callback)
self._register_in_callback_list(litellm._async_failure_callback)
except Exception:
pass
if getattr(proxy_server, "open_telemetry_logger", None) is None:
setattr(proxy_server, "open_telemetry_logger", self)
# ====================================================================== #
# LLM-call callbacks — the span is opened at the ``pre_call`` boundary and
# closed here. See ``log_pre_api_call``.
# ====================================================================== #
def log_pre_api_call(self, model, messages, kwargs):
"""Open the LLM-call span at the call boundary.
Runs synchronously inside the request task, before the upstream call —
the one place where the live server span is genuinely the ambient OTel
context — so the span parents to it natively, with no span threaded
through a metadata dict. The open span is stashed on the per-request
``LiteLLMLoggingObj`` (a typed object) and closed in the async callback.
When no recordable parent is visible (``pre_call`` was driven from a thread
pool for a sync-only provider, where contextvars — and so the anchor —
don't follow), creation is deferred: only the start time is recorded, and
the async callback — whose worker context was copied from the request task
and so still carries the anchor — creates the span then.
Synthetic proxy-gate error logs (auth/rate-limit rejections) also fire this
hook but never made an upstream call; they are tagged and skipped so no
phantom LLM-call span is produced.
"""
call = LLMCallEvent.from_dict(kwargs)
if call.is_no_upstream_call:
return
call_id = call.call_id
if call_id is None:
return
# Idempotent: a retried call may re-enter ``pre_call`` with the same
# call id; keep the first span so its start time is the true one.
if call_id in self._open_llm_calls:
return
start_time_ns = to_ns(datetime.now())
span: Span | None = None
# Parent to the request's anchored root span (stable across the request),
# falling back to ambient on the SDK path. Open the span live only when
# that resolves to a recordable parent; otherwise defer to the close
# callback (the thread-pool case, where the anchor isn't visible here).
parent_context = resolve_request_span_context()
if is_recordable_span(get_current_span(parent_context)):
span = self._emitter.start_span(
SpanRole.LLM_CALL,
call.provisional_span_name,
parent_context=parent_context,
start_time_ns=start_time_ns,
tracer=self._tenant_tracers.tracer_for(
self.tracer, call.dynamic_params
),
)
self._open_llm_calls[call_id] = _LLMCallSpan(
span=span, start_time_ns=start_time_ns
)
# Evict the oldest open call if the map is over budget. A call that opens
# but never closes (a stream that only fires stream events) would linger
# otherwise; the evicted span is simply dropped (never exported).
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
self._open_llm_calls.popitem(last=False)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
return
self._close_llm_call(kwargs, start_time, end_time)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
if self._emit_mcp_tool_call(kwargs, start_time, end_time):
return
self._close_llm_call(kwargs, start_time, end_time)
def _emit_mcp_tool_call(
self,
kwargs: Mapping[str, Any],
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> bool:
"""Emit an MCP tool-call span when the closed request was a tool call.
MCP tool calls reach the success/failure callbacks like any other request
(with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
no ``pre_call`` carrier — so they get their own CLIENT span here, parented
to the request's server span. Returns whether it handled the event, so the
caller skips the LLM-call path. The whole span is emitted at once (there is
no boundary to open it at), deduped on the call id by the emitter.
"""
raw_payload = kwargs.get("standard_logging_object")
if not raw_payload or not is_mcp_tool_call(
cast(Mapping[str, object], raw_payload)
):
return False
payload = cast("StandardLoggingPayload", raw_payload)
data = MCPToolCallSpanData.from_standard_logging_payload(
payload, capture_content=self.config.capture_span_content
)
# A stray LLM carrier from a ``pre_call`` that mis-fired for this id would
# otherwise linger until evicted; drop it so it's neither leaked nor closed
# as a phantom LLM span.
if data.identity.call_id:
self._open_llm_calls.pop(data.identity.call_id, None)
self._emitter.emit(
SpanRole.MCP_TOOL_CALL,
data,
parent_context=resolve_request_span_context(),
start_time_ns=to_ns(start_time),
end_time_ns=to_ns(end_time),
)
return True
def _close_llm_call(
self,
kwargs: Mapping[str, Any],
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
"""Finish the LLM-call span opened at ``pre_call`` (or create it deferred).
No carrier for this call id means ``pre_call`` never ran — the request was
rejected at the gate or blocked by a pre-call guardrail before any upstream
call — so there is nothing to record and no phantom span.
"""
call = LLMCallEvent.from_dict(kwargs)
call_id = call.call_id
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier = self._open_llm_calls.pop(call_id, None) if call_id else None
if carrier is None:
return None
payload = call.payload
if payload is None:
if carrier.span is not None:
# Opened at the boundary but the payload never materialized — end
# it (named provisionally) so it isn't leaked as an open span.
carrier.span.end(end_time=to_ns(end_time))
return None
data = LLMCallSpanData.from_standard_logging_payload(
payload, capture_content=self.config.capture_span_content
)
end_time_ns = to_ns(end_time)
if carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set
# status, and end it. Its parent (the server span) was captured at
# creation from real ambient context.
self._emitter.finish_span(
SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns
)
return carrier.span
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
# The worker copied the request task's context, which carries the anchored
# root span — parent to it (ambient fallback on the SDK path). Seed identity
# Baggage so the span — and the SDK path, which has none — is labeled
# consistently.
parent_ctx = resolve_request_span_context()
bag = promoted_baggage(
data.identity,
data.request_model,
promoted_keys=tuple(self.config.baggage_promoted_keys),
metadata_keys=tuple(self.config.baggage_metadata_keys),
team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
)
if bag:
parent_ctx = set_request_baggage(bag, context=parent_ctx)
return self._emitter.emit(
SpanRole.LLM_CALL,
data,
parent_context=parent_ctx,
start_time_ns=carrier.start_time_ns,
end_time_ns=end_time_ns,
tracer=self._tenant_tracers.tracer_for(self.tracer, call.dynamic_params),
)
# ====================================================================== #
# Service hooks
# ====================================================================== #
async def async_service_success_hook(
self,
payload: Any,
parent_otel_span: Span | None = None,
start_time: datetime | float | None = None,
end_time: datetime | float | None = None,
event_metadata: dict | None = None,
) -> None:
self._emit_service(
payload,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
event_metadata=event_metadata,
error_override=None,
)
async def async_service_failure_hook(
self,
payload: Any,
error: str | None = "",
parent_otel_span: Span | None = None,
start_time: datetime | float | None = None,
end_time: datetime | float | None = None,
event_metadata: dict | None = None,
) -> None:
self._emit_service(
payload,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
event_metadata=event_metadata,
error_override=error or "error",
)
def _emit_service(
self,
payload: Any,
*,
parent_otel_span: Span | None,
start_time: datetime | float | None,
end_time: datetime | float | None,
event_metadata: dict | None,
error_override: str | None,
) -> Span | None:
data = ServiceSpanData.from_payload(payload, event_metadata=event_metadata)
# Decide whether this service call is a span at all, and of what kind.
# ``None`` means metrics-only (framework instrumentation that duplicates a
# gen-AI span — ``self``/``router``/``proxy_pre_call`` — or ``auth``, which
# gets a live phase span instead). Those still feed Prometheus/Datadog via
# their own hooks; they just never enter the trace.
role = span_role_for_service(data.service_name)
if role is None:
return None
# A metrics-only ping with neither timing nor a parent (in-memory queue
# gauges) is not a traceable operation; a span for it would be a
# zero-duration root with no context, so skip it. Real background work
# (budget/reset jobs, spend flush) passes start/end times and still emits
# as a root; anything with a parent emits regardless.
if (
error_override is None
and start_time is None
and end_time is None
and parent_otel_span is None
):
return None
if error_override is not None and data.error is None:
data = ServiceSpanData(
service_name=data.service_name,
call_type=data.call_type,
error=SpanError(message=error_override),
event_metadata=data.event_metadata,
)
# Parent like every other span: ambient context first (so identity Baggage
# rides along and the call nests under whatever request phase is active —
# e.g. a DB lookup under the live ``auth`` span), falling back to the
# server span the proxy threaded as ``parent_otel_span``. A background
# service call has neither, so it starts its own root trace.
parent_context = resolve_parent_context(threaded=parent_otel_span)
return self._emitter.emit(
role,
data,
parent_context=parent_context,
start_time_ns=to_ns(start_time),
end_time_ns=to_ns(end_time),
)
# ====================================================================== #
# async_post_call_* hooks — emit guardrail spans. The server span's status
# / errors are the FastAPI instrumentor's job, so we don't touch it here.
# ====================================================================== #
def seed_request_identity(self, user_api_key_dict: Any, model: Any = None) -> None:
"""Attach request-identity Baggage to the current context + server span.
Seeding identity into Baggage makes **every** span emitted afterwards for
this request — LLM call, guardrail, DB call — inherit it via
``LiteLLMBaggageSpanProcessor``. Called once at the auth boundary (as soon
as the key resolves) so post-auth spans are labeled consistently; the
Baggage rides the request task's contextvar from there on. Auth-internal
DB lookups that run before the key is known stay unlabeled — identity
isn't determined yet, which is correct.
"""
try:
identity = RequestIdentity.from_user_api_key_auth(user_api_key_dict)
bag = promoted_baggage(
identity,
model,
promoted_keys=tuple(self.config.baggage_promoted_keys),
metadata_keys=tuple(self.config.baggage_metadata_keys),
team_metadata_keys=tuple(self.config.baggage_team_metadata_keys),
)
if bag:
# Attach (no detach): the contextvar is scoped to this request's
# asyncio task and is reclaimed when the task ends.
attach(set_request_baggage(bag, context=get_current()))
# The server span was started by the instrumentor before this ran,
# so the Baggage processor (which only fires at span start) won't
# backfill it — stamp identity on it directly.
server_span = get_current_span()
if is_recordable_span(server_span):
# Re-capture the anchor here too: this runs post-auth with the
# server span active and covers entrypoints that bypass
# ``create_litellm_proxy_request_started_span`` (e.g. the SDK
# path's ``async_pre_call_hook``). Idempotent.
set_request_root_span(server_span)
for key, value in bag.items():
server_span.set_attribute(key, value)
except Exception:
pass
@contextmanager
def start_phase_span(self, name: str) -> "Iterator[Span]":
span = self._emitter.start_span(SpanRole.SERVICE, name)
with use_span(span, end_on_exit=True):
yield span
async def async_pre_call_hook(
self,
user_api_key_dict: Any,
cache: Any,
data: dict,
call_type: Any,
) -> dict:
self.seed_request_identity(
user_api_key_dict,
model=model_from_request_data(data),
)
return data
def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None:
# Emitted by the guardrail-recording code the moment a guardrail finishes,
# not from a post-call hook — that hook does not fire on every path (a
# pass-through request that passes its guardrails never reaches it), which
# left passing guardrails without a span.
#
# A guardrail is a sibling of the LLM call under the request's root span,
# so parent it to the explicit anchor — never the active span, which during
# a pre_call guardrail can be the live ``auth`` phase span. Emit with the
# guardrail's actual execution window so a pre_call guardrail is placed
# before the LLM call rather than at emission time. One entry in, one span
# out — the module-level entry point routes each entry to this single
# registered logger so a guardrail is never emitted more than once.
data = GuardrailSpanData.from_logging_entry(entry)
self._emitter.emit(
SpanRole.GUARDRAIL,
data,
parent_context=resolve_request_span_context(),
start_time_ns=to_ns(data.start_time),
end_time_ns=to_ns(data.end_time),
)
def create_litellm_proxy_request_started_span(
self, start_time: datetime, headers: Mapping[str, str] | None
) -> Span | None:
span = get_current_span()
if not is_recordable_span(span):
return None
set_request_root_span(span)
return span
def _registered_v2_logger() -> "OpenTelemetryV2 | None":
try:
from litellm.proxy import proxy_server
except Exception:
return None
logger = getattr(proxy_server, "open_telemetry_logger", None)
return logger if isinstance(logger, OpenTelemetryV2) else None
def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None:
"""Emit a guardrail span on the registered v2 OTel logger.
Called by the guardrail-recording code the moment a guardrail finishes, so a
span is produced regardless of whether a post-call hook later runs (it does
not on the pass-through allow path). Routes through the single canonical
logger — the same one every other v2 entry point uses — so a guardrail
recorded once yields exactly one span; fanning out across every reachable
``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span
emission must never break guardrail evaluation.
"""
logger = _registered_v2_logger()
if logger is None:
return
try:
logger.emit_guardrail_span(entry)
except Exception:
pass
def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None:
logger = _registered_v2_logger()
if logger is not None:
logger.seed_request_identity(user_api_key_dict, model=model)
@contextmanager
def phase_span(name: str) -> "Iterator[Span | None]":
logger = _registered_v2_logger()
if logger is None:
yield None
return
with logger.start_phase_span(name) as span:
yield span

58
litellm/integrations/otel/mappers/__init__.py Normal file
View file

@ -0,0 +1,58 @@
"""Attribute mappers: pure ``LLMCallSpanData -> {attribute key: value}`` functions.
Composition over inheritance: vocabularies layer onto the same span. Listing
``["genai", "openinference"]`` in ``config.mapper_names`` makes every span
carry both the canonical ``gen_ai.*`` keys and the OpenInference (Arize +
Phoenix) keys. Add ``"langfuse"`` and it works for all three backends at once.
"""
from typing import Callable, Iterable
from litellm.integrations.otel.mappers.base import (
AttributeMap,
AttributeMapper,
AttrValue,
)
from litellm.integrations.otel.mappers.genai import GenAIMapper
from litellm.integrations.otel.mappers.langfuse import LangfuseMapper
from litellm.integrations.otel.mappers.langtrace import LangtraceMapper
from litellm.integrations.otel.mappers.legacy import LegacyMapper
from litellm.integrations.otel.mappers.openinference import OpenInferenceMapper
from litellm.integrations.otel.mappers.weave import WeaveMapper
# Registry keyed by ``config.mapper_names`` entries.
_MAPPER_BY_NAME: dict[str, Callable[[], AttributeMapper]] = {
"genai": GenAIMapper,
"legacy": LegacyMapper,
"openinference": OpenInferenceMapper,
"langfuse": LangfuseMapper,
"weave": WeaveMapper,
"langtrace": LangtraceMapper,
}
def resolve_mappers(names: Iterable[str]) -> list[AttributeMapper]:
"""Resolve mapper names to instances. Unknown names raise ``ValueError``."""
out: list[AttributeMapper] = []
for name in names:
factory = _MAPPER_BY_NAME.get(name)
if factory is None:
raise ValueError(
f"unknown mapper name {name!r}; known: " f"{sorted(_MAPPER_BY_NAME)}"
)
out.append(factory())
return out
__all__ = [
"AttributeMap",
"AttributeMapper",
"AttrValue",
"GenAIMapper",
"LangfuseMapper",
"LangtraceMapper",
"LegacyMapper",
"OpenInferenceMapper",
"WeaveMapper",
"resolve_mappers",
]

37
litellm/integrations/otel/mappers/base.py Normal file
View file

@ -0,0 +1,37 @@
"""Mapper protocol and attribute value types."""
from typing import Sequence
from typing_extensions import Protocol, runtime_checkable
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
MCPToolCallSpanData,
ServiceSpanData,
)
AttrScalar = str | bool | int | float
# Mirrors ``opentelemetry.util.types.AttributeValue`` (homogeneous sequences)
# without importing the SDK, so mappers stay OTel-free.
AttrValue = (
AttrScalar | Sequence[str] | Sequence[bool] | Sequence[int] | Sequence[float]
)
AttributeMap = dict[str, AttrValue]
# The closed set of span-data types the engine routes through the mapper chain.
# Server spans (PROXY_REQUEST + management routes) belong to the mounted FastAPI
# instrumentor, not the mapper chain.
SpanData = LLMCallSpanData | MCPToolCallSpanData | GuardrailSpanData | ServiceSpanData
@runtime_checkable
class AttributeMapper(Protocol):
"""Maps a typed span input to a flat dict of OTel span attributes.
One method per mapper, dispatched internally on the ``data`` type. The
engine calls this uniformly for every span kind — mappers that don't speak
a given type return ``{}``. This is why the engine contains no attribute keys.
"""
def map(self, data: SpanData) -> AttributeMap: ...

159
litellm/integrations/otel/mappers/genai.py Normal file
View file

@ -0,0 +1,159 @@
"""Canonical OpenTelemetry GenAI semantic-convention mapper (always active).
Owns the attribute schema for every span kind the engine emits — LLM call,
guardrail, and service — so the engine itself never references attribute keys.
Each span kind declares its schema as a flat ``attribute key -> extractor``
table: one lambda per mapping operation, applied against the typed span data.
"""
from typing import Callable
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import collect, drop_none
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
MCPToolCallSpanData,
ServiceSpanData,
ToolDefinition,
)
from litellm.integrations.otel.model.semconv import (
DB,
MCP,
Error,
GenAI,
LiteLLM,
Server,
)
from litellm.integrations.otel.model.spans import db_system
class GenAIMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
GenAI.OPERATION_NAME: lambda d: d.operation.value,
GenAI.PROVIDER_NAME: lambda d: d.provider or None,
GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
GenAI.REQUEST_TOP_K: lambda d: d.request_params.top_k,
GenAI.REQUEST_MAX_TOKENS: lambda d: d.request_params.max_tokens,
GenAI.REQUEST_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty,
GenAI.REQUEST_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty,
GenAI.REQUEST_STOP_SEQUENCES: lambda d: (
list(d.request_params.stop_sequences)
if d.request_params.stop_sequences
else None
),
GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
GenAI.RESPONSE_ID: lambda d: d.response_id,
GenAI.RESPONSE_FINISH_REASONS: lambda d: (
list(d.finish_reasons) if d.finish_reasons else None
),
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
Error.TYPE: lambda d: d.error.error_type if d.error else None,
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
# The provider/underlying model is only known once routing has picked a
# deployment, so it can't ride identity Baggage (seeded at auth, before
# routing) onto the boundary-born LLM span — stamp it directly here.
LiteLLM.PROVIDER_MODEL: lambda d: d.identity.provider_model or None,
f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
LiteLLM.REQUEST_STREAMING: lambda d: d.is_streaming,
}
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
"name": lambda t: t.name,
"description": lambda t: t.description or None,
"parameters": lambda t: t.parameters_json or None,
}
_MCP_ATTRS: dict[str, Callable[[MCPToolCallSpanData], AttrValue | None]] = {
GenAI.OPERATION_NAME: lambda d: d.operation.value,
MCP.METHOD_NAME: lambda d: d.method,
MCP.SESSION_ID: lambda d: d.session_id,
GenAI.TOOL_NAME: lambda d: d.tool_name or None,
GenAI.TOOL_CALL_ARGUMENTS: lambda d: d.arguments_json,
GenAI.TOOL_CALL_RESULT: lambda d: d.result_json,
LiteLLM.MCP_SERVER_NAME: lambda d: d.server_name,
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost,
}
_GUARDRAIL_ATTRS: dict[str, Callable[[GuardrailSpanData], AttrValue | None]] = {
LiteLLM.GUARDRAIL_NAME: lambda d: d.guardrail_name,
LiteLLM.GUARDRAIL_MODE: lambda d: d.mode,
LiteLLM.GUARDRAIL_STATUS: lambda d: d.status,
LiteLLM.GUARDRAIL_PROVIDER: lambda d: d.provider,
LiteLLM.GUARDRAIL_ACTION: lambda d: d.action,
LiteLLM.GUARDRAIL_RESPONSE: lambda d: d.response_json,
LiteLLM.GUARDRAIL_VIOLATION_CATEGORIES: lambda d: (
list(d.violation_categories) if d.violation_categories else None
),
LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score,
LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score,
LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count,
LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration,
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
}
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {
LiteLLM.SERVICE_NAME: lambda d: d.service_name,
LiteLLM.SERVICE_CALL_TYPE: lambda d: d.call_type,
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case MCPToolCallSpanData():
return collect(self._MCP_ATTRS, data)
case GuardrailSpanData():
return self._guardrail(data)
case ServiceSpanData():
return self._service(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
attrs = collect(cls._LLM_CALL_ATTRS, data)
attrs.update(
drop_none(
{
f"gen_ai.tool.{idx}.{suffix}": extract(tool)
for idx, tool in enumerate(data.tools)
for suffix, extract in cls._TOOL_ATTRS.items()
}
)
)
return attrs
@classmethod
def _guardrail(cls, data: GuardrailSpanData) -> AttributeMap:
return collect(cls._GUARDRAIL_ATTRS, data)
@classmethod
def _service(cls, data: ServiceSpanData) -> AttributeMap:
attrs = collect(cls._SERVICE_ATTRS, data)
# An outbound datastore call (DB_CALL / CLIENT span) also carries db.*
# semconv. Internal services (router, budget jobs, …) have no db.system,
# so they get only the litellm.service.* keys above.
system = db_system(data.service_name)
if system is not None:
attrs[DB.SYSTEM_NAME] = system
if data.call_type:
attrs[DB.OPERATION_NAME] = data.call_type
attrs.update(
{
f"{LiteLLM.METADATA_PREFIX}{key}": value
for key, value in data.event_metadata.items()
}
)
return attrs

84
litellm/integrations/otel/mappers/langfuse.py Normal file
View file

@ -0,0 +1,84 @@
"""Langfuse OTLP attribute mapper.
Langfuse ingests OTLP spans and reads from its own vendor namespace
(``langfuse.observation.*``, ``langfuse.trace.*``). Compose this mapper after
``GenAIMapper`` to send canonical + Langfuse-flavored spans simultaneously.
Every attribute is declared as a ``key -> extractor`` table entry (one callable
per mapping operation): ``_LLM_CALL_ATTRS`` for scalars and ``_BLOB_ATTRS`` for
the JSON-serialized payloads. ``_llm_call`` just applies both tables.
"""
import json
from typing import Callable
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import (
collect,
json_if,
output_messages,
serialize_messages,
)
from litellm.integrations.otel.model.payloads import (
LLMCallSpanData,
LLMRequestParams,
LLMUsage,
)
class LangfuseMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"langfuse.observation.type": lambda d: "generation",
"langfuse.observation.model.name": lambda d: d.request_model or None,
"langfuse.observation.metadata.provider": lambda d: d.provider or None,
"langfuse.observation.id": lambda d: d.identity.call_id or None,
"langfuse.trace.metadata.team_id": lambda d: d.identity.team_id or None,
"langfuse.trace.metadata.team_alias": lambda d: d.identity.team_alias or None,
}
# Sub-tables folded into their respective JSON blobs.
_MODEL_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = {
"temperature": lambda rp: rp.temperature,
"top_p": lambda rp: rp.top_p,
"max_tokens": lambda rp: rp.max_tokens,
"frequency_penalty": lambda rp: rp.frequency_penalty,
"presence_penalty": lambda rp: rp.presence_penalty,
"seed": lambda rp: rp.seed,
}
_USAGE_FIELDS: dict[str, Callable[[LLMUsage], AttrValue | None]] = {
"input": lambda u: u.input_tokens,
"output": lambda u: u.output_tokens,
"total": lambda u: u.total_tokens,
}
# JSON-payload attributes: each builder returns the serialized blob or None.
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"langfuse.observation.model.parameters": lambda d: json_if(
collect(LangfuseMapper._MODEL_PARAMS, d.request_params)
),
"langfuse.observation.input": lambda d: serialize_messages(d.messages_in),
"langfuse.observation.output": lambda d: serialize_messages(output_messages(d)),
"langfuse.observation.usage_details": lambda d: json_if(
collect(LangfuseMapper._USAGE_FIELDS, d.usage)
),
"langfuse.observation.cost_details": lambda d: (
json.dumps({"total": d.response_cost})
if d.response_cost is not None
else None
),
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
return {
**collect(cls._LLM_CALL_ATTRS, data),
**collect(cls._BLOB_ATTRS, data),
}

64
litellm/integrations/otel/mappers/langtrace.py Normal file
View file

@ -0,0 +1,64 @@
"""Langtrace attribute mapper.
Produces Langtrace's attribute vocabulary so a span can be ingested by a
Langtrace backend. Compose it alongside other mappers like any other
vocabulary.
Scalar attributes are declared as a flat ``key -> extractor`` table (one lambda
per mapping operation); the prompt/completion blobs are serialized as a tail.
"""
from typing import Callable
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import (
collect,
json_or_none,
output_messages,
)
from litellm.integrations.otel.model.payloads import LLMCallSpanData
class LangtraceMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"gen_ai.operation.name": lambda d: "chat",
"langtrace.service.name": lambda d: d.provider or None,
"llm.model": lambda d: d.request_model or None,
"gen_ai.response.model": lambda d: d.response_model or None,
"gen_ai.response_id": lambda d: d.response_id or None,
"gen_ai.system_fingerprint": lambda d: d.system_fingerprint or None,
"llm.temperature": lambda d: d.request_params.temperature,
"llm.top_p": lambda d: d.request_params.top_p,
"llm.top_k": lambda d: d.request_params.top_k,
"llm.max_tokens": lambda d: d.request_params.max_tokens,
"llm.frequency_penalty": lambda d: d.request_params.frequency_penalty,
"llm.presence_penalty": lambda d: d.request_params.presence_penalty,
"llm.stream": lambda d: d.is_streaming,
"llm.token.counts.prompt": lambda d: d.usage.input_tokens,
"llm.token.counts.completion": lambda d: d.usage.output_tokens,
"llm.token.counts.total": lambda d: d.usage.total_tokens,
}
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"llm.prompts": lambda d: (
json_or_none(list(d.messages_in)) if d.messages_in else None
),
"llm.completions": lambda d: (
json_or_none(output_messages(d)) if d.choices_out else None
),
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
return {
**collect(cls._LLM_CALL_ATTRS, data),
**collect(cls._BLOB_ATTRS, data),
}

97
litellm/integrations/otel/mappers/legacy.py Normal file
View file

@ -0,0 +1,97 @@
"""Mapper for the older semantic-convention attribute vocabulary.
Emits attributes under the semconv-ai / Traceloop key names (e.g.
``gen_ai.system``, ``gen_ai.usage.prompt_tokens``, ``llm.is_streaming``) plus a
few bare, unprefixed service keys (``service``, ``call_type``, ``error``), for
backends that consume those names.
Like ``GenAIMapper``, each span kind declares its schema as a flat
``attribute key -> extractor`` table: one lambda per mapping operation.
"""
from typing import Callable, Final
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import collect, drop_none
from litellm.integrations.otel.model.payloads import (
LLMCallSpanData,
ServiceSpanData,
ToolDefinition,
)
# Attribute keys in the semconv-ai / Traceloop vocabulary.
_LEGACY_SYSTEM: Final = "gen_ai.system"
_LEGACY_PROMPT_TOKENS: Final = "gen_ai.usage.prompt_tokens"
_LEGACY_COMPLETION_TOKENS: Final = "gen_ai.usage.completion_tokens"
_LEGACY_TOTAL_TOKENS: Final = "gen_ai.usage.total_tokens"
_LEGACY_IS_STREAMING: Final = "llm.is_streaming"
_LEGACY_TOP_K: Final = "llm.top_k"
_LEGACY_FREQUENCY_PENALTY: Final = "llm.frequency_penalty"
_LEGACY_PRESENCE_PENALTY: Final = "llm.presence_penalty"
_LEGACY_STOP_SEQUENCES: Final = "llm.chat.stop_sequences"
_LEGACY_SERVICE: Final = "service"
_LEGACY_CALL_TYPE: Final = "call_type"
_LEGACY_ERROR: Final = "error"
class LegacyMapper:
"""Emits LLM-call and service attributes under the older key names."""
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
_LEGACY_SYSTEM: lambda d: d.provider or None,
_LEGACY_PROMPT_TOKENS: lambda d: d.usage.input_tokens,
_LEGACY_COMPLETION_TOKENS: lambda d: d.usage.output_tokens,
_LEGACY_TOTAL_TOKENS: lambda d: d.usage.total_tokens,
_LEGACY_IS_STREAMING: lambda d: d.is_streaming,
_LEGACY_TOP_K: lambda d: d.request_params.top_k,
_LEGACY_FREQUENCY_PENALTY: lambda d: d.request_params.frequency_penalty,
_LEGACY_PRESENCE_PENALTY: lambda d: d.request_params.presence_penalty,
_LEGACY_STOP_SEQUENCES: lambda d: (
list(d.request_params.stop_sequences)
if d.request_params.stop_sequences
else None
),
}
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
"name": lambda t: t.name,
"description": lambda t: t.description or None,
"parameters": lambda t: t.parameters_json or None,
}
_SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {
_LEGACY_SERVICE: lambda d: d.service_name,
_LEGACY_CALL_TYPE: lambda d: d.call_type,
_LEGACY_ERROR: lambda d: (
d.error.message if d.error is not None and d.error.message else None
),
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case ServiceSpanData():
return self._service(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
attrs = collect(cls._LLM_CALL_ATTRS, data)
attrs.update(
drop_none(
{
f"llm.request.functions.{idx}.{suffix}": extract(tool)
for idx, tool in enumerate(data.tools)
for suffix, extract in cls._TOOL_ATTRS.items()
}
)
)
return attrs
@classmethod
def _service(cls, data: ServiceSpanData) -> AttributeMap:
attrs = collect(cls._SERVICE_ATTRS, data)
attrs.update(dict(data.event_metadata))
return attrs

128
litellm/integrations/otel/mappers/openinference.py Normal file
View file

@ -0,0 +1,128 @@
"""OpenInference attribute mapper (Arize + Arize-Phoenix shared vocabulary).
Spec: https://github.com/Arize-ai/openinference/tree/main/spec — the standard
both Arize and Phoenix consume. Composing this mapper after ``GenAIMapper``
gives the same span both vocabularies, so a single trace lights up Arize +
Phoenix + any other OpenInference-aware backend simultaneously.
"""
import json
from typing import Callable, Sequence
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import (
collect,
drop_none,
json_if,
message_content,
output_messages,
)
from litellm.integrations.otel.model.payloads import (
LLMCallSpanData,
LLMRequestParams,
ToolDefinition,
)
class OpenInferenceMapper:
"""Emits OpenInference attributes for LLM_CALL spans.
Key families (per the OpenInference spec):
- ``openinference.span.kind`` — discriminator (``"LLM"`` here)
- ``llm.model_name`` / ``llm.provider`` / ``llm.invocation_parameters``
- ``llm.input_messages.{i}.message.role`` / ``...content``
- ``llm.output_messages.{i}.message.role`` / ``...content``
- ``llm.token_count.prompt`` / ``...completion`` / ``...total``
- ``input.value`` / ``output.value`` — JSON-serialized request / response
"""
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"openinference.span.kind": lambda d: "LLM",
"llm.model_name": lambda d: d.request_model or None,
"llm.provider": lambda d: d.provider or None,
"llm.token_count.prompt": lambda d: d.usage.input_tokens,
"llm.token_count.completion": lambda d: d.usage.output_tokens,
"llm.token_count.total": lambda d: d.usage.total_tokens,
}
# Folded into the ``llm.invocation_parameters`` JSON blob.
_INVOCATION_PARAMS: dict[str, Callable[[LLMRequestParams], AttrValue | None]] = {
"temperature": lambda rp: rp.temperature,
"top_p": lambda rp: rp.top_p,
"top_k": lambda rp: rp.top_k,
"max_tokens": lambda rp: rp.max_tokens,
"frequency_penalty": lambda rp: rp.frequency_penalty,
"presence_penalty": lambda rp: rp.presence_penalty,
"seed": lambda rp: rp.seed,
}
# Per-tool extractors, keyed by the ``llm.tools.{idx}.*`` suffix.
_TOOL_ATTRS: dict[str, Callable[[ToolDefinition], AttrValue | None]] = {
"tool.name": lambda t: t.name,
"tool.description": lambda t: t.description or None,
"tool.json_schema": lambda t: t.parameters_json or None,
}
# JSON-payload attributes: each builder returns the serialized blob or None.
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"llm.invocation_parameters": lambda d: json_if(
collect(OpenInferenceMapper._INVOCATION_PARAMS, d.request_params)
),
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
return {
**collect(cls._LLM_CALL_ATTRS, data),
**collect(cls._BLOB_ATTRS, data),
**cls._messages("llm.input_messages", "input.value", data.messages_in),
**cls._messages(
"llm.output_messages", "output.value", output_messages(data)
),
**cls._tools(data),
}
@staticmethod
def _messages(
prefix: str, value_key: str, messages: Sequence[object]
) -> AttributeMap:
"""Per-message ``{prefix}.{idx}.message.*`` keys + the ``value_key`` blob."""
parsed = [
(m.get("role") if isinstance(m, dict) else None, message_content(m))
for m in messages
]
attrs = drop_none(
{
key: value
for idx, (role, content) in enumerate(parsed)
for key, value in (
(
f"{prefix}.{idx}.message.role",
role if isinstance(role, str) else None,
),
(f"{prefix}.{idx}.message.content", content),
)
}
)
if parsed:
attrs[value_key] = json.dumps(
[{"role": role, "content": content} for role, content in parsed]
)
return attrs
@classmethod
def _tools(cls, data: LLMCallSpanData) -> AttributeMap:
return drop_none(
{
f"llm.tools.{idx}.{suffix}": extract(tool)
for idx, tool in enumerate(data.tools)
for suffix, extract in cls._TOOL_ATTRS.items()
}
)

76
litellm/integrations/otel/mappers/utils.py Normal file
View file

@ -0,0 +1,76 @@
"""Shared helpers for the attribute mappers.
Small, mapper-agnostic utilities — JSON serialization, message extraction, and
extractor-table application — pulled out of the individual mapper modules so
they live in one place.
"""
import json
from typing import Callable, Mapping, Sequence
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData
def drop_none(values: Mapping[str, AttrValue | None]) -> AttributeMap:
"""Return ``values`` with ``None``-valued entries removed."""
return {k: v for k, v in values.items() if v is not None}
def collect(table: Mapping[str, Callable], source: object) -> AttributeMap:
"""Apply an extractor table to ``source``, dropping ``None`` results."""
return drop_none({key: extract(source) for key, extract in table.items()})
def json_if(payload: Mapping[str, object]) -> str | None:
"""JSON-serialize ``payload`` only when it's non-empty; else ``None``."""
return json.dumps(payload) if payload else None
def json_or_none(value: object) -> str | None:
"""JSON-serialize ``value`` (falling back to ``str``); ``None`` on failure."""
try:
return json.dumps(value, default=str)
except Exception:
return None
def stringify_message(message: object) -> str | None:
"""JSON-serialize a chat message dict; ``None`` if not a dict or on failure."""
if not isinstance(message, dict):
return None
try:
return json.dumps(message, default=str)
except Exception:
return None
def serialize_messages(messages: Sequence[object]) -> str | None:
"""Round-trip a sequence of message dicts through ``stringify_message``."""
serialized = [
json.loads(s) for s in (stringify_message(m) for m in messages) if s is not None
]
return json.dumps(serialized) if serialized else None
def message_content(message: object) -> str | None:
"""Extract the textual ``content`` from a chat message dict."""
if not isinstance(message, dict):
return None
content = message.get("content")
if isinstance(content, str):
return content
if isinstance(content, list):
# multimodal: concatenate text parts only
parts = [
part.get("text", "")
for part in content
if isinstance(part, dict) and part.get("type") == "text"
]
return "".join(p for p in parts if isinstance(p, str)) or None
return None
def output_messages(data: LLMCallSpanData) -> list:
"""The ``message`` payload of each response choice."""
return [c.get("message") for c in data.choices_out if isinstance(c, dict)]

48
litellm/integrations/otel/mappers/weave.py Normal file
View file

@ -0,0 +1,48 @@
"""Weave (W&B) attribute mapper.
Weave consumes OpenInference + a small set of Weave-specific keys (display
name, thread id, output value). This mapper layers the latter on top of
OpenInference's vocabulary — compose ``["genai", "openinference", "weave"]``
to feed a Weave backend.
"""
from typing import Callable
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue, SpanData
from litellm.integrations.otel.mappers.utils import collect, json_or_none
from litellm.integrations.otel.model.payloads import LLMCallSpanData
class WeaveMapper:
"""Maps ``LLMCallSpanData`` to Weave's vendor attributes."""
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
# ``display_name`` has the form ``"{operation} {model}"``. The span
# name already covers that, but Weave reads this attribute too.
"weave.display_name": lambda d: (
f"{d.operation.value} {d.request_model}" if d.request_model else None
),
"weave.call_id": lambda d: d.identity.call_id or None,
}
# JSON-payload attributes: each builder returns the serialized blob or None.
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
# Weave treats the response choices as the "output" payload.
"weave.output": lambda d: (
json_or_none(list(d.choices_out)) if d.choices_out else None
),
}
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():
return self._llm_call(data)
case _:
return {}
@classmethod
def _llm_call(cls, data: LLMCallSpanData) -> AttributeMap:
return {
**collect(cls._LLM_CALL_ATTRS, data),
**collect(cls._BLOB_ATTRS, data),
}

0
litellm/integrations/otel/model/__init__.py Normal file
View file

112
litellm/integrations/otel/model/baggage.py Normal file
View file

@ -0,0 +1,112 @@
"""Baggage promotion: request-identity values carried across child spans.
A bounded set of identity values is written into OpenTelemetry Baggage on the
LLM-call span so that child spans (guardrail, service) inherit them.
``providers.LiteLLMBaggageSpanProcessor`` reads Baggage at span start and stamps
the allowlisted keys onto every span.
This module is the single place baggage is defined: ``_PROMOTABLE`` maps each
promotable attribute key to how its value is read, and the ``*_KEYS`` defaults
select what is promoted unless the config overrides them. ``TEAM_METADATA``'s
extractor filters the team's free-form metadata to the sub-keys an operator
allowlists via ``baggage_team_metadata_keys`` (default none), so the blob is
never promoted whole.
"""
import json
from collections.abc import Callable, Mapping
from typing import Final
from litellm.integrations.otel.model.metadata import RequestIdentity
from litellm.integrations.otel.model.semconv import GenAI, LiteLLM
# Attribute key -> value extractor over (identity, request_model,
# team_metadata_keys). The single definition of what may be promoted and under
# which key. Only the ``TEAM_METADATA`` extractor consults team_metadata_keys
# (to filter the team's metadata to an allowlist); the rest ignore it.
_PROMOTABLE: Final[
dict[str, Callable[[RequestIdentity, str | None, tuple[str, ...]], str | None]]
] = {
LiteLLM.TEAM_ID: lambda identity, model, team_metadata_keys: identity.team_id,
LiteLLM.TEAM_ALIAS: lambda identity, model, team_metadata_keys: identity.team_alias,
LiteLLM.TEAM_METADATA: lambda identity, model, team_metadata_keys: _filtered_team_metadata_json(
identity.team_metadata, team_metadata_keys
),
LiteLLM.KEY_HASH: lambda identity, model, team_metadata_keys: identity.key_hash,
LiteLLM.END_USER: lambda identity, model, team_metadata_keys: identity.end_user,
GenAI.REQUEST_MODEL: lambda identity, model, team_metadata_keys: model,
LiteLLM.PROVIDER_MODEL: lambda identity, model, team_metadata_keys: identity.provider_model,
}
# Keys promoted by default (a subset of ``_PROMOTABLE``). ``END_USER`` is
# promotable but off by default — it identifies an individual user, so stamping
# it onto every span is opt-in via ``config.baggage_promoted_keys``.
BAGGAGE_PROMOTED_KEYS: Final[tuple[str, ...]] = (
LiteLLM.TEAM_ID,
LiteLLM.TEAM_ALIAS,
LiteLLM.TEAM_METADATA,
LiteLLM.KEY_HASH,
GenAI.REQUEST_MODEL,
LiteLLM.PROVIDER_MODEL,
)
# Metadata sub-keys eligible for promotion under the ``litellm.metadata.*``
# namespace. The full metadata blob is never promoted; only this allowlist is.
DEFAULT_BAGGAGE_METADATA_KEYS: Final[tuple[str, ...]] = (
"user_api_key_org_id",
"user_api_key_user_id",
"user_api_key_alias",
"user_api_key_end_user_id",
"requester_ip_address",
)
# Sub-keys of the team's free-form metadata eligible for promotion under
# ``litellm.team.metadata``. Empty by default: a team's metadata can hold
# arbitrary operator data, so none of it is promoted until each key is
# explicitly allowlisted via ``config.baggage_team_metadata_keys``.
DEFAULT_BAGGAGE_TEAM_METADATA_KEYS: Final[tuple[str, ...]] = ()
def promoted_baggage(
identity: RequestIdentity,
request_model: str | None,
promoted_keys: tuple[str, ...],
metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_METADATA_KEYS,
team_metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_TEAM_METADATA_KEYS,
) -> dict[str, str]:
"""Identity values to write into Baggage, filtered to ``promoted_keys``.
``promoted_keys`` selects from ``_PROMOTABLE``; ``metadata_keys`` selects
sub-keys of ``identity.metadata`` to promote under ``litellm.metadata.*``;
``team_metadata_keys`` selects sub-keys of the team's metadata to promote
under ``litellm.team.metadata``. Empty values are dropped.
"""
out: dict[str, str] = {}
for key, extract in _PROMOTABLE.items():
if key in promoted_keys:
value = extract(identity, request_model, team_metadata_keys)
if value:
out[key] = value
for meta_key in metadata_keys:
value = identity.metadata.get(meta_key)
if value:
out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value
return out
def _filtered_team_metadata_json(
metadata: Mapping[str, object] | None,
allowed_keys: tuple[str, ...],
) -> str | None:
"""JSON-serialize only the allowlisted sub-keys of a team's metadata.
Returns ``None`` when nothing is allowlisted or no allowlisted key is
present, so the empty case is dropped rather than promoting ``"{}"``. Keys
are sorted for a stable, diff-friendly value.
"""
if not isinstance(metadata, Mapping) or not allowed_keys:
return None
filtered = {key: metadata[key] for key in allowed_keys if key in metadata}
if not filtered:
return None
return json.dumps(filtered, default=str, sort_keys=True)

252
litellm/integrations/otel/model/config.py Normal file
View file

@ -0,0 +1,252 @@
"""Typed configuration for the OpenTelemetry instrumentation."""
from typing import Any, List
from pydantic import AliasChoices, BaseModel, Field, field_validator, model_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
from typing_extensions import Annotated
from litellm.integrations.otel.model.baggage import (
BAGGAGE_PROMOTED_KEYS,
DEFAULT_BAGGAGE_METADATA_KEYS,
DEFAULT_BAGGAGE_TEAM_METADATA_KEYS,
)
#: Master feature-flag env var. The logger is inert until this is truthy.
OTEL_V2_ENV = "LITELLM_OTEL_V2"
class CaptureMessageContent(str):
NO_CONTENT = "no_content"
SPAN_ONLY = "span_only"
EVENT_ONLY = "event_only"
SPAN_AND_EVENT = "span_and_event"
class _OTelV2Flag(BaseSettings):
model_config = SettingsConfigDict(extra="ignore")
enabled: bool = Field(default=False, validation_alias=AliasChoices(OTEL_V2_ENV))
def is_otel_v2_enabled() -> bool:
return _OTelV2Flag().enabled
class ExporterSpec(BaseModel):
"""One span-export destination.
The shared ``TracerProvider`` attaches one ``SpanProcessor`` per spec, so
listing several specs sends every span to all of them at once (e.g. Arize +
Phoenix + your own Honeycomb).
"""
model_config = {"extra": "forbid"}
kind: str = Field(
default="console",
description="console | in_memory | otlp_http | otlp_grpc | <factory kind>",
)
endpoint: str | None = None
headers: str | None = None
options: dict[str, str] | None = Field(
default=None,
description=(
"Factory-specific configuration for a custom exporter ``kind`` "
"registered via ``providers.register_exporter_factory`` (e.g. an "
"API key a lazy-auth exporter fetches a token with). Ignored by the "
"built-in console/in_memory/otlp exporters."
),
)
use_simple_processor: bool | None = Field(
default=None,
description=(
"Force SimpleSpanProcessor regardless of exporter kind. Default: "
"auto (Simple for console/in_memory, Batch otherwise)."
),
)
class OpenTelemetryV2Config(BaseSettings):
model_config = SettingsConfigDict(populate_by_name=True, extra="ignore")
# ----- single-destination shorthand, read from standard OTEL_* envs ----- #
exporter: str = Field(
default="console",
validation_alias=AliasChoices("OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_PROTOCOL"),
description=(
"Exporter kind for the single-destination shorthand. The model "
"validator folds this (with ``endpoint`` / ``headers``) into a "
"one-entry ``exporters`` list when ``exporters`` is empty; set "
"``exporters`` directly for multiple destinations."
),
)
endpoint: str | None = Field(
default=None,
validation_alias=AliasChoices("OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"),
)
headers: str | None = Field(
default=None,
validation_alias=AliasChoices("OTEL_HEADERS", "OTEL_EXPORTER_OTLP_HEADERS"),
)
service_name: str = Field(
default="litellm", validation_alias=AliasChoices("OTEL_SERVICE_NAME")
)
deployment_environment: str | None = Field(
default=None, validation_alias=AliasChoices("OTEL_ENVIRONMENT_NAME")
)
enable_metrics: bool = Field(
default=False,
validation_alias=AliasChoices("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS"),
)
enable_events: bool = Field(
default=False,
validation_alias=AliasChoices("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS"),
)
capture_message_content: str = Field(
default=CaptureMessageContent.NO_CONTENT,
validation_alias=AliasChoices(
"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
),
)
legacy_compat: bool = Field(
default=True, validation_alias=AliasChoices("LITELLM_OTEL_LEGACY_COMPAT")
)
# ----- explicit multi-destination / vocabulary configuration ------------ #
exporters: list[ExporterSpec] = Field(
default_factory=list,
description=(
"One destination per spec. The shared TracerProvider attaches a "
"SpanProcessor per entry. When empty, the model validator folds "
"the ``exporter`` / ``endpoint`` / ``headers`` shorthand into a "
"single spec so there is always at least one destination."
),
)
mapper_names: Annotated[List[str], NoDecode] = Field(
default_factory=lambda: ["genai"],
description=(
"Ordered attribute vocabularies to emit. ``genai`` is the "
"canonical OTel GenAI vocabulary and is always placed first. "
"Vendor names: ``openinference`` (Arize + Phoenix), ``langfuse``, "
"``weave``, ``langtrace``."
),
)
resource_attributes: dict[str, str] = Field(
default_factory=dict,
description=(
"Extra Resource attributes beyond ``service.name`` and "
"``deployment.environment`` (e.g. integration-specific markers)."
),
)
baggage_promoted_keys: Annotated[List[str], NoDecode] = Field(
default_factory=lambda: list(BAGGAGE_PROMOTED_KEYS),
validation_alias=AliasChoices(
"baggage_promoted_keys", "LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS"
),
description=(
"Identity attribute keys written into Baggage and stamped on every "
"child span (e.g. ``litellm.team.id``). Configure via the "
"``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS`` env var (comma-separated) or "
"``callback_settings.otel.baggage_promoted_keys`` in config.yaml (a "
"YAML list)."
),
)
baggage_metadata_keys: Annotated[List[str], NoDecode] = Field(
default_factory=lambda: list(DEFAULT_BAGGAGE_METADATA_KEYS),
validation_alias=AliasChoices(
"baggage_metadata_keys", "LITELLM_OTEL_BAGGAGE_METADATA_KEYS"
),
description=(
"Metadata sub-keys promoted under the ``litellm.metadata.*`` "
"namespace. Configure via the ``LITELLM_OTEL_BAGGAGE_METADATA_KEYS`` "
"env var (comma-separated) or "
"``callback_settings.otel.baggage_metadata_keys`` in config.yaml."
),
)
baggage_team_metadata_keys: Annotated[List[str], NoDecode] = Field(
default_factory=lambda: list(DEFAULT_BAGGAGE_TEAM_METADATA_KEYS),
validation_alias=AliasChoices(
"baggage_team_metadata_keys", "LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS"
),
description=(
"Sub-keys of the team's free-form metadata promoted under "
"``litellm.team.metadata``. Empty by default so none of a team's "
"metadata leaves the process until explicitly allowlisted. Configure "
"via the ``LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS`` env var "
"(comma-separated) or "
"``callback_settings.otel.baggage_team_metadata_keys`` in config.yaml."
),
)
@field_validator(
"baggage_promoted_keys",
"baggage_metadata_keys",
"baggage_team_metadata_keys",
"mapper_names",
mode="before",
)
@classmethod
def _split_csv(cls, value: Any) -> Any:
"""Accept a comma-separated string for list fields.
Env vars are strings, but these fields are lists. Pydantic-settings would
otherwise require JSON for a list env var; splitting on commas here lets
an operator write ``LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS=litellm.team.id,litellm.api_key.hash``.
YAML lists (from ``callback_settings.otel.*``) and real lists pass through
unchanged.
"""
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
return value
@model_validator(mode="after")
def _normalize(self) -> "OpenTelemetryV2Config":
# An endpoint with the default exporter kind implies OTLP/HTTP.
if self.endpoint and self.exporter == "console":
self.exporter = "otlp_http"
# When no explicit destinations are given, fold the single-destination
# shorthand into one spec so the provider always has a destination.
if not self.exporters:
self.exporters = [
ExporterSpec(
kind=self.exporter,
endpoint=self.endpoint,
headers=self.headers,
)
]
# Ensure ``genai`` is always present and first.
names = list(self.mapper_names)
if "genai" in names:
names = ["genai"] + [n for n in names if n != "genai"]
else:
names = ["genai"] + names
# When enabled, also emit attribute keys under their semconv-ai /
# Traceloop names via the ``legacy`` mapper. Append it at the tail so
# the canonical ``genai`` keys win on any conflict.
if self.legacy_compat and "legacy" not in names:
names.append("legacy")
self.mapper_names = names
return self
@property
def capture_span_content(self) -> bool:
"""Whether prompt/response content may be stamped as span attributes.
Defaults off (``no_content``): an operator must opt in before message
bodies leave the process, so a user request can never force its prompt
or completion into the configured backend while capture is disabled.
"""
return self.capture_message_content in (
CaptureMessageContent.SPAN_ONLY,
CaptureMessageContent.SPAN_AND_EVENT,
)
@classmethod
def from_env(cls) -> "OpenTelemetryV2Config":
return cls()

294
litellm/integrations/otel/model/metadata.py Normal file
View file

@ -0,0 +1,294 @@
"""The single translation layer between a request's metadata and the spans.
Every relevant field litellm exposes about a request — the user-facing model,
the model actually dispatched to the provider, the deployment, and the caller's
identity (team, key, end-user) — is parsed **once**, here, out of the
``StandardLoggingPayload`` (or a ``UserAPIKeyAuth`` at the auth boundary). Span
data, baggage promotion, and the mappers then read these typed fields instead of
each digging into the raw ``metadata`` / ``hidden_params`` dicts.
Two models live here because a request's identity is known *before* its model
resolution is:
* :class:`RequestIdentity` — team / key / end-user, seeded into Baggage at the
auth boundary (``from_user_api_key_auth``), before routing has picked a
deployment. ``provider_model`` is therefore absent from that early seed and is
only filled in from the payload once the call closes.
* :class:`RequestContext` — the full picture available at close: the resolved
request vs. provider model split, plus the response model, model group, model
id, and api base, wrapping the :class:`RequestIdentity`.
The request-vs-provider model split is the subtle part. On the proxy a caller
asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment
(e.g. ``azure/my-deployment``); the two are distinct and both worth recording.
``StandardLoggingPayload`` exposes them as:
* ``model_group`` — the user-facing name the caller requested.
* ``model`` — already reconstructed (see ``reconstruct_model_name``) to the name
litellm dispatched to the provider (the deployment, provider-prefixed).
* ``hidden_params.litellm_model_name`` — a secondary source for the dispatched
model (populated only on some call paths, e.g. files).
So ``gen_ai.request.model`` is the *group* (falling back to the call model on the
SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched*
model. They coincide on the SDK path, which is correct.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Mapping, cast
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.integrations.otel.model.semconv import resolve_operation
from litellm.integrations.otel.model.utils import as_str
if TYPE_CHECKING:
from litellm.types.utils import StandardLoggingPayload
@dataclass(frozen=True)
class RequestIdentity:
call_id: str | None = None
team_id: str | None = None
team_alias: str | None = None
# The team's free-form metadata, carried raw (empty/missing -> None) and
# filtered to an operator allowlist only at Baggage-promotion time, so an
# unconfigured deployment never promotes any of it.
team_metadata: Mapping[str, Any] | None = None
key_hash: str | None = None
end_user: str | None = None
# The model litellm dispatched to the provider. Only known once the call
# completes (routing has picked a deployment), so it's absent from the
# auth-time seed and filled only from the payload.
provider_model: str | None = None
metadata: Mapping[str, str] = field(default_factory=dict)
@classmethod
def from_payload(cls, payload: "StandardLoggingPayload") -> "RequestIdentity":
"""Parse caller identity out of a closed request's payload metadata.
``provider_model`` is resolved here too (see :func:`resolve_provider_model`)
so the identity carried into Baggage labels every span with the dispatched
model, not just the user-facing one.
"""
raw_meta = cast(Mapping[str, object], payload.get("metadata") or {})
metadata = {
key: str(value)
for key, value in raw_meta.items()
if isinstance(value, (str, bool, int, float))
}
return cls(
call_id=as_str(payload.get("litellm_call_id")) or as_str(payload.get("id")),
# StandardLoggingMetadata's canonical key is ``user_api_key_team_id``;
# the bare ``team_id`` is a legacy alias and is often empty, so prefer
# the canonical key and fall back to the alias.
team_id=as_str(raw_meta.get("user_api_key_team_id"))
or as_str(raw_meta.get("team_id")),
team_alias=as_str(raw_meta.get("user_api_key_team_alias"))
or as_str(raw_meta.get("team_alias")),
team_metadata=_team_metadata_dict(
raw_meta.get("user_api_key_team_metadata")
),
key_hash=as_str(raw_meta.get("user_api_key_hash")),
end_user=as_str(payload.get("end_user"))
or as_str(raw_meta.get("user_api_key_end_user_id")),
provider_model=resolve_provider_model(payload),
metadata=metadata,
)
@classmethod
def from_user_api_key_auth(cls, auth: object) -> "RequestIdentity":
"""Identity from a ``UserAPIKeyAuth`` (duck-typed to keep this module
free of a proxy import).
Used in the pre-call hook to seed Baggage early — before any LLM,
guardrail, or service span is created — so the whole request's spans
inherit identity, not just the LLM-call span. Metadata sub-keys use the
``user_api_key_*`` names that ``baggage.DEFAULT_BAGGAGE_METADATA_KEYS``
promotes.
"""
get = lambda name: getattr(auth, name, None) # noqa: E731
metadata = {
meta_key: str(value)
for meta_key, attr in (
("user_api_key_user_id", "user_id"),
("user_api_key_org_id", "org_id"),
("user_api_key_alias", "key_alias"),
("user_api_key_end_user_id", "end_user_id"),
)
if (value := get(attr))
}
return cls(
team_id=as_str(get("team_id")),
team_alias=as_str(get("team_alias")),
team_metadata=_team_metadata_dict(get("team_metadata")),
key_hash=as_str(get("api_key")),
end_user=as_str(get("end_user_id")),
# ``provider_model`` is unknown at the auth boundary — routing hasn't
# picked a deployment yet — so it's only populated from the payload.
metadata=metadata,
)
@dataclass(frozen=True)
class RequestContext:
"""The fully-resolved view of a closed request, parsed once from the payload.
``request_model`` is the user-facing requested model and ``provider_model``
(on :attr:`identity`) is the model litellm dispatched to the provider; the two
differ on the proxy (group vs. deployment) and coincide on the SDK path.
"""
request_model: str
response_model: str | None
model_group: str | None
model_id: str | None
api_base: str | None
identity: RequestIdentity
@property
def provider_model(self) -> str | None:
"""The dispatched-model name, carried on the identity for Baggage."""
return self.identity.provider_model
@classmethod
def from_standard_logging_payload(
cls, payload: "StandardLoggingPayload"
) -> "RequestContext":
raw_meta = cast(Mapping[str, object], payload.get("metadata") or {})
hidden = cast(Mapping[str, object], payload.get("hidden_params") or {})
raw_response = payload.get("response")
response = cast(
Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}
)
model_group = as_str(payload.get("model_group")) or as_str(
raw_meta.get("model_group")
)
return cls(
# The user asked for the group; fall back to the call model on the SDK
# path, which has no group. Empty string (never None) so the span name
# builder and the mapper see a plain string.
request_model=model_group or as_str(payload.get("model")) or "",
response_model=as_str(response.get("model")),
model_group=model_group,
model_id=as_str(payload.get("model_id"))
or _model_info_id(raw_meta.get("model_info")),
api_base=as_str(payload.get("api_base")) or as_str(hidden.get("api_base")),
identity=RequestIdentity.from_payload(payload),
)
# --- live-callback kwargs parsing ------------------------------------------- #
#
# The model and helpers below parse the *live* callback ``kwargs`` god object (and
# the raw pre/post-call ``data`` dicts) — the untyped request state that reaches a
# ``CustomLogger`` before, or instead of, a ``StandardLoggingPayload``. They live
# here, with the payload/auth parsers, so every read out of a request's raw dicts
# is in one place rather than scattered across the ``CustomLogger``.
@dataclass(frozen=True)
class LLMCallEvent:
"""The typed view of the live callback ``kwargs`` (``model_call_details``).
litellm hands every callback an untyped ``kwargs`` god object. The fields the
OTel logger needs out of it are parsed **once**, here, so the ``CustomLogger``
reads typed attributes instead of digging into the dict at each boundary.
"""
# The ``litellm_call_id`` correlating ``pre_call`` with the close callback.
# Present in ``model_call_details`` at ``pre_call`` and in both the kwargs and
# the ``standard_logging_object`` at success/failure, so it's a stable key for
# the open-call carrier — no back-reference to the logging object required (the
# object isn't reachable from the callback kwargs at ``pre_call`` time).
call_id: str | None
# The ``StandardLoggingPayload`` carried on a success/failure callback; ``None``
# at ``pre_call``, or when the call closed before any payload materialized (so
# there is nothing to stamp on the span).
payload: "StandardLoggingPayload | None"
# The ``standard_callback_dynamic_params`` routing the call to a per-tenant
# tracer (its own exporter/endpoint), or ``None`` when the call isn't scoped.
dynamic_params: Any
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
provisional_span_name: str
@classmethod
def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent":
raw_payload = kwargs.get("standard_logging_object")
payload = cast("StandardLoggingPayload", raw_payload) if raw_payload else None
operation = resolve_operation(as_str(kwargs.get("call_type")))
model = as_str(kwargs.get("model")) or ""
return cls(
call_id=_call_id(payload, kwargs),
payload=payload,
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
provisional_span_name=f"{operation.value} {model}".strip(),
)
def _call_id(
payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]
) -> str | None:
"""The call id from the payload (when closed) or the bare kwargs (at pre_call)."""
if payload is not None:
call_id = as_str(payload.get("litellm_call_id")) or as_str(payload.get("id"))
if call_id:
return call_id
return as_str(kwargs.get("litellm_call_id"))
def model_from_request_data(data: object) -> str | None:
"""The user-facing ``model`` from a pre-call ``data`` dict (``None`` if absent).
Read at the auth boundary to label early Baggage before routing has resolved
a deployment; ``data`` is duck-typed since it arrives untyped from the proxy.
"""
if isinstance(data, Mapping):
return as_str(data.get("model"))
return None
def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None:
"""The model litellm dispatched to the provider, from the payload.
Prefers the explicit ``hidden_params.litellm_model_name`` (set on call paths
that know it, e.g. files), then the top-level ``model`` — which
``reconstruct_model_name`` has already resolved to the deployment's
provider-prefixed name. Returns ``None`` only when neither is present.
"""
raw_meta = cast(Mapping[str, object], payload.get("metadata") or {})
hidden = cast(Mapping[str, object], payload.get("hidden_params") or {})
return (
# ``deployment`` survives only on paths that don't strip it from metadata;
# harmless (and most precise) to prefer it when present.
as_str(raw_meta.get("deployment"))
or as_str(hidden.get("litellm_model_name"))
or as_str(payload.get("model"))
)
def _model_info_id(model_info: object) -> str | None:
"""The deployment id from a ``metadata.model_info`` sub-dict, if present."""
if isinstance(model_info, Mapping):
return as_str(model_info.get("id"))
return None
def _team_metadata_dict(value: object) -> Mapping[str, Any] | None:
"""The team's free-form metadata as a raw mapping, or ``None`` when missing
or empty.
Carried raw on the identity and filtered to an operator allowlist only at
Baggage-promotion time (see ``baggage.promoted_baggage``), so an empty case
is dropped rather than carrying a useless ``{}``.
"""
if isinstance(value, Mapping) and value:
return dict(value)
return None

542
litellm/integrations/otel/model/payloads.py Normal file
View file

@ -0,0 +1,542 @@
"""Typed span-data inputs: frozen dataclasses the engine and mappers consume."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from enum import Enum
from typing import TYPE_CHECKING, ClassVar, Mapping, cast
from urllib.parse import urlsplit
from litellm.integrations.otel.model.metadata import (
RequestContext,
RequestIdentity,
)
from litellm.integrations.otel.model.semconv import (
GenAIOperation,
MCPMethod,
resolve_operation,
resolve_provider,
)
from litellm.integrations.otel.model.utils import (
as_bool,
as_float,
as_int,
as_str,
as_str_tuple,
)
# ``RequestIdentity`` and the request-metadata translation now live in
# :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep
# resolving it.
__all__ = [
"RequestContext",
"RequestIdentity",
"GuardrailSpanData",
"LLMCallSpanData",
"LLMRequestParams",
"LLMUsage",
"MCPToolCallSpanData",
"ProxyRequestSpanData",
"ServerInfo",
"ServiceSpanData",
"SpanError",
"ToolDefinition",
"is_mcp_tool_call",
]
if TYPE_CHECKING:
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
StandardLoggingPayload,
)
# --- typed sub-structures ---------------------------------------------------- #
@dataclass(frozen=True)
class LLMRequestParams:
temperature: float | None = None
top_p: float | None = None
top_k: int | None = None
max_tokens: int | None = None
frequency_penalty: float | None = None
presence_penalty: float | None = None
stop_sequences: tuple[str, ...] | None = None
seed: int | None = None
@classmethod
def from_model_parameters(cls, params: Mapping[str, object]) -> "LLMRequestParams":
max_tokens = as_int(params.get("max_tokens"))
if max_tokens is None:
max_tokens = as_int(params.get("max_completion_tokens"))
return cls(
temperature=as_float(params.get("temperature")),
top_p=as_float(params.get("top_p")),
top_k=as_int(params.get("top_k")),
max_tokens=max_tokens,
frequency_penalty=as_float(params.get("frequency_penalty")),
presence_penalty=as_float(params.get("presence_penalty")),
stop_sequences=as_str_tuple(params.get("stop")),
seed=as_int(params.get("seed")),
)
@dataclass(frozen=True)
class LLMUsage:
input_tokens: int | None = None
output_tokens: int | None = None
total_tokens: int | None = None
@dataclass(frozen=True)
class SpanError:
error_type: str | None = None
message: str | None = None
@dataclass(frozen=True)
class ServerInfo:
address: str | None = None
port: int | None = None
@classmethod
def from_api_base(cls, api_base: str | None) -> ServerInfo | None:
if not api_base:
return None
parsed = urlsplit(api_base if "://" in api_base else f"//{api_base}")
if not parsed.hostname:
return None
return cls(address=parsed.hostname, port=parsed.port)
@dataclass(frozen=True)
class GuardrailSpanData:
guardrail_name: str
mode: str | None = None
status: str | None = None
masked_entity_count: int | None = None
provider: str | None = None
action: str | None = None
# The guardrail verdict / provider response (e.g. the moderation result),
# JSON-serialized. This is the detail that belongs on the guardrail span.
response_json: str | None = None
violation_categories: tuple[str, ...] = ()
confidence_score: float | None = None
risk_score: float | None = None
duration: float | None = None
# Actual execution window (epoch seconds) from the logging entry, so the span
# is placed when the guardrail really ran — a pre_call guardrail before the
# LLM call — rather than at post-call emission time.
start_time: float | None = None
end_time: float | None = None
# Provider-agnostic configuration/detection metadata (see
# ``StandardLoggingGuardrailInformation``). Present for any guardrail that
# populates them, not just one provider's shape.
guardrail_id: str | None = None
policy_template: str | None = None
detection_method: str | None = None
# Set when the guardrail intervened/blocked or failed, so the emitter marks
# the span ERROR — a blocking guardrail is an error outcome for that span.
error: SpanError | None = None
# Guardrail statuses that mean the guardrail did not pass the request through.
_ERROR_STATUSES: ClassVar[frozenset[str]] = frozenset(
{"guardrail_intervened", "guardrail_failed_to_respond"}
)
@classmethod
def from_logging_entry(
cls, entry: "StandardLoggingGuardrailInformation"
) -> "GuardrailSpanData":
"""Build from one ``standard_logging_guardrail_information`` entry.
Reads the canonical, provider-agnostic ``StandardLoggingGuardrailInformation``
keys only — no guessing at a single provider's field names. Values that are
typed as enums or lists (e.g. ``guardrail_mode``) are normalized to a
stable string rather than assumed to already be plain strings.
"""
get = cast(Mapping[str, object], entry).get
status = as_str(get("guardrail_status"))
response = get("guardrail_response")
error = (
SpanError(error_type=status, message=as_str(get("guardrail_action")))
if status in cls._ERROR_STATUSES
else None
)
return cls(
guardrail_name=as_str(get("guardrail_name")) or "guardrail",
mode=_guardrail_mode_str(get("guardrail_mode")),
status=status,
masked_entity_count=_total_masked_entities(get("masked_entity_count")),
provider=as_str(get("guardrail_provider")),
action=as_str(get("guardrail_action")),
response_json=_json_or_none(response) if response is not None else None,
violation_categories=as_str_tuple(get("violation_categories")) or (),
confidence_score=as_float(get("confidence_score")),
risk_score=as_float(get("risk_score")),
duration=as_float(get("duration")),
start_time=as_float(get("start_time")),
end_time=as_float(get("end_time")),
guardrail_id=as_str(get("guardrail_id")),
policy_template=as_str(get("policy_template")),
detection_method=as_str(get("detection_method")),
error=error,
)
@dataclass(frozen=True)
class ServiceSpanData:
service_name: str
call_type: str | None = None
error: SpanError | None = None
# Caller-supplied attributes to stamp on the service span, passed through
# from ``async_service_*_hook(event_metadata=...)``. The mapper owns how
# these are namespaced: the canonical vocabulary uses ``litellm.metadata.*``
# keys, the semconv-ai / Traceloop vocabulary uses the bare key names.
event_metadata: Mapping[str, str] = field(default_factory=dict)
@classmethod
def from_payload(
cls,
payload: "ServiceLoggerPayload",
event_metadata: Mapping[str, object] | None = None,
) -> "ServiceSpanData":
# ``payload.service`` is a ``ServiceTypes(str, Enum)`` and ``error`` is
# ``Optional[str]`` on the Pydantic model — no defensive reads needed.
# ``event_metadata`` is sanitized: the legacy service decorators pass raw
# call-site data (live objects, full request metadata, response headers),
# none of which belongs on a span.
return cls(
service_name=payload.service.value,
call_type=payload.call_type,
error=SpanError(message=payload.error) if payload.error else None,
event_metadata=sanitize_event_metadata(event_metadata),
)
@dataclass(frozen=True)
class ProxyRequestSpanData:
http_method: str
route: str
url_path: str | None = None
status_code: int | None = None
identity: RequestIdentity | None = None
# --- the primary LLM-call model ---------------------------------------------- #
@dataclass(frozen=True)
class ToolDefinition:
"""A single function/tool declared on a chat-completion request."""
name: str
description: str | None = None
parameters_json: str | None = (
None # JSON-serialized schema (str so it's an AttrValue)
)
@dataclass(frozen=True)
class LLMCallSpanData:
operation: GenAIOperation
provider: str
request_model: str
response_model: str | None
response_id: str | None
request_params: LLMRequestParams
usage: LLMUsage
finish_reasons: tuple[str, ...]
error: SpanError | None
response_cost: float | None
server: ServerInfo | None
identity: RequestIdentity
is_streaming: bool | None = None
tools: tuple[ToolDefinition, ...] = ()
# Raw messages and response, needed by vendor mappers (OpenInference,
# Langfuse, Weave) that stamp message-level attributes. ``messages_in`` is
# the request payload; ``choices_out`` mirrors ``response.choices`` from
# the StandardLoggingPayload. Both are tuples of immutable mappings so the
# dataclass stays hashable and frozen.
messages_in: tuple[Mapping[str, object], ...] = ()
choices_out: tuple[Mapping[str, object], ...] = ()
system_fingerprint: str | None = None
@classmethod
def from_standard_logging_payload(
cls, payload: "StandardLoggingPayload", capture_content: bool = False
) -> "LLMCallSpanData":
params = cast(Mapping[str, object], payload.get("model_parameters") or {})
# The single parse of the request's metadata — the request-vs-provider
# model split, the response model, api base, and identity all come from
# here rather than being re-derived from the raw payload dicts.
context = RequestContext.from_standard_logging_payload(payload)
# Normalize ``response`` to a dict once so the content/id reads below are a
# plain ``.get`` — no repeated ``isinstance`` guards.
raw_response = payload.get("response")
response = cast(
Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}
)
choices_out = _dicts(response.get("choices"))
# ``finish_reasons`` is metadata, not content, so derive it from
# ``choices_out`` before gating. The raw message/choice bodies are only
# retained when content capture is enabled (see ``capture_span_content``);
# otherwise the content-bearing mappers receive empty sequences and emit
# no prompt/response text.
finish_reasons = _finish_reasons(choices_out)
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
request_model=context.request_model,
response_model=context.response_model,
response_id=as_str(response.get("id")),
request_params=LLMRequestParams.from_model_parameters(params),
usage=LLMUsage(
input_tokens=as_int(payload.get("prompt_tokens")),
output_tokens=as_int(payload.get("completion_tokens")),
total_tokens=as_int(payload.get("total_tokens")),
),
finish_reasons=finish_reasons,
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
server=ServerInfo.from_api_base(context.api_base),
identity=context.identity,
is_streaming=as_bool(payload.get("stream")),
tools=_extract_tools(params),
messages_in=_dicts(payload.get("messages")) if capture_content else (),
choices_out=choices_out if capture_content else (),
system_fingerprint=as_str(response.get("system_fingerprint")),
)
# --- the MCP tool-call model ------------------------------------------------- #
@dataclass(frozen=True)
class MCPToolCallSpanData:
"""One MCP ``tools/call`` execution, parsed from a closed request's payload.
The proxy is an MCP *client* to the upstream server it forwards the call to,
so this is a CLIENT span. ``arguments_json``/``result_json`` are the tool's
input/output — sensitive content, so they're only retained when content
capture is enabled, mirroring ``LLMCallSpanData``'s message bodies.
"""
operation: GenAIOperation
method: str
tool_name: str
server_name: str | None
session_id: str | None
arguments_json: str | None
result_json: str | None
error: SpanError | None
response_cost: float | None
identity: RequestIdentity
@classmethod
def from_standard_logging_payload(
cls, payload: "StandardLoggingPayload", capture_content: bool = False
) -> "MCPToolCallSpanData":
meta = _mcp_tool_call_metadata(cast(Mapping[str, object], payload))
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
method=MCPMethod.TOOLS_CALL.value,
tool_name=as_str(meta.get("name")) or "",
server_name=as_str(meta.get("mcp_server_name")),
session_id=as_str(meta.get("mcp_session_id")),
arguments_json=(
_json_or_none(meta.get("arguments"))
if capture_content and meta.get("arguments") is not None
else None
),
result_json=(
_json_or_none(meta.get("result"))
if capture_content and meta.get("result") is not None
else None
),
error=_parse_error(payload),
response_cost=as_float(payload.get("response_cost")),
identity=RequestContext.from_standard_logging_payload(payload).identity,
)
def _mcp_tool_call_metadata(payload: Mapping[str, object]) -> Mapping[str, object]:
"""The MCP gateway's tool-call metadata, which lives under
``StandardLoggingPayload.metadata`` (a ``StandardLoggingMetadata`` key), not
at the payload's top level."""
metadata = payload.get("metadata")
if not isinstance(metadata, Mapping):
return {}
meta = metadata.get("mcp_tool_call_metadata")
return meta if isinstance(meta, Mapping) else {}
def is_mcp_tool_call(payload: Mapping[str, object]) -> bool:
"""Whether a closed request's payload is an MCP tool call rather than an LLM
call — true when the MCP gateway stamped its tool-call metadata, or the call
type says so on a path that hasn't populated the metadata yet."""
return bool(_mcp_tool_call_metadata(payload)) or (
payload.get("call_type") == "call_mcp_tool"
)
# --- service event_metadata sanitization ------------------------------------ #
# Substrings (case-insensitive) of keys that must never reach a span: secrets,
# tokens, and raw request/response dumps the legacy service decorators pass.
_SENSITIVE_METADATA_SUBSTRINGS: tuple[str, ...] = (
"api_key",
"token",
"secret",
"password",
"cookie",
"authorization",
"header",
"hidden_params",
)
# Keys that carry raw call-site internals — live objects, full kwargs/args. The
# operation name is already the span's ``call_type``, so ``function_name`` is
# redundant.
_DROP_METADATA_KEYS: frozenset = frozenset(
{"function_kwargs", "function_args", "function_name"}
)
_MAX_METADATA_VALUE_LEN = 1024
_MAX_METADATA_ITEMS = 32
def sanitize_event_metadata(
event_metadata: Mapping[str, object] | None,
) -> dict[str, str]:
"""Reduce caller-supplied ``event_metadata`` to span-safe string attributes.
Keeps only primitive values (str/int/float/bool) under non-sensitive keys —
never ``repr()``-ing objects, dicts, or lists, never stamping secrets/headers,
and bounding the count and per-value length. This is the single chokepoint:
both the GenAI and legacy mappers read the cleaned result.
"""
if not event_metadata:
return {}
clean: dict[str, str] = {}
for key, value in event_metadata.items():
if len(clean) >= _MAX_METADATA_ITEMS:
break
if not isinstance(key, str) or key in _DROP_METADATA_KEYS:
continue
lowered = key.lower()
if any(token in lowered for token in _SENSITIVE_METADATA_SUBSTRINGS):
continue
# ``bool`` is a subclass of ``int``, so it's covered. Non-primitive values
# (objects, dicts, lists) are dropped rather than stringified.
if isinstance(value, (str, int, float)):
clean[key] = str(value)[:_MAX_METADATA_VALUE_LEN]
return clean
def _json_or_none(value: object) -> str | None:
"""JSON-serialize ``value`` (already-string values pass through). ``None`` on failure."""
if isinstance(value, str):
return value
try:
return json.dumps(value, default=str)
except Exception:
return None
def _guardrail_mode_str(value: object) -> str | None:
"""Normalize ``guardrail_mode`` to a stable string.
``guardrail_mode`` is typed as a ``GuardrailEventHooks`` enum, a list of them,
or a ``GuardrailMode`` — not a plain string. Emit the enum *value* (e.g.
``"pre_call"``) rather than ``str(enum)`` (``"GuardrailEventHooks.pre_call"``),
and join a list of modes so a guardrail that runs at multiple hooks is
represented faithfully.
"""
if value is None:
return None
if isinstance(value, (list, tuple)):
parts: list[str] = []
for item in value:
if item is None:
continue
part = as_str(item.value) if isinstance(item, Enum) else as_str(item)
if part:
parts.append(part)
return ",".join(parts) or None
if isinstance(value, Enum):
return as_str(value.value)
return as_str(value)
def _total_masked_entities(value: object) -> int | None:
"""``masked_entity_count`` is a ``{entity_type: count}`` map — sum to a total."""
if isinstance(value, Mapping):
total = sum(v for v in value.values() if isinstance(v, int))
return total or None
return as_int(value)
def _dicts(value: object) -> tuple[Mapping[str, object], ...]:
"""The dict items of ``value`` (when it's a list), as a tuple. Else empty."""
if not isinstance(value, list):
return ()
return tuple(item for item in value if isinstance(item, dict))
def _finish_reasons(choices: tuple[Mapping[str, object], ...]) -> tuple[str, ...]:
"""Non-empty ``finish_reason`` of each response choice."""
return tuple(r for c in choices if (r := as_str(c.get("finish_reason"))))
def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None:
"""A ``SpanError`` for a failed request, or ``None`` on success."""
if payload.get("status") != "failure":
return None
info = cast(Mapping[str, object], payload.get("error_information") or {})
return SpanError(
error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")),
message=as_str(info.get("error_message")) or as_str(payload.get("error_str")),
)
def _tool_from_entry(entry: object) -> ToolDefinition | None:
"""One ``tools``/``functions`` entry → ``ToolDefinition``, or ``None`` if unusable."""
if not isinstance(entry, dict):
return None
fn = entry.get("function") if "function" in entry else entry
if not isinstance(fn, dict):
return None
name = as_str(fn.get("name"))
if not name:
return None
params = fn.get("parameters")
parameters_json: str | None = None
if params is not None:
try:
parameters_json = json.dumps(params, default=str)
except Exception:
parameters_json = None
return ToolDefinition(
name=name,
description=as_str(fn.get("description")),
parameters_json=parameters_json,
)
def _extract_tools(
model_parameters: Mapping[str, object],
) -> tuple[ToolDefinition, ...]:
"""Pull declared tools from request params (OpenAI / Anthropic shape).
Accepts the chat-completion ``tools=[{"type":"function", "function":
{...}}, ...]`` shape, and falls back to the ``functions=[...]`` shape.
Returns an empty tuple when neither is present.
"""
raw_tools = model_parameters.get("tools")
if not isinstance(raw_tools, list):
raw_tools = model_parameters.get("functions") # ``functions`` shape
if not isinstance(raw_tools, list):
return ()
return tuple(t for entry in raw_tools if (t := _tool_from_entry(entry)) is not None)

273
litellm/integrations/otel/model/semconv.py Normal file
View file

@ -0,0 +1,273 @@
"""
Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anything
without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
"""
from enum import Enum
from typing import Final
class GenAIOperation(str, Enum):
"""Values for ``gen_ai.operation.name``."""
CHAT = "chat"
TEXT_COMPLETION = "text_completion"
EMBEDDINGS = "embeddings"
GENERATE_CONTENT = "generate_content"
CREATE_AGENT = "create_agent" # reserved for future agent spans
INVOKE_AGENT = "invoke_agent" # reserved for future agent spans
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
class GenAIProvider(str, Enum):
"""Common values for the ``gen_ai.provider.name`` attribute."""
OPENAI = "openai"
ANTHROPIC = "anthropic"
AWS_BEDROCK = "aws.bedrock"
AZURE_AI_OPENAI = "azure.ai.openai"
AZURE_AI_INFERENCE = "azure.ai.inference"
GCP_GEMINI = "gcp.gemini"
GCP_VERTEX_AI = "gcp.vertex_ai"
COHERE = "cohere"
MISTRAL_AI = "mistral_ai"
DEEPSEEK = "deepseek"
GROQ = "groq"
PERPLEXITY = "perplexity"
X_AI = "x_ai"
IBM_WATSONX_AI = "ibm.watsonx.ai"
class MCPMethod(str, Enum):
"""Well-known values for ``mcp.method.name`` that litellm's MCP gateway
serves. The value is the JSON-RPC method exactly as it travels on the wire."""
TOOLS_CALL = "tools/call"
TOOLS_LIST = "tools/list"
PROMPTS_GET = "prompts/get"
PROMPTS_LIST = "prompts/list"
class GenAI:
"""Canonical OTel GenAI span-attribute keys."""
# request
OPERATION_NAME: Final = "gen_ai.operation.name"
PROVIDER_NAME: Final = "gen_ai.provider.name"
REQUEST_MODEL: Final = "gen_ai.request.model"
REQUEST_TEMPERATURE: Final = "gen_ai.request.temperature"
REQUEST_TOP_P: Final = "gen_ai.request.top_p"
REQUEST_TOP_K: Final = "gen_ai.request.top_k"
REQUEST_MAX_TOKENS: Final = "gen_ai.request.max_tokens"
REQUEST_FREQUENCY_PENALTY: Final = "gen_ai.request.frequency_penalty"
REQUEST_PRESENCE_PENALTY: Final = "gen_ai.request.presence_penalty"
REQUEST_STOP_SEQUENCES: Final = "gen_ai.request.stop_sequences"
REQUEST_SEED: Final = "gen_ai.request.seed"
REQUEST_CHOICE_COUNT: Final = "gen_ai.request.choice.count"
REQUEST_ENCODING_FORMATS: Final = "gen_ai.request.encoding_formats"
# response
RESPONSE_ID: Final = "gen_ai.response.id"
RESPONSE_MODEL: Final = "gen_ai.response.model"
RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons"
# usage
USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
# content (opt-in, gated by capture mode)
INPUT_MESSAGES: Final = "gen_ai.input.messages"
OUTPUT_MESSAGES: Final = "gen_ai.output.messages"
SYSTEM_INSTRUCTIONS: Final = "gen_ai.system_instructions"
OUTPUT_TYPE: Final = "gen_ai.output.type"
CONVERSATION_ID: Final = "gen_ai.conversation.id"
# agent (reserved)
AGENT_ID: Final = "gen_ai.agent.id"
AGENT_NAME: Final = "gen_ai.agent.name"
# tool / tool-call (stamped on MCP tool-call spans). Arguments and result are
# the tool's input/output payloads — sensitive, so they're opt-in and gated by
# the same content-capture mode as prompt/response content.
TOOL_NAME: Final = "gen_ai.tool.name"
TOOL_CALL_ID: Final = "gen_ai.tool.call.id"
TOOL_CALL_ARGUMENTS: Final = "gen_ai.tool.call.arguments"
TOOL_CALL_RESULT: Final = "gen_ai.tool.call.result"
# prompt (MCP ``prompts/get`` etc.)
PROMPT_NAME: Final = "gen_ai.prompt.name"
class MCP:
"""OTel GenAI MCP (Model Context Protocol) span-attribute keys.
``METHOD_NAME`` is the only key litellm populates from a closed request today;
the rest are part of the convention's vocabulary and are stamped when the
corresponding signal (session, protocol version, resource) is available.
"""
METHOD_NAME: Final = "mcp.method.name"
SESSION_ID: Final = "mcp.session.id"
PROTOCOL_VERSION: Final = "mcp.protocol.version"
RESOURCE_URI: Final = "mcp.resource.uri"
class JsonRpc:
"""JSON-RPC keys carried on MCP spans. The error/status code lives in the
``rpc.*`` namespace per semconv, not ``jsonrpc.*``."""
REQUEST_ID: Final = "jsonrpc.request.id"
PROTOCOL_VERSION: Final = "jsonrpc.protocol.version"
RESPONSE_STATUS_CODE: Final = "rpc.response.status_code"
class NetworkTransport(str, Enum):
"""Well-known values for ``network.transport``."""
TCP = "tcp"
UDP = "udp"
QUIC = "quic"
UNIX = "unix"
PIPE = "pipe"
class Network:
"""OTel network keys, recommended on MCP spans to describe the transport
carrying the JSON-RPC messages (stdio pipe, HTTP, websocket, …)."""
PROTOCOL_NAME: Final = "network.protocol.name"
PROTOCOL_VERSION: Final = "network.protocol.version"
TRANSPORT: Final = "network.transport"
class Client:
"""Peer (client) network keys, stamped on MCP *server* spans the same way
``server.*`` is stamped on client spans."""
ADDRESS: Final = "client.address"
PORT: Final = "client.port"
class Error:
TYPE: Final = "error.type"
class Server:
ADDRESS: Final = "server.address"
PORT: Final = "server.port"
class DB:
"""Database / cache client-span keys (OTel ``db.*`` semconv).
Stamped on ``DB_CALL`` spans (redis / postgres), which are CLIENT spans for
outbound datastore calls — not on the INTERNAL ``SERVICE`` spans.
"""
SYSTEM_NAME: Final = "db.system.name"
OPERATION_NAME: Final = "db.operation.name"
class HTTP:
"""HTTP server-span keys. Belong on the SERVER span only (never promoted)."""
REQUEST_METHOD: Final = "http.request.method"
ROUTE: Final = "http.route"
RESPONSE_STATUS_CODE: Final = "http.response.status_code"
URL_PATH: Final = "url.path"
class LiteLLM:
"""Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
CALL_ID: Final = "litellm.call_id"
COST_PREFIX: Final = "litellm.cost."
METADATA_PREFIX: Final = "litellm.metadata."
TEAM_ID: Final = "litellm.team.id"
TEAM_ALIAS: Final = "litellm.team.alias"
# The team's free-form metadata dict, JSON-serialized into a single value.
TEAM_METADATA: Final = "litellm.team.metadata"
KEY_HASH: Final = "litellm.api_key.hash"
END_USER: Final = "litellm.end_user.id"
# The model string litellm actually sent to the provider (the deployment's
# ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``.
PROVIDER_MODEL: Final = "litellm.provider.model"
REQUEST_STREAMING: Final = "litellm.request.streaming"
GUARDRAIL_NAME: Final = "litellm.guardrail.name"
GUARDRAIL_MODE: Final = "litellm.guardrail.mode"
GUARDRAIL_STATUS: Final = "litellm.guardrail.status"
GUARDRAIL_PROVIDER: Final = "litellm.guardrail.provider"
GUARDRAIL_ACTION: Final = "litellm.guardrail.action"
GUARDRAIL_RESPONSE: Final = "litellm.guardrail.response"
GUARDRAIL_VIOLATION_CATEGORIES: Final = "litellm.guardrail.violation_categories"
GUARDRAIL_CONFIDENCE_SCORE: Final = "litellm.guardrail.confidence_score"
GUARDRAIL_RISK_SCORE: Final = "litellm.guardrail.risk_score"
GUARDRAIL_MASKED_ENTITY_COUNT: Final = "litellm.guardrail.masked_entity_count"
GUARDRAIL_DURATION: Final = "litellm.guardrail.duration"
GUARDRAIL_ID: Final = "litellm.guardrail.id"
GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"
GUARDRAIL_DETECTION_METHOD: Final = "litellm.guardrail.detection_method"
SERVICE_NAME: Final = "litellm.service.name"
SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
# The logical name of the MCP server a tool call was routed to. There is no
# semconv key for an MCP server's *name* (the convention uses ``server.address``
# for its network location), so it lives under the vendor namespace.
MCP_SERVER_NAME: Final = "litellm.mcp.server.name"
class Metric:
"""GenAI metric instrument names."""
TOKEN_USAGE: Final = "gen_ai.client.token.usage"
OPERATION_DURATION: Final = "gen_ai.client.operation.duration"
# litellm ``custom_llm_provider`` -> ``gen_ai.provider.name`` value.
_PROVIDER_BY_LITELLM: dict[str, GenAIProvider] = {
"openai": GenAIProvider.OPENAI,
"text-completion-openai": GenAIProvider.OPENAI,
"azure": GenAIProvider.AZURE_AI_OPENAI,
"azure_ai": GenAIProvider.AZURE_AI_INFERENCE,
"anthropic": GenAIProvider.ANTHROPIC,
"bedrock": GenAIProvider.AWS_BEDROCK,
"bedrock_converse": GenAIProvider.AWS_BEDROCK,
"vertex_ai": GenAIProvider.GCP_VERTEX_AI,
"vertex_ai_beta": GenAIProvider.GCP_VERTEX_AI,
"gemini": GenAIProvider.GCP_GEMINI,
"cohere": GenAIProvider.COHERE,
"cohere_chat": GenAIProvider.COHERE,
"mistral": GenAIProvider.MISTRAL_AI,
"deepseek": GenAIProvider.DEEPSEEK,
"groq": GenAIProvider.GROQ,
"perplexity": GenAIProvider.PERPLEXITY,
"xai": GenAIProvider.X_AI,
"watsonx": GenAIProvider.IBM_WATSONX_AI,
}
# litellm ``call_type`` -> ``gen_ai.operation.name``.
_OPERATION_BY_CALL_TYPE: dict[str, GenAIOperation] = {
"completion": GenAIOperation.CHAT,
"acompletion": GenAIOperation.CHAT,
"completion_with_retries": GenAIOperation.CHAT,
"text_completion": GenAIOperation.TEXT_COMPLETION,
"atext_completion": GenAIOperation.TEXT_COMPLETION,
"embedding": GenAIOperation.EMBEDDINGS,
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
}
def resolve_provider(custom_llm_provider: str | None) -> str:
"""Map a litellm provider string to a ``gen_ai.provider.name`` value.
Unknown providers pass through verbatim — the convention explicitly allows
provider-specific values, so an unmapped name is still valid.
"""
if not custom_llm_provider:
return ""
mapped = _PROVIDER_BY_LITELLM.get(custom_llm_provider.lower())
return mapped.value if mapped is not None else custom_llm_provider
def resolve_operation(call_type: str | None) -> GenAIOperation:
"""Map a litellm ``call_type`` to a ``gen_ai.operation.name`` value."""
if not call_type:
return GenAIOperation.CHAT
return _OPERATION_BY_CALL_TYPE.get(call_type.lower(), GenAIOperation.CHAT)

215
litellm/integrations/otel/model/spans.py Normal file
View file

@ -0,0 +1,215 @@
"""
This module declares every span the instrumentation can emit and the hierarchy.
Span-name patterns live here as typed builder functions.
Canonical hierarchy::
PROXY_REQUEST (SERVER, root) # owned by the FastAPI instrumentor
├── SERVICE (INTERNAL) # auth phase span (live; see logger.phase_span)
│ └── DB_CALL (CLIENT) # its key/user/team lookups nest here
├── GUARDRAIL (INTERNAL) # request-lifecycle hook, sibling of LLM_CALL
├── LLM_CALL (CLIENT)
└── DB_CALL (CLIENT) # e.g. the spend-log write
Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
hooks are orchestrated by the request lifecycle (a pre-call guardrail runs
before the LLM call even starts), so a guardrail is a sibling of the LLM call,
not a child of it. The emitter parents every span to the ambient OTel context
(the active server span), which matches this.
Not every service call becomes a span — :func:`span_role_for_service` decides:
- ``DB_CALL`` (CLIENT) — outbound datastores (redis, postgres,
``batch_write_to_db``), carrying ``db.*`` semconv.
- ``SERVICE`` (INTERNAL) — genuine internal work worth a span (background
budget/reset jobs, pod-lock manager).
- ``None`` (metrics-only) — framework instrumentation that duplicates a gen-AI
span (``self`` = the ``track_llm_api_timing`` wrapper, ``router``,
``proxy_pre_call``) or ``auth`` (which gets a live phase span instead). These
still feed Prometheus/Datadog; they just never enter the trace.
``DB_CALL`` and ``SERVICE`` are built from the same ``ServiceSpanData``; only the
role (hence span kind and attribute vocabulary) differs. A service call can fire
outside any request (a background job), in which case it parents to no server
span and starts its own root trace rather than being dropped.
Management/admin endpoints are ordinary FastAPI routes — their SERVER spans are
owned by the instrumentor too, so they don't appear as a role here.
"""
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
MCPToolCallSpanData,
ProxyRequestSpanData,
ServiceSpanData,
)
class SpanRole(str, Enum):
PROXY_REQUEST = "proxy_request"
LLM_CALL = "llm_call"
MCP_TOOL_CALL = "mcp_tool_call"
GUARDRAIL = "guardrail"
DB_CALL = "db_call"
SERVICE = "service"
class LiteLLMSpanKind(str, Enum):
SERVER = "server"
CLIENT = "client"
INTERNAL = "internal"
PRODUCER = "producer"
CONSUMER = "consumer"
@dataclass(frozen=True)
class SpanSpec:
role: SpanRole
kind: LiteLLMSpanKind
parent: SpanRole | None
SPAN_REGISTRY: dict[SpanRole, SpanSpec] = {
SpanRole.PROXY_REQUEST: SpanSpec(
SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None
),
SpanRole.LLM_CALL: SpanSpec(
SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST
),
# The proxy is an MCP client to the upstream server it dispatches the tool
# call to, so this is a CLIENT span, sibling of the LLM call under the request.
SpanRole.MCP_TOOL_CALL: SpanSpec(
SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST
),
SpanRole.GUARDRAIL: SpanSpec(
SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST
),
SpanRole.DB_CALL: SpanSpec(
SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST
),
SpanRole.SERVICE: SpanSpec(
SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST
),
}
# ``ServiceTypes`` value -> ``db.system.name``. These are outbound datastore
# calls and become CLIENT ``DB_CALL`` spans; ``redis_``-prefixed names cover the
# redis-backed spend queues. Any service not mapped here is litellm-internal work
# and stays an INTERNAL ``SERVICE`` span. This table is the single source of
# datastore knowledge — both the role classifier and the mapper read it.
_DB_SYSTEM_BY_SERVICE: dict[str, str] = {
"redis": "redis",
"postgres": "postgresql",
"batch_write_to_db": "postgresql",
}
def db_system(service_name: str) -> str | None:
"""The ``db.system.name`` for a datastore service, else ``None``.
``None`` means the service is not an outbound datastore call. Redis-backed
spend queues (``redis_*``) map to ``redis``.
"""
if service_name in _DB_SYSTEM_BY_SERVICE:
return _DB_SYSTEM_BY_SERVICE[service_name]
if service_name.startswith("redis_"):
return "redis"
return None
# ``ServiceTypes`` values that are NOT emitted as spans — they are framework
# instrumentation that either duplicates a gen-AI span or has a better home as a
# Prometheus/Datadog metric. They still flow to those metric backends via their
# own hooks; the v2 logger just does not put them in the trace:
#
# - ``self`` — ``track_llm_api_timing`` wraps the LLM call; the
# ``chat {model}`` CLIENT span already represents it.
# - ``router`` — wraps the whole request; duplicates the server span.
# - ``proxy_pre_call`` — per-callback pre-call timing; a guardrail's real span
# is ``execute_guardrail {name}``.
# - ``auth`` — emitted instead as a live phase span (see
# ``logger.phase_span``) so its DB lookups nest under it,
# not as a flat post-hoc service span.
_METRICS_ONLY_SERVICES: frozenset[str] = frozenset(
{"self", "router", "proxy_pre_call", "auth"}
)
def span_role_for_service(service_name: str) -> SpanRole | None:
"""The span role for a service call, or ``None`` when it must not be a span.
``DB_CALL`` for outbound datastores, ``SERVICE`` for genuine internal work
worth a span (background jobs), and ``None`` for framework instrumentation
that duplicates a gen-AI span or belongs in metrics only
(see ``_METRICS_ONLY_SERVICES``).
"""
if service_name in _METRICS_ONLY_SERVICES:
return None
return SpanRole.DB_CALL if db_system(service_name) is not None else SpanRole.SERVICE
# --- span name builders (the naming convention, per role) ------------------- #
# The name the FastAPI instrumentor gives the root server span. V2 never creates
# this span (the instrumentor owns it), but it anchors request-level spans to it
# and tests assert against it by name, so the literal lives here with the rest of
# the span vocabulary rather than being duplicated at each call site.
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
def llm_call_span_name(data: "LLMCallSpanData") -> str:
"""``"{operation} {model}"`` e.g. ``"chat gpt-4o"`` (GenAI semconv)."""
model = data.request_model or ""
return f"{data.operation.value} {model}".strip()
def mcp_tool_call_span_name(data: "MCPToolCallSpanData") -> str:
"""``"{mcp.method.name} {tool}"`` e.g. ``"tools/call get-weather"`` (MCP semconv)."""
return f"{data.method} {data.tool_name}".strip()
def proxy_request_span_name(data: "ProxyRequestSpanData") -> str:
"""``"{method} {route}"`` (HTTP semconv)."""
return f"{data.http_method} {data.route}".strip()
def guardrail_span_name(data: "GuardrailSpanData") -> str:
return f"execute_guardrail {data.guardrail_name}".strip()
def service_span_name(data: "ServiceSpanData") -> str:
"""``"{service} {call_type}"`` e.g. ``"redis set"`` — service name alone when
no call type is known, so identically-named calls stay distinguishable."""
return f"{data.service_name} {data.call_type or ''}".strip()
def root_roles() -> list[SpanRole]:
"""Roles that start a new trace (no in-process parent)."""
return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
def child_roles(parent: SpanRole) -> list[SpanRole]:
return [role for role, spec in SPAN_REGISTRY.items() if spec.parent == parent]
def validate_registry(
registry: dict[SpanRole, SpanSpec] | None = None,
) -> None:
reg = registry if registry is not None else SPAN_REGISTRY
for role, spec in reg.items():
if spec.role is not role:
raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
if spec.parent is not None and spec.parent not in reg:
raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
missing = [role for role in SpanRole if role not in reg]
if missing:
raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")

103
litellm/integrations/otel/model/utils.py Normal file
View file

@ -0,0 +1,103 @@
"""Shared, OpenTelemetry-free helpers for the otel integration.
Generic value coercion (for reading heterogeneous logging-payload dicts), time
conversion, and header parsing — pulled out of the individual modules so they
live in one place. Deliberately free of any ``opentelemetry`` import so the
OTel-free sources of truth (payloads, semconv, spans, config) can use it too.
"""
from datetime import datetime
def as_str(value: object) -> str | None:
if value is None:
return None
if isinstance(value, str):
return value
return str(value)
def as_int(value: object) -> int | None:
if isinstance(value, bool):
return int(value)
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value)
if isinstance(value, str):
try:
return int(value)
except ValueError:
return None
return None
def as_float(value: object) -> float | None:
if isinstance(value, bool):
return float(value)
if isinstance(value, (int, float)):
return float(value)
if isinstance(value, str):
try:
return float(value)
except ValueError:
return None
return None
def as_bool(value: object) -> bool | None:
if value is None:
return None
if isinstance(value, bool):
return value
return bool(value)
def as_str_tuple(value: object) -> tuple[str, ...] | None:
if value is None:
return None
if isinstance(value, str):
return (value,)
if isinstance(value, (list, tuple)):
return tuple(str(v) for v in value)
return None
def to_ns(value: datetime | float | int | None) -> int | None:
"""Coerce a datetime / epoch value to integer nanoseconds."""
if value is None:
return None
if isinstance(value, datetime):
return int(value.timestamp() * 1e9)
if isinstance(value, (int, float)) and not isinstance(value, bool):
return int(float(value) * 1e9)
return None
def to_seconds(value: datetime | float | int | str | None) -> float | None:
"""Coerce a datetime / epoch / formatted-string value to epoch seconds."""
if value is None:
return None
if isinstance(value, datetime):
return value.timestamp()
if isinstance(value, (int, float)) and not isinstance(value, bool):
return float(value)
if isinstance(value, str):
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
try:
return datetime.strptime(value, fmt).timestamp()
except ValueError:
continue
return None
def parse_headers(raw: str | None) -> dict[str, str]:
"""Parse an OTLP ``"k=v,k=v"`` header string into a dict."""
headers: dict[str, str] = {}
if not raw:
return headers
for pair in raw.split(","):
if "=" in pair:
key, _, value = pair.partition("=")
headers[key.strip()] = value.strip()
return headers

130
litellm/integrations/otel/mount.py Normal file
View file

@ -0,0 +1,130 @@
"""FastAPI server-span instrumentation — the proxy mounts this at app creation.
``opentelemetry-instrumentation-fastapi`` creates the SERVER span for each HTTP
route and extracts inbound ``traceparent`` headers. This module owns the one call
site that attaches it to the proxy app, plus the passthrough span-naming hook, so
``proxy_server`` stays free of OTel details.
The ``FastAPIInstrumentor`` import is kept lazy (inside :func:`instrument_fastapi_app`,
after the gate check) so importing this module never requires the optional
``opentelemetry-instrumentation-fastapi`` package and pulls in nothing OTel-related
when the feature gate is off.
"""
import os
from typing import Any
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import is_otel_v2_enabled
# Routes excluded from server-span tracing by default: high-frequency pollers and
# static UI/docs assets, none of which are LLM traffic. Entries are substring-matched
# against the request path (unanchored, so they survive a ``server_root_path`` prefix
# and each entry also covers everything beneath it — e.g. ``/health`` covers
# ``/health/readiness``). Operators override the whole set via the standard
# ``OTEL_PYTHON_FASTAPI_EXCLUDED_URLS`` env var (set "" to trace everything).
_DEFAULT_EXCLUDED_ROUTES = (
"/health", # load-balancer liveness/readiness polling
"/metrics", # Prometheus scrape (also drops the /model/metrics admin analytics)
"/litellm-asset-prefix", # hashed UI asset bundles
"/_next", # Next.js static JS/CSS chunks (root-level mount)
"/ui", # admin UI single-page app
"/swagger", # static Swagger UI assets
"/docs", # FastAPI Swagger docs page
"/redoc", # FastAPI ReDoc docs page
"/openapi.json", # OpenAPI schema
"favicon", # /favicon.ico + /get_favicon
"/.well-known", # UI config discovery
)
_DEFAULT_EXCLUDED_URLS = ",".join(_DEFAULT_EXCLUDED_ROUTES)
# Passthrough routes are catch-alls (e.g. "/openai/{endpoint:path}"), so the
# default OTel server-span name "{method} {route}" collapses every upstream
# endpoint into "POST /openai/{endpoint:path}". The hook below renames those spans
# to the real request path so each endpoint is distinguishable. Non-catch-all
# routes keep their low-cardinality template name.
PASSTHROUGH_PREFIXES = frozenset(
{
"openai",
"openai_passthrough",
"anthropic",
"azure",
"azure_ai",
"bedrock",
"cohere",
"cursor",
"gemini",
"mistral",
"vllm",
"vertex_ai",
"vertex-ai",
"assemblyai",
"eu.assemblyai",
"milvus",
}
)
def _passthrough_span_name_hook(span: Any, scope: dict) -> None:
"""FastAPI ``server_request_hook``: give passthrough server spans a useful name.
The instrumentation matches the route at span creation, so both the span name
and ``http.route`` are set to the catch-all template (``/openai/{endpoint:path}``)
before this hook runs. Rewrite both to the real request path so each upstream
endpoint is distinguishable. (The ASGI ``http receive``/``http send`` sub-spans
can't be renamed from here — their name is captured at creation — so they are
dropped via ``exclude_spans`` at instrumentation time.)
"""
try:
if span is None or not span.is_recording():
return
path = scope.get("path") or ""
method = scope.get("method") or ""
first_segment = path.lstrip("/").split("/", 1)[0]
if first_segment in PASSTHROUGH_PREFIXES:
span.update_name(f"{method} {path}".strip())
span.set_attribute("http.route", path)
except Exception:
pass
def instrument_fastapi_app(app: Any) -> None:
"""Attach OTel server-span instrumentation to the proxy FastAPI app.
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
after config load (see ``proxy_startup_event``), and the proxy delegates to it.
That way server spans and gen-ai spans share one provider and the same trace.
"""
try:
if not is_otel_v2_enabled():
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the
# base ``litellm[proxy]`` install. Importing it at module top would make
# ``proxy_server``'s unconditional ``import`` of this module crash when the
# package is absent, even with the gate off.
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
excluded_urls = (
os.environ.get("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS")
if "OTEL_PYTHON_FASTAPI_EXCLUDED_URLS" in os.environ
else _DEFAULT_EXCLUDED_URLS
)
FastAPIInstrumentor.instrument_app(
app,
excluded_urls=excluded_urls,
server_request_hook=_passthrough_span_name_hook,
# Drop the ASGI "http receive"/"http send" lifecycle sub-spans: they
# are low-value noise and (for passthrough) carry the catch-all route
# template in their name, which can't be rewritten from a hook.
exclude_spans=["receive", "send"],
)
except Exception as e:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: %s", e)

0
litellm/integrations/otel/plumbing/__init__.py Normal file
View file

127
litellm/integrations/otel/plumbing/context.py Normal file
View file

@ -0,0 +1,127 @@
"""Trace-context + Baggage helpers."""
from contextvars import ContextVar
from typing import Mapping
from opentelemetry import baggage
from opentelemetry.context import Context, get_current
from opentelemetry.trace import Span, get_current_span, set_span_in_context
from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
_PROPAGATOR = TraceContextTextMapPropagator()
# The request's root span — the FastAPI-owned SERVER span — captured ONCE when the
# proxy first resolves it, so request-level spans (the LLM call, guardrails) can
# parent to it EXPLICITLY instead of to whatever span happens to be active at the
# instant they are emitted. Ambient-only parenting (``get_current_span()``) is
# wrong at two boundaries:
# * inside the ``auth`` phase span the active span is the auth span, so an LLM /
# guardrail span emitted there would nest under auth instead of being its
# sibling; and
# * in a detached success task (pass-through logs success from a fire-and-forget
# ``asyncio.create_task``) the server span may not be active at all, orphaning
# the span into a brand-new trace.
# A ``ContextVar`` (not a request attribute) so it rides the request task's context
# and is inherited by ``asyncio.create_task`` children — i.e. the async logging
# callbacks that close the span. It is never reset: the contextvar dies with the
# request task, so there is nothing to leak.
_request_root_span: "ContextVar[Span | None]" = ContextVar(
"litellm_otel_request_root_span", default=None
)
def set_request_root_span(span: Span) -> None:
"""Anchor the request's root (server) span for explicit child parenting.
No-ops for a non-recordable span so a bad capture can never replace a good one
with a phantom parent. Idempotent — the proxy captures the same server span at
more than one entry point.
"""
if is_recordable_span(span):
_request_root_span.set(span)
def request_root_span() -> "Span | None":
"""The anchored request root span, or ``None`` outside a proxy request."""
span = _request_root_span.get()
return span if is_recordable_span(span) else None
def set_request_baggage(
values: Mapping[str, str], context: Context | None = None
) -> Context:
"""Return a context with ``values`` written into Baggage."""
ctx = context
for key, value in values.items():
ctx = baggage.set_baggage(key, value, context=ctx)
return ctx if ctx is not None else (context or get_current())
def get_baggage_attributes(context: Context | None = None) -> dict[str, str]:
"""All Baggage entries on ``context`` as strings."""
return {key: str(value) for key, value in baggage.get_all(context).items()}
def context_from_span(span: Span, context: Context | None = None) -> Context:
"""A context with ``span`` as the active span (for explicit parenting)."""
return set_span_in_context(span, context=context)
def resolve_parent_context(threaded: Span | None = None) -> Context:
"""The context a child span should parent under.
Ambient-first: parent to the active OTel context (the server span, restored
by the logging worker or active in the request task), falling back to a span
passed explicitly (``threaded``) only when the ambient context has no
recordable span — e.g. a background service call with no request on the
stack. When neither is recordable the ambient context is returned unchanged,
so the span starts a new root trace.
Only service/DB spans pass ``threaded`` (the ``parent_otel_span`` handed to
the service hook). Request-level spans — the LLM call and guardrails — are
created where the server span is genuinely ambient, so they never need it.
"""
ctx = get_current()
if is_recordable_span(threaded) and not is_recordable_span(get_current_span(ctx)):
ctx = context_from_span(threaded, context=ctx) # type: ignore[arg-type]
return ctx
def resolve_request_span_context() -> Context:
"""The parent context for a request-level span (the LLM call, a guardrail).
These are direct children of the request's root server span — siblings of the
``auth`` phase span and of each other, never nested under whatever span is
momentarily active. So prefer the explicitly anchored root span; fall back to
ambient context only when there is no anchor (the SDK / no-proxy path), where
the span legitimately starts its own root trace.
Unlike :func:`resolve_parent_context` (used by DB/service spans, which DO want
to nest under the active phase span, e.g. an auth DB lookup under ``auth``),
this never returns the active span when an anchor exists.
"""
root = request_root_span()
if root is not None:
return context_from_span(root)
return get_current()
def is_recordable_span(obj: object) -> bool:
"""True if ``obj`` is a live span with a valid context (safe to parent under)."""
if not isinstance(obj, Span):
return False
try:
ctx = obj.get_span_context()
except Exception:
return False
return ctx is not None and ctx.is_valid
def extract_traceparent(headers: Mapping[str, str]) -> Context | None:
"""Extract a remote parent context from incoming HTTP headers, if present."""
if not any(key.lower() == "traceparent" for key in headers):
return None
carrier = {str(key).lower(): value for key, value in headers.items()}
return _PROPAGATOR.extract(carrier)

28
litellm/integrations/otel/plumbing/metrics.py Normal file
View file

@ -0,0 +1,28 @@
"""GenAI client metrics (token usage + operation duration histograms)."""
from dataclasses import dataclass
from opentelemetry.metrics import Histogram, Meter
from litellm.integrations.otel.model.semconv import Metric
@dataclass(frozen=True)
class GenAIMetrics:
token_usage: Histogram
operation_duration: Histogram
def create_genai_metrics(meter: Meter) -> GenAIMetrics:
return GenAIMetrics(
token_usage=meter.create_histogram(
name=Metric.TOKEN_USAGE,
unit="{token}",
description="Number of tokens used per GenAI request.",
),
operation_duration=meter.create_histogram(
name=Metric.OPERATION_DURATION,
unit="s",
description="GenAI operation duration.",
),
)

220
litellm/integrations/otel/plumbing/providers.py Normal file
View file

@ -0,0 +1,220 @@
"""Provider / exporter factory + the Baggage span processor."""
from typing import Callable, Iterable
from opentelemetry import baggage
from opentelemetry.context import Context
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import (
BatchSpanProcessor,
ConsoleSpanExporter,
SimpleSpanProcessor,
SpanExporter,
)
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from opentelemetry.trace import Span, SpanKind, Tracer
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.model.semconv import LiteLLM
from litellm.integrations.otel.model.spans import LiteLLMSpanKind
# Re-exported so ``providers.parse_headers`` remains a stable entry point.
from litellm.integrations.otel.model.utils import parse_headers as parse_headers
_SPAN_KIND_BY_ROLE_KIND: dict[LiteLLMSpanKind, SpanKind] = {
LiteLLMSpanKind.SERVER: SpanKind.SERVER,
LiteLLMSpanKind.CLIENT: SpanKind.CLIENT,
LiteLLMSpanKind.INTERNAL: SpanKind.INTERNAL,
LiteLLMSpanKind.PRODUCER: SpanKind.PRODUCER,
LiteLLMSpanKind.CONSUMER: SpanKind.CONSUMER,
}
def to_otel_span_kind(kind: LiteLLMSpanKind) -> SpanKind:
return _SPAN_KIND_BY_ROLE_KIND[kind]
# Custom exporter factories keyed by ``ExporterSpec.kind``. A preset registers
# one here when its destination needs construction logic the built-in kinds
# can't express — e.g. an exporter that fetches an auth token lazily on its
# first export (off the event loop) instead of blocking at config-build time.
# Keeping the registry here lets this module stay vendor-agnostic: the factory
# lives with the integration that needs it.
_EXPORTER_FACTORIES: dict[str, Callable[[ExporterSpec], SpanExporter]] = {}
def register_exporter_factory(
kind: str, factory: Callable[[ExporterSpec], SpanExporter]
) -> None:
"""Register a custom exporter ``factory`` for the exporter ``kind``."""
_EXPORTER_FACTORIES[kind.lower()] = factory
class LiteLLMBaggageSpanProcessor(SpanProcessor):
"""Stamps an allowlisted set of Baggage entries onto every span at start."""
def __init__(
self,
allowed_keys: Iterable[str],
allowed_prefixes: tuple[str, ...] = (LiteLLM.METADATA_PREFIX,),
) -> None:
self._allowed_keys = frozenset(allowed_keys)
self._allowed_prefixes = tuple(allowed_prefixes)
def _is_allowed(self, key: str) -> bool:
return key in self._allowed_keys or any(
key.startswith(prefix) for prefix in self._allowed_prefixes
)
def on_start(self, span: Span, parent_context: Context | None = None) -> None:
for key, value in baggage.get_all(parent_context).items():
if self._is_allowed(key) and isinstance(value, (str, bool, int, float)):
span.set_attribute(key, value)
def on_end(self, span: ReadableSpan) -> None: # noqa: D401 - no-op
return None
def shutdown(self) -> None:
return None
def force_flush(self, timeout_millis: int = 30000) -> bool:
return True
def _otlp_traces_endpoint(endpoint: str | None) -> str | None:
"""Point an OTLP/HTTP base endpoint at the ``/v1/traces`` signal path.
``OTEL_EXPORTER_OTLP_ENDPOINT`` is a base URL (e.g. ``http://host:4318``).
The OTLP/HTTP exporter only appends the ``/v1/traces`` path when it reads
that env var itself; when an endpoint is passed explicitly it is used
verbatim, so a base URL would POST to the root and the collector returns
404. Append the signal path here (leaving an already-correct path intact).
"""
if not endpoint:
return endpoint
endpoint = endpoint.rstrip("/")
# Splunk Observability uses ``/v2/trace/otlp``; never rewrite it.
if endpoint.endswith("/v1/traces") or "/v2/trace/otlp" in endpoint:
return endpoint
for other_signal in ("/v1/logs", "/v1/metrics"):
if endpoint.endswith(other_signal):
return endpoint[: -len(other_signal)] + "/v1/traces"
return endpoint + "/v1/traces"
def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
kind = (spec.kind or "console").lower()
factory = _EXPORTER_FACTORIES.get(kind)
if factory is not None:
return factory(spec)
if kind in ("in_memory", "inmemory", "memory"):
return InMemorySpanExporter()
if kind in ("otlp_http", "http", "http/protobuf", "http/json"):
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as HTTPExporter,
)
return HTTPExporter(
endpoint=_otlp_traces_endpoint(spec.endpoint),
headers=parse_headers(spec.headers),
)
if kind in ("otlp_grpc", "grpc"):
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
OTLPSpanExporter as GRPCExporter,
)
return GRPCExporter(endpoint=spec.endpoint, headers=parse_headers(spec.headers))
return ConsoleSpanExporter()
def _processor_for(exporter: SpanExporter, use_simple: bool | None) -> SpanProcessor:
"""Pick a Simple or Batch span processor for ``exporter``.
When ``use_simple`` is unset, default to Simple for console and in-memory
exporters (spans export synchronously, which tests rely on) and Batch for
everything else (the right export semantics for production).
"""
if use_simple is None:
use_simple = isinstance(exporter, (ConsoleSpanExporter, InMemorySpanExporter))
return SimpleSpanProcessor(exporter) if use_simple else BatchSpanProcessor(exporter)
def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
"""Build a single exporter from the top-level config fields.
Convenience for the common single-exporter case (and for tests): reads the
``exporter`` / ``endpoint`` / ``headers`` fields. To configure multiple
exporters, populate ``config.exporters`` directly.
"""
return _exporter_from_spec(
ExporterSpec(
kind=config.exporter, endpoint=config.endpoint, headers=config.headers
)
)
def build_resource(config: OpenTelemetryV2Config) -> Resource:
attributes: dict[str, str] = {"service.name": config.service_name}
if config.deployment_environment:
attributes["deployment.environment"] = config.deployment_environment
attributes.update(config.resource_attributes)
return Resource.create(attributes)
def build_tracer_provider(
config: OpenTelemetryV2Config,
exporter: SpanExporter | None = None,
baggage_processor: SpanProcessor | None = None,
use_simple_processor: bool | None = None,
) -> TracerProvider:
"""Build the shared :class:`TracerProvider`.
Attach the Baggage processor first (so identity attributes land on each
span before any export decision), then add one ``SpanProcessor`` per
``config.exporters`` entry — this is what fans spans out to multiple
backends. ``exporter`` and ``use_simple_processor`` are explicit overrides:
pass a single exporter to attach exactly that one (used by tests).
"""
provider = TracerProvider(resource=build_resource(config))
if baggage_processor is None:
baggage_processor = LiteLLMBaggageSpanProcessor(
allowed_keys=config.baggage_promoted_keys
)
provider.add_span_processor(baggage_processor)
if exporter is not None:
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
return provider
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
for spec in config.exporters:
exp = _exporter_from_spec(spec)
provider.add_span_processor(
_processor_for(
exp,
(
spec.use_simple_processor
if spec.use_simple_processor is not None
else use_simple_processor
),
)
)
return provider
def get_tracer(provider: TracerProvider, name: str = "litellm") -> Tracer:
return provider.get_tracer(name)
def in_memory_provider(
config: OpenTelemetryV2Config | None = None,
) -> tuple[TracerProvider, InMemorySpanExporter]:
"""Convenience for tests: a provider exporting to an in-memory buffer."""
cfg = config or OpenTelemetryV2Config(exporter="in_memory")
exporter = InMemorySpanExporter()
provider = build_tracer_provider(cfg, exporter=exporter)
return provider, exporter

101
litellm/integrations/otel/plumbing/routing.py Normal file
View file

@ -0,0 +1,101 @@
"""Per-request multi-tenant tracer routing.
When a request carries team/key vendor credentials in
``standard_callback_dynamic_params``, its spans must export through a
``TracerProvider`` whose OTLP headers carry those credentials.
``TenantTracerCache`` builds and caches one provider per distinct credential
set, and otherwise hands back the logger's default tracer. This lets a single
logger fan requests out to many tenants without needing a logger per tenant.
"""
from collections import OrderedDict
from typing import Any, Mapping
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.trace import Tracer
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.presets import dynamic_otlp_headers
from litellm.integrations.otel.plumbing.providers import (
build_tracer_provider,
get_tracer,
)
# Exporter kinds that ignore headers — never rewritten with dynamic credentials.
_NON_OTLP_KINDS = ("console", "in_memory", "inmemory", "memory")
# Cap on distinct credential-scoped providers held at once. ``dynamic_params``
# can be populated from request metadata, so an unbounded cache lets a caller
# spawn one ``TracerProvider`` (plus its ``BatchSpanProcessor`` background
# thread) per unique credential set and exhaust the proxy. The LRU bound keeps
# the working set of active tenants resident while flushing and shutting down
# evicted providers so their threads are reclaimed.
_MAX_CACHED_PROVIDERS = 256
def _shutdown_provider(provider: TracerProvider) -> None:
"""Flush + stop an evicted provider's processors (reclaims their threads).
``TracerProvider.shutdown`` force-flushes each ``SpanProcessor`` before
stopping it, so any spans already handed to a ``BatchSpanProcessor`` are
exported rather than dropped. Best-effort: a shutdown failure must not break
the request that triggered the eviction.
"""
try:
provider.shutdown()
except Exception as e: # pragma: no cover - defensive
verbose_logger.debug("OTel V2: error shutting down evicted provider: %s", e)
class TenantTracerCache:
"""Credential-scoped ``TracerProvider`` cache keyed by the dynamic headers."""
def __init__(
self,
config: OpenTelemetryV2Config,
callback_name: str | None,
tracer_name: str,
) -> None:
self._config = config
self._callback_name = callback_name
self._tracer_name = tracer_name
self._providers: "OrderedDict[tuple[tuple[str, str], ...], TracerProvider]" = (
OrderedDict()
)
def tracer_for(self, default: Tracer, dynamic_params: Any) -> Tracer:
"""Return the tracer for this request.
Use ``default`` unless the request's dynamic credentials require a
credential-scoped tracer, in which case build (or reuse) one. The cache
is a bounded LRU: the least-recently-used provider is flushed and shut
down on overflow so its exporter threads don't accumulate.
"""
headers = dynamic_otlp_headers(self._callback_name, dynamic_params)
if not headers:
return default
cache_key = tuple(sorted(headers.items()))
provider = self._providers.get(cache_key)
if provider is not None:
self._providers.move_to_end(cache_key)
else:
provider = build_tracer_provider(self._config_with_headers(headers))
self._providers[cache_key] = provider
if len(self._providers) > _MAX_CACHED_PROVIDERS:
_, evicted = self._providers.popitem(last=False)
_shutdown_provider(evicted)
return get_tracer(provider, self._tracer_name)
def _config_with_headers(self, headers: Mapping[str, str]) -> OpenTelemetryV2Config:
"""Clone the config, replacing OTLP exporter headers with ``headers``."""
header_str = ",".join(f"{key}={value}" for key, value in headers.items())
exporters = [
(
spec
if spec.kind.lower() in _NON_OTLP_KINDS
else spec.model_copy(update={"headers": header_str})
)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})

78
litellm/integrations/otel/presets/__init__.py Normal file
View file

@ -0,0 +1,78 @@
"""Integration presets — each one returns an :class:`OpenTelemetryV2Config`.
A preset is a callable that reads an integration's env vars and returns an
``OpenTelemetryV2Config`` describing the exporter destination, the mapper
vocabularies to apply, and any resource attributes. ``PRESET_BY_CALLBACK``
maps a callback name (``"arize"``, ``"langfuse_otel"``, ...) to its preset so
the factory in ``litellm_logging`` can resolve a name and build a single
``OpenTelemetryV2`` instance from the result.
"""
from typing import Callable
from litellm.integrations.otel.presets.agentops import agentops_preset
from litellm.integrations.otel.presets.arize import arize_dynamic_headers, arize_preset
from litellm.integrations.otel.presets.base import Preset
from litellm.integrations.otel.presets.langfuse import (
langfuse_dynamic_headers,
langfuse_preset,
)
from litellm.integrations.otel.presets.langtrace import langtrace_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.phoenix import phoenix_preset
from litellm.integrations.otel.presets.weave import weave_dynamic_headers, weave_preset
from litellm.types.utils import StandardCallbackDynamicParams
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
#: registered value matches the preset interface.
PRESET_BY_CALLBACK: dict[str, Preset] = {
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"weave_otel": weave_preset,
}
#: Callback name → per-request OTLP header builder (team/key multi-tenant
#: routing). Only integrations that support dynamic credentials appear here —
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
#: default tracer.
DYNAMIC_HEADERS_BY_CALLBACK: dict[
str, Callable[[StandardCallbackDynamicParams], dict[str, str]]
] = {
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
def dynamic_otlp_headers(
callback_name: str | None,
dynamic_params: StandardCallbackDynamicParams | None,
) -> dict[str, str] | None:
"""Per-request OTLP headers for ``callback_name``, or ``None`` if N/A.
``None`` means "no per-request routing" — the caller uses its default tracer.
"""
builder = DYNAMIC_HEADERS_BY_CALLBACK.get(callback_name or "")
if builder is None or not dynamic_params:
return None
headers = builder(dynamic_params)
return headers or None
__all__ = [
"PRESET_BY_CALLBACK",
"DYNAMIC_HEADERS_BY_CALLBACK",
"Preset",
"dynamic_otlp_headers",
"agentops_preset",
"arize_preset",
"langfuse_preset",
"langtrace_preset",
"levo_preset",
"phoenix_preset",
"weave_preset",
]

139
litellm/integrations/otel/presets/agentops.py Normal file
View file

@ -0,0 +1,139 @@
"""AgentOps preset — OTLP/HTTP to AgentOps' endpoint with a lazily-fetched JWT.
AgentOps authenticates with a short-lived JWT minted from the API key. Fetching
it is blocking network I/O, so it must never run on the event loop: callback
construction (where presets are built) can run inside the proxy's async startup
or, in the SDK, on the first request. Instead of fetching at config-build time,
this preset registers a custom exporter (``kind="agentops"``) that mints the JWT
**on its first export** — which the ``BatchSpanProcessor`` runs in its own
worker thread, off any event loop — and caches it for the process lifetime.
"""
from typing import Any
import httpx
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.plumbing.providers import register_exporter_factory
_AGENTOPS_ENDPOINT = "https://otlp.agentops.cloud/v1/traces"
_AGENTOPS_AUTH_ENDPOINT = "https://api.agentops.ai/v3/auth/token"
_AGENTOPS_EXPORTER_KIND = "agentops"
class _AgentOpsSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
api_key: str | None = Field(default=None, validation_alias="AGENTOPS_API_KEY")
service_name: str = Field(
default="agentops", validation_alias="AGENTOPS_SERVICE_NAME"
)
environment: str | None = Field(
default=None, validation_alias="AGENTOPS_ENVIRONMENT"
)
def agentops_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
"""Build the AgentOps config without any network I/O.
The ``agentops`` exporter mints (and caches) the JWT lazily on its first
export, so this stays non-blocking. ``project.id`` is therefore not a
resource attribute — it is encoded in the JWT, which AgentOps uses to route
the trace to the right project.
"""
settings = _AgentOpsSettings()
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=_AGENTOPS_EXPORTER_KIND,
endpoint=_AGENTOPS_ENDPOINT,
options=(
{"api_key": settings.api_key} if settings.api_key else None
),
),
],
"resource_attributes": {
**base.resource_attributes,
"service.name": settings.service_name,
"telemetry.sdk.name": "agentops",
**(
{"deployment.environment": settings.environment}
if settings.environment
else {}
),
},
}
)
def _build_agentops_exporter(spec: ExporterSpec) -> Any:
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
class _LazyAuthAgentOpsExporter(OTLPSpanExporter):
"""OTLP/HTTP exporter that mints the AgentOps JWT on its first export.
``export`` runs in the ``BatchSpanProcessor`` worker thread, so the
blocking token fetch never touches an event loop. The result is cached
after the first attempt (success or failure) so it runs at most once.
"""
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
super().__init__(endpoint=endpoint)
self._agentops_api_key = api_key
self._auth_resolved = False
def _ensure_authenticated(self) -> None:
if self._auth_resolved:
return
self._auth_resolved = True
if not self._agentops_api_key:
return
try:
token = _fetch_agentops_jwt(self._agentops_api_key).get("token")
if token:
# ``_session`` is the requests.Session the base exporter
# POSTs through; updating its Authorization header is how the
# minted JWT reaches every subsequent export.
self._session.headers["Authorization"] = f"Bearer {token}"
except Exception as e:
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)
def export(self, spans: Any) -> Any:
self._ensure_authenticated()
return super().export(spans)
options = spec.options or {}
return _LazyAuthAgentOpsExporter(
endpoint=spec.endpoint, api_key=options.get("api_key")
)
def _fetch_agentops_jwt(api_key: str) -> dict[str, Any]:
# Own a short-lived client rather than ``_get_httpx_client()``: that returns
# a process-wide cached ``HTTPHandler`` whose connection pool is shared by
# every caller, so closing it here would break concurrent/subsequent
# requests. This one-shot auth call gets its own client to close.
with httpx.Client(timeout=10) as client:
response = client.post(
url=_AGENTOPS_AUTH_ENDPOINT,
headers={"Content-Type": "application/json", "Connection": "keep-alive"},
json={"api_key": api_key},
)
if response.status_code != 200:
raise RuntimeError(f"Failed to fetch AgentOps token: {response.text}")
return response.json()
register_exporter_factory(_AGENTOPS_EXPORTER_KIND, _build_agentops_exporter)

75
litellm/integrations/otel/presets/arize.py Normal file
View file

@ -0,0 +1,75 @@
"""Arize preset — OTLP exporter to Arize + OpenInference vocabulary."""
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm.integrations.arize.arize import ArizeLogger as _V1ArizeLogger
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
class _ArizeSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
# Standard OTLP headers env var, used as the fallback when no Arize
# credentials are configured.
otlp_traces_headers: str | None = Field(
default=None, validation_alias="OTEL_EXPORTER_OTLP_TRACES_HEADERS"
)
def arize_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
arize_cfg = _V1ArizeLogger.get_arize_config()
headers = _arize_headers(arize_cfg)
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=arize_cfg.protocol or "otlp_grpc",
endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1",
headers=headers,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
"resource_attributes": {
**base.resource_attributes,
**(
{"model_id": arize_cfg.project_name}
if arize_cfg.project_name
else {}
),
},
}
)
def _arize_headers(arize_cfg) -> str | None:
pieces = []
if arize_cfg.space_id or arize_cfg.space_key:
pieces.append(f"space_id={arize_cfg.space_id or arize_cfg.space_key}")
if arize_cfg.api_key:
pieces.append(f"api_key={arize_cfg.api_key}")
if not pieces:
# Fall back to the standard OTLP headers env var when no Arize
# credentials are configured.
return _ArizeSettings().otlp_traces_headers
return ",".join(pieces)
def arize_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request Arize OTLP headers from team/key dynamic params."""
headers: dict[str, str] = {}
# ``arize_space_key`` is the suggested param and wins over ``arize_space_id``.
space = params.get("arize_space_key") or params.get("arize_space_id")
if space:
headers["arize-space-id"] = space
api_key = params.get("arize_api_key")
if api_key:
headers["api_key"] = api_key
return headers

25
litellm/integrations/otel/presets/base.py Normal file
View file

@ -0,0 +1,25 @@
"""Preset interface.
A preset is a callable that reads its integration's env vars and produces an
:class:`OpenTelemetryV2Config` (exporter list + mapper-name list + resource
attributes). This ``Protocol`` pins that contract so ``PRESET_BY_CALLBACK`` and
the factory in ``litellm_logging`` are type-checked structurally against it,
matching the ``AttributeMapper`` protocol the mappers use.
"""
from typing import Protocol, runtime_checkable
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
@runtime_checkable
class Preset(Protocol):
"""Reads an integration's env config and returns an ``OpenTelemetryV2Config``.
``config_overrides`` lets one preset layer onto another's config (or onto
test-supplied defaults); the factory calls presets with no arguments.
"""
def __call__(
self, *, config_overrides: OpenTelemetryV2Config | None = None
) -> OpenTelemetryV2Config: ...

43
litellm/integrations/otel/presets/langfuse.py Normal file
View file

@ -0,0 +1,43 @@
"""Langfuse-OTEL preset."""
from litellm.integrations.langfuse.langfuse_otel import (
LangfuseOtelLogger as _V1Langfuse,
)
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
def langfuse_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
cfg = _V1Langfuse.get_langfuse_otel_config()
kind = cfg.exporter if isinstance(cfg.exporter, str) else "otlp_http"
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=kind,
endpoint=cfg.endpoint,
headers=cfg.headers,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "langfuse"),
}
)
def langfuse_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request Langfuse OTLP headers from team/key dynamic params."""
public_key = params.get("langfuse_public_key")
secret_key = params.get("langfuse_secret_key")
if public_key and secret_key:
return {
"Authorization": _V1Langfuse._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
}
return {}

22
litellm/integrations/otel/presets/langtrace.py Normal file
View file

@ -0,0 +1,22 @@
"""Langtrace preset — Langtrace consumes generic OTLP + a vendor mapper."""
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.presets.utils import ensure_mappers
def langtrace_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
"""Compose the Langtrace mapper on top of the customer's OTLP destination.
Unlike Arize / Phoenix / Langfuse, Langtrace doesn't ship its own endpoint
— users point their existing OTLP collector at Langtrace and just
need the vendor attribute schema applied to outgoing spans.
"""
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"mapper_names": ensure_mappers(base.mapper_names, "langtrace"),
}
)

24
litellm/integrations/otel/presets/levo.py Normal file
View file

@ -0,0 +1,24 @@
"""Levo preset — OTLP/HTTP to a Levo collector with org+workspace headers."""
from litellm.integrations.levo.levo import LevoLogger as _V1Levo
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
def levo_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
cfg = _V1Levo.get_levo_config()
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=cfg.endpoint,
headers=cfg.otlp_auth_headers,
),
],
}
)

48
litellm/integrations/otel/presets/phoenix.py Normal file
View file

@ -0,0 +1,48 @@
"""Arize-Phoenix preset."""
from pydantic import AliasChoices, Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm.integrations.arize.arize_phoenix import (
ArizePhoenixLogger as _V1Phoenix,
)
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.presets.utils import ensure_mappers
class _PhoenixSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
project_name: str = Field(
default="default",
validation_alias=AliasChoices(
"PHOENIX_PROJECT_NAME", "PHOENIX_COLLECTOR_PROJECT_NAME"
),
)
def phoenix_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
cfg = _V1Phoenix.get_arize_phoenix_config()
headers = cfg.otlp_auth_headers if hasattr(cfg, "otlp_auth_headers") else None
project_name = _PhoenixSettings().project_name
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http",
endpoint=cfg.endpoint,
headers=headers,
),
],
"mapper_names": ensure_mappers(base.mapper_names, "openinference"),
"resource_attributes": {
**base.resource_attributes,
"openinference.project.name": project_name,
},
}
)

16
litellm/integrations/otel/presets/utils.py Normal file
View file

@ -0,0 +1,16 @@
"""Shared helpers for the integration presets."""
from typing import Iterable
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
"""Return ``mapper_names`` with each of ``names`` appended if not already present.
Order is preserved and duplicates are skipped, so composing several presets
(or re-applying one) never double-adds a vocabulary.
"""
result = list(mapper_names)
for name in names:
if name not in result:
result.append(name)
return result

43
litellm/integrations/otel/presets/weave.py Normal file
View file

@ -0,0 +1,43 @@
"""Weave (W&B) preset."""
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.integrations.weave.weave_otel import (
_get_weave_authorization_header,
get_weave_otel_config,
)
from litellm.types.utils import StandardCallbackDynamicParams
def weave_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
weave_cfg = get_weave_otel_config()
base = config_overrides or OpenTelemetryV2Config()
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind=weave_cfg.protocol or "otlp_http",
endpoint=weave_cfg.endpoint,
headers=weave_cfg.otlp_auth_headers,
),
],
# Weave consumes OpenInference + a small Weave-specific overlay.
"mapper_names": ensure_mappers(base.mapper_names, "openinference", "weave"),
}
)
def weave_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request Weave OTLP headers from team/key dynamic params."""
headers: dict[str, str] = {}
api_key = params.get("wandb_api_key")
if api_key:
headers["Authorization"] = _get_weave_authorization_header(api_key=api_key)
project_id = params.get("weave_project_id")
if project_id:
headers["project_id"] = project_id
return headers

Some files were not shown because too many files have changed in this diff Show more