scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-23 05:34:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 24

Merge branch 'main' into fix/blocked-flag-refusal-before-auto-router

Browse source
This commit is contained in:
Tufail Akram 2026-10-07 15:13:19 +05:30 • committed by GitHub
commit 5a6335f0fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1409 changed files with 76868 additions and 46692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

213
.circleci/config.yml
View file

@ -892,40 +892,6 @@ jobs:
paths:
- router_unit_tests_coverage.xml
- router_unit_tests_coverage
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
docker:
- *python312_image
working_directory: ~/project
resource_class: medium
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
# Run pytest and generate JUnit XML report
- setup_litellm_enterprise_pip
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-v \
--junitxml=test-results/junit.xml \
--durations=5 \
-k \"assistants\""
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
llm_translation_testing:
docker:
- *python312_image
@ -1021,49 +987,6 @@ jobs:
paths:
- realtime_translation_coverage.xml
- realtime_translation_coverage
agent_testing:
docker:
- *python312_image
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
# Run pytest and generate JUnit XML report
- run:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \
-vv -s \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
- run:
name: Rename the coverage files
command: |
mv coverage.xml agent_coverage.xml
mv .coverage agent_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- agent_coverage.xml
- agent_coverage
guardrails_testing:
docker:
- *python312_image
@ -1706,6 +1629,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -1729,9 +1657,8 @@ jobs:
command: |
docker run -d \
-p 4001:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
--name schema-seed \
--add-host=host.docker.internal:host-gateway \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
@ -1802,6 +1729,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -1825,9 +1757,8 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e USE_PRISMA_MIGRATE=True \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e AZURE_API_KEY=$AZURE_API_KEY \
@ -1872,7 +1803,7 @@ jobs:
name: Seed the routing strategy through /config/update
command: |
curl --noproxy '*' -sSf -X POST http://localhost:4000/config/update \
-H 'Authorization: Bearer sk-1234' -H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
-d '{"router_settings": {"routing_strategy": "usage-based-routing-v2"}}'
- run:
name: Run tests
@ -1901,6 +1832,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -1922,9 +1858,8 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e AZURE_API_BASE=$AZURE_API_BASE \
-e AZURE_API_VERSION="2024-05-01-preview" \
@ -1987,6 +1922,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2012,12 +1952,11 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
@ -2073,12 +2012,11 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE="bad-license" \
@ -2122,6 +2060,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2151,11 +2094,10 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=host.docker.internal \
-e REDIS_PORT=6379 \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
@ -2210,6 +2152,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2234,12 +2181,11 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
@ -2257,12 +2203,11 @@ jobs:
command: |
docker run -d \
-p 4001:4001 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e USE_DDTRACE=True \
@ -2309,6 +2254,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2335,10 +2285,9 @@ jobs:
docker run -d \
--restart on-failure \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e STORE_MODEL_IN_DB="True" \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e TEAM_METADATA_VALIDATION_SERVICE_URL=http://host.docker.internal:9414/validate \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
@ -2388,6 +2337,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
# Remove Docker CLI installation since it's already available in machine executor
@ -2410,12 +2364,11 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
@ -2472,6 +2425,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2493,9 +2451,8 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e GEMINI_API_KEY=$GEMINI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
@ -2563,6 +2520,11 @@ jobs:
working_directory: ~/project
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
@ -2585,9 +2547,8 @@ jobs:
command: |
docker run -d \
-p 4000:4000 \
-e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
@ -2613,7 +2574,7 @@ jobs:
command: |
mkdir -p test-results
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
@ -2647,7 +2608,7 @@ jobs:
- run:
name: Combine Coverage
command: |
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage
uv tool run --from 'coverage[toml]==7.10.6' coverage xml
- codecov/upload:
file: ./coverage.xml
@ -2676,6 +2637,11 @@ jobs:
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes:
category: client
- setup_google_dns
@ -2766,8 +2732,6 @@ jobs:
- run:
name: Start LiteLLM proxy
environment:
LITELLM_MASTER_KEY: "sk-1234"
LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
SERVER_ROOT_PATH: ""
@ -2787,7 +2751,7 @@ jobs:
name: Wait for proxy to be ready
command: |
for i in $(seq 1 60); do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer sk-1234" 2>/dev/null || true)
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
echo "Proxy is ready"
exit 0
@ -2834,6 +2798,11 @@ jobs:
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes:
category: client
- setup_google_dns
@ -2910,8 +2879,6 @@ jobs:
- run:
name: Start LiteLLM proxy under a server root path
environment:
LITELLM_MASTER_KEY: "sk-1234"
LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
DISABLE_SCHEMA_UPDATE: "true"
# Output flows to this step's own log, so a boot crash is visible here
@ -2926,7 +2893,7 @@ jobs:
name: Wait for prefixed proxy to be ready
command: |
for i in $(seq 1 60); do
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
if [ "$HTTP_CODE" = "200" ]; then
echo "Prefixed proxy is ready"
exit 0
@ -3019,6 +2986,11 @@ jobs:
PYTHONPATH: tests/e2e
steps:
- checkout_migration_source
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- install_uv
- install_rust
- restore_cache:
@ -3138,6 +3110,11 @@ jobs:
E2E_FIXTURE_MODE: live
steps:
- checkout
- run:
name: Generate LiteLLM master key
command: |
key="$(openssl rand -hex 16)"
printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV"
- skip_if_unrelated_changes:
category: provider-harness
- setup_litellm_test_deps
@ -3167,9 +3144,13 @@ jobs:
parallelism:
type: integer
default: 1
resource_class:
type: enum
enum: [large, xlarge]
default: large
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
resource_class: << parameters.resource_class >>
working_directory: ~/project
parallelism: << parameters.parallelism >>
steps:
@ -3512,7 +3493,11 @@ workflows:
name: integration-<< matrix.suite >>
matrix:
parameters:
suite: [management, accounting, database, providers, mcp, sdk, cost, security, browser]
suite: [management, accounting, database, providers, sdk, cost, security, browser]
- integration_contracts:
name: integration-mcp
suite: mcp
resource_class: xlarge
- integration_contracts:
name: integration-extensions
suite: extensions
@ -3570,7 +3555,6 @@ workflows:
- local_testing_part1
- local_testing_part2
- langfuse_logging_unit_tests
- litellm_assistants_api_testing
- litellm_router_testing
- litellm_router_unit_testing
- auth_ui_unit_tests
@ -3604,7 +3588,6 @@ workflows:
- build_docker_database_image
- llm_translation_testing
- realtime_translation_testing
- agent_testing
- guardrails_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing
@ -3619,7 +3602,6 @@ workflows:
- upload-coverage:
requires:
- realtime_translation_testing
- agent_testing
- google_generate_content_endpoint_testing
- guardrails_testing
- ocr_testing
@ -3633,7 +3615,6 @@ workflows:
- langfuse_logging_unit_tests
- local_testing_part1
- local_testing_part2
- litellm_assistants_api_testing
- litellm_router_unit_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:

2
.circleci/scripts/run_integration.sh
View file

@ -72,7 +72,7 @@ export PATH="$PWD/.venv/bin:$PATH"
export PYTHONPATH="$PWD:$PWD/tests:$PWD/tests/e2e"
export DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:5432/circle_test"
export REDIS_HOST=127.0.0.1 REDIS_PORT=6379
export LITELLM_MASTER_KEY=sk-integration-master LITELLM_SALT_KEY=sk-integration-salt
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)" LITELLM_SALT_KEY=sk-integration-salt
export LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True
export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1
export INTEGRATION_PROXY_URL=http://127.0.0.1:4000

5
.gitguardian.yaml
View file

@ -84,10 +84,6 @@ secret:
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
@ -108,4 +104,3 @@ secret:
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b

3
.github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

@ -150,7 +150,7 @@ For each e2e test you added or changed, list the manual steps a reviewer can fol
Example checklists:
- tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py::TestKeyRateLimits::test_rpm_limit_blocks_over_limit - a key allowed 2 requests a minute serves exactly 2 and refuses the 3rd
- [ ] Generate a limited key: curl -X POST http://localhost:4000/key/generate -H "Authorization: Bearer sk-1234" -d '{"rpm_limit": 2}'
- [ ] Generate a limited key: curl -X POST http://localhost:4000/key/generate -H "Authorization: Bearer $LITELLM_MASTER_KEY" -d '{"rpm_limit": 2}'
- [ ] Send three /v1/chat/completions requests with that key inside one minute
- [ ] Expect the first two to return 200 and the third to return 429 naming the rpm limit
- [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
@ -164,4 +164,3 @@ Example checklists:
## Final Attestation
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

BIN
.github/assets/lens-datasets/add-to-dataset-dialog.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 248 KiB

BIN
.github/assets/lens-datasets/dataset-case-panel.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 128 KiB

BIN
.github/assets/lens-datasets/dataset-detail.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 101 KiB

BIN
.github/assets/lens-datasets/datasets-list.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 132 KiB

BIN
.github/assets/lens-datasets/old-revision.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 184 KiB

BIN
.github/assets/lens-datasets/trace-header.png vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 259 KiB

7
.github/workflows/_test-unit-base.yml vendored
View file

@ -3,6 +3,11 @@ name: _Unit Test Base (Reusable)
on:
workflow_call:
inputs:
python-version:
description: "Python version used to install dependencies and run tests"
required: false
type: string
default: "3.12"
test-path:
description: >-
Space-separated pytest paths to run. A path that no longer exists is
@ -78,7 +83,7 @@ permissions:
contents: read
env:
UV_PYTHON: "3.12"
UV_PYTHON: ${{ inputs.python-version }}
LITELLM_LOCAL_MODEL_COST_MAP: "True"
jobs:

10
.github/workflows/test-code-quality.yml vendored
View file

@ -95,7 +95,9 @@ jobs:
- name: test_e2e_metadata
env:
PYTHONPATH: tests/e2e
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
run: |
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
- name: Check merge smoke harness
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
@ -163,6 +165,12 @@ jobs:
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
- name: test_check_no_publicly_known_master_key
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
- name: check_unbounded_in_lists (fails on findings not in the baseline)
run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py

15
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -39,6 +39,21 @@ concurrency:
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
lens-python-310:
name: Lens Python 3.10
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
python-version: "3.10"
test-path: tests/unit/proxy/lens/test_inference.py
workers: 0
reruns: 0
timeout-minutes: 5
artifact-name: lens-python-310
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list

4
Dockerfile
View file

@ -9,8 +9,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.25.2
ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin

6
ci_cd/TEST_KEY_PATTERNS.md
View file

@ -5,7 +5,7 @@ Standard patterns for test/mock keys and credentials in the LiteLLM codebase to
## How GitGuardian Works
GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
- **Low entropy** values (like `sk-test`, `postgres`) are automatically ignored
- **High entropy** values (realistic-looking secrets) trigger detection
- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
@ -15,8 +15,8 @@ GitGuardian uses **machine learning and entropy analysis**, not just pattern mat
These won't trigger GitGuardian's ML detector:
```python
api_key = "sk-1234"
api_key = "sk-12345"
api_key = "sk-test"
api_key = "sk-mock"
database_password = "postgres"
token = "test123"
```

13
cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb vendored
View file

@ -16,6 +16,8 @@
"\n",
"These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n",
"\n",
"Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running authenticated examples\n",
"\n",
"For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n",
"\n",
"To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
@ -95,9 +97,10 @@
},
"outputs": [],
"source": [
"import os\n",
"from openai import OpenAI\n",
"client = OpenAI(\n",
" api_key=\"sk-1234\", # [OPTIONAL] set if you set one on proxy, else set \"\"\n",
" api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
" base_url=\"http://0.0.0.0:4000\",\n",
")\n",
"\n",
@ -298,14 +301,14 @@
" engine=\"azure-gpt-3.5\", # model_name on litellm proxy\n",
" temperature=0.0,\n",
" azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n",
" api_key=\"sk-1234\", # litellm proxy API Key\n",
" api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
" api_version=\"2023-07-01-preview\",\n",
")\n",
"\n",
"embed_model = AzureOpenAIEmbedding(\n",
" deployment_name=\"azure-embedding-model\",\n",
" azure_endpoint=\"http://0.0.0.0:4000\",\n",
" api_key=\"sk-1234\",\n",
" api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n",
" api_version=\"2023-07-01-preview\",\n",
")\n",
"\n",
@ -341,7 +344,7 @@
"\n",
"const model = new ChatOpenAI({\n",
" modelName: \"gpt-4\",\n",
" openAIApiKey: \"sk-1234\",\n",
" openAIApiKey: process.env.LITELLM_MASTER_KEY,\n",
" modelKwargs: {\"metadata\": \"hello world\"} // 👈 PASS Additional params here\n",
"}, {\n",
" basePath: \"http://0.0.0.0:4000\",\n",
@ -372,7 +375,7 @@
"const { OpenAI } = require('openai');\n",
"\n",
"const openai = new OpenAI({\n",
" apiKey: \"sk-1234\", // This is the default and can be omitted\n",
" apiKey: process.env.LITELLM_MASTER_KEY,\n",
" baseURL: \"http://0.0.0.0:4000\"\n",
"});\n",
"\n",

7
cookbook/Proxy_Batch_Users.ipynb vendored
View file

@ -6,7 +6,9 @@
"id": "680oRk1af-xJ"
},
"source": [
"# Environment Setup"
"# Environment Setup\n",
"\n",
"Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running the cells"
]
},
{
@ -17,6 +19,7 @@
},
"outputs": [],
"source": [
"import os\n",
"import csv\n",
"from typing import Optional\n",
"import httpx\n",
@ -24,7 +27,7 @@
"import asyncio\n",
"\n",
"proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n",
"master_key = \"sk-1234\" # 👈 SET TO PROXY MASTER KEY"
"master_key = os.environ[\"LITELLM_MASTER_KEY\"]"
]
},
{

3
cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md
View file

@ -47,7 +47,7 @@ Set your environment variables:
```bash
export ANTHROPIC_API_KEY="your-anthropic-api-key"
export LITELLM_MASTER_KEY="sk-1234567890" # Generate a secure key
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
```
## Step 2: Start Proxy
@ -292,4 +292,3 @@ model_list:
- [LiteLLM Documentation](https://docs.litellm.ai/)
- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)

10
cookbook/anthropic_agent_sdk/README.md
View file

@ -20,6 +20,12 @@ litellm --model claude-sonnet-4-20250514
litellm --config config.yaml
```
Generate a master key before starting the proxy:
```bash
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
```
### 3. Run the chat
**Basic Agent (no MCP):**
@ -58,7 +64,7 @@ Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
```
@ -98,7 +104,7 @@ The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
```python
# Point to LiteLLM gateway (not Anthropic)
os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM key
os.environ["ANTHROPIC_API_KEY"] = os.environ["LITELLM_API_KEY"]
# Use any model configured in LiteLLM
options = ClaudeAgentOptions(

2
cookbook/anthropic_agent_sdk/common.py
View file

@ -13,7 +13,7 @@ class Config:
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
LITELLM_API_KEY = os.environ["LITELLM_API_KEY"]
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")

4
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
View file

@ -1,8 +1,10 @@
import os
from openai import OpenAI
client = OpenAI(
base_url="http://0.0.0.0:4000",
api_key="sk-1234",
api_key=os.environ["LITELLM_MASTER_KEY"],
)
BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"

4
cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
View file

@ -4,10 +4,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import os
import openai
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
api_key=os.environ["LITELLM_MASTER_KEY"],
base_url="http://localhost:4000", # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")

8
cookbook/livekit_agent_sdk/README.md
View file

@ -15,12 +15,14 @@ pip install livekit-agents[xai] websockets
```bash
# With xAI
export XAI_API_KEY="your-xai-key"
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
litellm --config config.yaml --port 4000
```
### 3. Run the voice agent
```bash
export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
python main.py
```
@ -32,7 +34,7 @@ Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_API_KEY="$LITELLM_MASTER_KEY"
export LITELLM_MODEL="grok-voice-agent"
```
@ -59,7 +61,7 @@ model_list:
mode: realtime
general_settings:
master_key: sk-1234
master_key: os.environ/LITELLM_MASTER_KEY
```
Then start: `litellm --config config.yaml --port 4000`
@ -73,7 +75,7 @@ from livekit.plugins import xai
model = xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234", # LiteLLM proxy key
api_key=os.environ["LITELLM_MASTER_KEY"],
base_url="http://localhost:4000", # Point to LiteLLM
)
```

2
cookbook/livekit_agent_sdk/config.example.yaml
View file

@ -17,4 +17,4 @@ litellm_settings:
drop_params: True
general_settings:
master_key: sk-1234 # Change this to a secure key
master_key: os.environ/LITELLM_MASTER_KEY

2
cookbook/livekit_agent_sdk/main.py
View file

@ -13,7 +13,7 @@ import websockets
# Configuration
PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
API_KEY = os.environ["LITELLM_API_KEY"]
MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")

2
cookbook/misc/config.yaml
View file

@ -58,7 +58,7 @@ litellm_settings:
context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
general_settings:
master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys
master_key: os.environ/LITELLM_MASTER_KEY
store_model_in_db: True
proxy_budget_rescheduler_min_time: 60
proxy_budget_rescheduler_max_time: 64

4
cookbook/misc/migrate_proxy_config.py
View file

@ -12,6 +12,8 @@ Step 2: reads `model_list` and loops through all models
Step 3: calls `<proxy-base-url>/model/new` for each model
"""
import os
import yaml
import requests
@ -89,7 +91,7 @@ def migrate_models(config_file, proxy_base_url):
# Usage
config_file = "config.yaml"
proxy_base_url = "http://0.0.0.0:4000"
master_key = "sk-1234"
master_key = os.environ["LITELLM_MASTER_KEY"]
print(f"config_file: {config_file}")
print(f"proxy_base_url: {proxy_base_url}")
migrate_models(config_file, proxy_base_url)

4
cookbook/misc/test_responses_api.py
View file

@ -1,8 +1,10 @@
import base64
import os
from openai import OpenAI
import time
client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
client = OpenAI(base_url="http://0.0.0.0:4001", api_key=os.environ["LITELLM_MASTER_KEY"])
# Function to encode the image

8
cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb vendored
View file

@ -4,7 +4,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n"
"# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n",
"\n",
"Set `LITELLM_PROXY_API_KEY` in the notebook environment to a key accepted by the proxy before running the model cell\n"
]
},
{
@ -160,11 +162,13 @@
},
"outputs": [],
"source": [
"import os\n",
"\n",
"model = ChatOpenAI(\n",
" openai_api_base=\"LITELLM_PROXY_BASE_URL\", # e.g.: http://0.0.0.0:4000\n",
" model = \"gpt-3.5-turbo\", # LITELLM 'model_name'\n",
" temperature=0.1, \n",
" api_key=\"LITELLM_PROXY_API_KEY\" # e.g.: \"sk-1234\"\n",
" api_key=os.environ[\"LITELLM_PROXY_API_KEY\"]\n",
")"
]
},

3
cookbook/mock_prompt_management_server/README.md
View file

@ -69,7 +69,7 @@ litellm --config config.yaml
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
-d '{
"model": "gpt-3.5-turbo",
"prompt_id": "hello-world-prompt",
@ -290,4 +290,3 @@ Before deploying to production:
## Questions?
This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).

3
cookbook/nova_sonic_realtime.py
View file

@ -10,6 +10,7 @@ Prerequisites:
- websockets installed: pip install websockets
Usage:
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"
python nova_sonic_realtime.py
"""
@ -33,7 +34,7 @@ CHUNK_SIZE = 1024
# LiteLLM proxy configuration
LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
LITELLM_API_KEY = "sk-12345" # Your LiteLLM API key
LITELLM_API_KEY = os.environ["LITELLM_MASTER_KEY"]
class RealtimeClient:

6
cookbook/veo_video_generation.py
View file

@ -24,8 +24,8 @@ class VeoVideoGenerator:
def __init__(
self,
api_key: str,
base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234",
):
"""
Initialize the Veo video generator.
@ -274,12 +274,12 @@ def main():
Configure these environment variables:
- LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
- LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
- LITELLM_API_KEY: API key for LiteLLM proxy authentication
"""
# Configuration from environment or defaults
base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
api_key = os.environ["LITELLM_API_KEY"]
print("🚀 Starting Veo Video Generation Example")
print(f"📡 Using LiteLLM proxy at: {base_url}")

1
deploy/lens/Dockerfile
View file

@ -27,6 +27,7 @@ WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
COPY litellm/proxy/lens/context_pipeline.py litellm/proxy/lens/agent_review.py litellm/proxy/lens/agent_runtime.py litellm/proxy/lens/agent_workspace.py litellm/proxy/lens/python_tool.py litellm/proxy/lens/activity.py litellm/proxy/lens/agent_context.py /app/lens/
COPY litellm/proxy/lens/reviews.py litellm/proxy/lens/reconciliation.py /app/lens/
COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
COPY --from=builder /app/python.seccomp /app/lens/python.seccomp
COPY deploy/lens/python_runtime.py /tmp/python_runtime.py

9
deploy/lens/Dockerfile.dockerignore
View file

@ -2,14 +2,11 @@
!deploy/
!deploy/lens/
!deploy/lens/requirements.lock
!deploy/lens/python_policy.c
!deploy/lens/python_runtime.py
!litellm/
!litellm/proxy/
!litellm/proxy/lens/
!litellm/proxy/lens/__init__.py
!litellm/proxy/lens/models.py
!litellm/proxy/lens/trace_store.py
!litellm/proxy/lens/analysis.py
!litellm/proxy/lens/worker.py
!litellm/proxy/lens/release.py
!litellm/proxy/lens/*.py
!litellm/proxy/lens/prompts/
!litellm/proxy/lens/prompts/**

49
deploy/lens/README.md
View file

@ -21,7 +21,7 @@ docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
docker compose -f docker/docker-compose.tracing.yml up -d --build
```
Open `http://localhost:4002/ui/` and sign in as `admin` with password `sk-1234`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
Open `http://localhost:4002/ui/` and sign in as `admin` with the key saved in `.lens-dev/master_key`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
```bash
export LITELLM_URL=http://litellm:4000
@ -106,7 +106,7 @@ The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, pr
If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
Setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
## Configure a lens
@ -114,17 +114,17 @@ Choose agent runs, individual LLM requests, or both. The matching-activity previ
Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage
Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Each scan recalculates the window and reuses completed reviews when the selected trace content, expected behavior, enabled checks and analysis model are unchanged. Budget, name and schedule edits preserve reuse. Duplicate a lens when you want a separate investigation without changing an existing monitor
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. A running scan retains its analysis settings and selected execution IDs across retries. Budget edits apply to subsequent model calls, including those in an active scan
## Read the results
Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
Use the batch selector or Scans tab to reopen previous results. Each batch keeps its own findings, settings, selected runs, coverage and cost. Older batches created before snapshot support remain available through accumulated findings. The Runs tab lists the selected batch's sample and can filter per-run observations, including runs without an observed issue and runs with insufficient evidence. These observations precede the final evidence investigation. Linked-run counts on findings include cited counterexamples, so they are not failure counts
Use the **Investigation run** selector or **History** to reopen previous results. Each run keeps its new or updated findings, settings, selected traces, coverage and cost. An unchanged rerun adds no findings; choose **All accumulated findings** to see saved findings across runs. The **Agent traces** tab lists the selected sample, including traces without an observed issue and traces with insufficient evidence. The tab is called **LLM requests** or **Traces and requests** for those activity types. Findings distinguish distinct affected traces from contributing investigation runs. Counterexamples remain visible as evidence without increasing the affected count. Merged finding links continue to resolve to the retained finding
Choose **This is expected** and explain why to teach later scans about acceptable behavior. Feedback is kept with the lens and included in subsequent reviews. It does not alter historical evidence or exempt different problems
Enter an explanation under **What should Lens remember?** and choose **This is expected** to dismiss expected behavior, **Mark resolved** after fixing an issue, or **Reopen** to reopen a resolved issue. These actions save the feedback together with the status; typing feedback alone neither saves it nor resolves the finding. Feedback informs later analysis and reconciliation without invalidating completed reviews. It stays with the finding when evidence recurs, does not alter historical evidence, and does not exempt different problems
## What a scan does
@ -132,13 +132,15 @@ The proxy selects executions received or updated within the configured lookback
A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
The worker prepares a workspace containing the selected execution metadata and reviews executions in parallel. Reviewers receive their assignment and use catalog, read, search and optional Python tools to inspect evidence, including nested agents and other sampled executions. Tools retrieve original content from the gateway when requested; the worker does not preload the sampled traces or inject them into each model request. Python receives selected evidence as streamed input. Completed reviews retain cited excerpts and metadata. Observation batches are grouped in parallel, reconciled, and investigated against the original evidence. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs
The worker reads complete selected trace content to compute fingerprints before making paid model calls. Completed review checkpoints are reused only within the same lens when both the complete content and investigation criteria match. Criteria are the expected behavior, enabled check IDs and instructions, and analysis model. Lens fingerprints these values rather than using the time of an unrelated settings edit. Scope and sampling changes preserve matching reviews for traces selected again; duplicating a lens starts an independent set of reviews. Initial reviews are confined to their assigned trace and retain observations, cited excerpts and metadata. Reviewers use catalog, read, search and optional Python tools; Python receives selected evidence as streamed input. Pending observation batches are grouped in parallel and investigated against the original evidence, then reconciled with saved findings. A recurring cause extends its existing finding, preserving feedback, earlier evidence and contributing run history
If every selected review is already incorporated into findings, the run completes with a reuse count, zero model calls and zero analysis cost, including when the monthly budget is exhausted. The run remains in history. A partially failed run preserves completed review checkpoints; pending grouping or investigation can still need paid model calls even when all selected trace reviews are reused. A trace without a complete matching checkpoint needs a review. Live progress distinguishes reviews eligible for reuse from reviews actually recorded; failed or cancelled runs report only the reuse they completed
There is no fixed total run, span, candidate or investigation-turn cutoff. Agents can replace their active conversation with working notes. If a request exceeds the configured model's context window, the worker compacts the conversation automatically and resumes with references to its archived tool history. Original evidence remains accessible through the gateway while it is available and retained. Tool results and working notes remain accessible during the investigation; character ranges make even a single oversized result readable in pieces. A review reports an error if the task or its replacement notes cannot fit. Context windows, the configured budget, worker resources and recorded evidence still bound practical work. The investigator has no browsing, code-editing or production-action tools
The live review drawer shows loading, trace review, parallel grouping, reconciliation and candidate investigation. It reports current model and tool operations, including context compaction, and retains tool-call counts on completed trace reviews. These counts describe attempted calls, not successful executions. This progress channel contains operation metadata, not Python code or tool output. Preliminary observations remain separate from final findings; the final finding format and evidence links are unchanged
The live review drawer shows loading, trace review, parallel grouping, reconciliation and candidate investigation. It reports current model and tool operations, including context compaction, and retains tool-call counts on completed trace reviews. These counts describe attempted calls, not successful executions. This progress channel contains operation metadata, not Python code or tool output. Preliminary observations remain separate from final findings and their validated evidence
Each model response must match its JSON schema. A malformed response gets one repair attempt through the same budget controls. A session review that remains invalid or cannot fit marks that execution unassessable while other reviews continue. Broken evidence pagination or missing content pages return tool errors so the agent can inspect narrower spans or other evidence. Unreadable citations receive repair feedback. Verified excerpts remain available without fetching their source again. The affected source counts as partial, including failures discovered during later investigations, while the reviewer owns its assessment. Candidate investigation errors preserve completed findings. Source and analysis errors remain visible and mark the final scan as failed; transport errors, cancellation and budget exhaustion stop the scan. Both the worker and proxy validate quoted evidence against original content. Per-run issue assessments follow supporting citations, including evidence found by another run's reviewer; counterexamples do not mark a run affected. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
Each model response must match its JSON schema. A malformed response gets one repair attempt through the same budget controls. A session review that remains invalid or cannot fit marks that execution unassessable while other reviews continue. Broken evidence pagination or missing content pages return tool errors so the agent can inspect narrower spans or other evidence. Unreadable citations receive repair feedback. Verified excerpts remain available without fetching their source again. The affected source counts as partial, including failures discovered during later investigations, while the reviewer owns its assessment. Findings are published only after comparison with each other and saved findings finishes. If analysis stops before that comparison completes, completed trace reviews and their evidence remain saved for reuse, and the run retains its assessments and error. A later run can retry grouping and investigation. Runs with useful completed assessments or reconciled findings show partial results; total failures are marked failed. Transport errors, cancellation and budget exhaustion stop further analysis. Both the worker and proxy validate quoted evidence against original content. Per-run issue assessments follow supporting citations, including evidence found by another run's reviewer; counterexamples do not mark a run affected. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
@ -146,9 +148,15 @@ Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable ex
PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Both the Lens budget and the selected virtual key’s budgets, model permissions, and rate limits apply. Analysis spend appears under that key in Virtual Keys and normal request logs, with Lens, scan, and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Existing workers need a billing key assigned in **Set up analysis** before they can resume
Lens checks which selected traces have reusable reviews before requesting model budget. Reuse needs no model call or reservation. New trace reviews and unfinished grouping or investigation can incur cost
V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them
Before each model call, Lens reserves a conservative allowance based on the input and permitted output. The summary separates settled monthly spend, unexpired reservations and available budget. With a $100 limit, $45 spent and $10 reserved, $45 is available for additional calls. Successful calls settle to recorded cost and release unused capacity. Failed or timed-out requests release their hold; abandoned holds expire after the proxy request timeout plus a grace period
Calls wait when concurrent reservations temporarily hold the remaining capacity. Waiting and model execution share the proxy's request timeout. If one request's allowance exceeds the unspent monthly budget, the error reports what the request needs and what remains. Reduce the deployment's output allowance or increase the limit. Paid analysis stops when the monthly limit is spent; completed reviews can still be reused. The monthly budget renews on the UTC calendar month
The assigned virtual key has independent budgets, model permissions and rate limits. Several investigations can share that key, so its limit can stop analysis even when one lens has budget left. Every worker needs a billing key assigned through worker setup or **Settings**. Analysis spend appears under that key in **Virtual Keys** and normal request logs, with Lens, run and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Terminal budget, authentication or transport failures stop the scan after applicable retries and preserve completed checkpoints
Lens requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Changed content requires a matching review before reuse. Findings should be reviewed by a person before acting on them
## API access
@ -221,18 +229,17 @@ The default workspace retrieves trace content on demand. Python calls have tempo
To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
## Upgrading from the original Lens API
The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`, and worker claims use `lens_id`. Upgrade the proxy and recreate every worker with the image shown by the upgraded dashboard before starting new scans. Update API clients to the new paths and response fields. Old worker images cannot poll the renamed API
Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push
## Release compatibility
Gateway and worker builds carry the same `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release
Gateway and worker builds carry the same `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched
PostgreSQL stores complete review checkpoints in `LiteLLM_LensReview`, alongside Lens records and run history. Schema migrations preserve saved investigations, findings, history, worker credentials and billing assignments without rewriting stored Lens records
Drain active scans, stop workers, back up the database, and deploy all gateway replicas as a coordinated replacement or traffic cutover. Keep traffic paused until every gateway replica uses the selected build and its migrations have completed. Mixed gateway versions sharing Lens data are not supported because every replica must understand the stored records. Pausing workers alone does not prevent dashboard or API writes. Recreate workers with the matching image and their existing tokens, then resume traffic and schedules. Scanning waits until a compatible worker connects
Use the shipped migration history for databases containing Lens data. The schema guard stops `--use_prisma_db_push` before changes if it detects Lens tables that require renaming; run the shipped rename migration against the configured schema before retrying. Fresh databases and databases with the current table names can use database push
A rollback to a gateway that cannot read saved Lens records requires restoring a compatible database backup. Database push from such a build can also remove the review table. Test recovery on a separate database and account for all gateway data written after the backup
The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Set an explicit `LENS_WORKER_IMAGE` for worker-only Compose. Verify that the image exists and matches the gateway before deploying it

4
docker/Dockerfile.database
View file

@ -9,8 +9,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.25.2
ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin

4
docker/Dockerfile.non_root
View file

@ -7,8 +7,8 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.25.2
ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin

3
docker/docker-compose.tracing.yml
View file

@ -9,8 +9,7 @@ services:
LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:-}
command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
environment:
LITELLM_MASTER_KEY: sk-1234
LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set LITELLM_MASTER_KEY}
LITELLM_SALT_KEY: sk-local-tracing-salt-key
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
STORE_MODEL_IN_DB: "True"

8
enterprise/enterprise_hooks/__init__.py
View file

@ -1,15 +1,15 @@
from typing import Dict, Literal, Type, Union
from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles
from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
from litellm_enterprise.proxy.hooks.managed_vector_stores import (
_PROXY_LiteLLMManagedVectorStores,
PROXY_LiteLLMManagedVectorStores,
)
from litellm.integrations.custom_logger import CustomLogger
ENTERPRISE_PROXY_HOOKS: Dict[str, Type[CustomLogger]] = {
"managed_files": _PROXY_LiteLLMManagedFiles,
"managed_vector_stores": _PROXY_LiteLLMManagedVectorStores,
"managed_files": PROXY_LiteLLMManagedFiles,
"managed_vector_stores": PROXY_LiteLLMManagedVectorStores,
}

3
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -20,7 +20,7 @@ from litellm._logging import verbose_proxy_logger
from fastapi import HTTPException
class _ENTERPRISE_BannedKeywords(CustomLogger):
class ENTERPRISE_BannedKeywords(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self):
@ -114,3 +114,4 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
response: str,
):
self.test_violation(test_str=response)
_ENTERPRISE_BannedKeywords = ENTERPRISE_BannedKeywords

3
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -21,7 +21,7 @@ from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
from litellm.proxy.utils import PrismaClient
class _ENTERPRISE_BlockedUserList(CustomLogger):
class ENTERPRISE_BlockedUserList(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self, prisma_client: Optional[PrismaClient]):
@ -128,3 +128,4 @@ class _ENTERPRISE_BlockedUserList(CustomLogger):
str(e)
)
)
_ENTERPRISE_BlockedUserList = ENTERPRISE_BlockedUserList

5
enterprise/enterprise_hooks/google_text_moderation.py
View file

@ -16,7 +16,7 @@ from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
class _ENTERPRISE_GoogleTextModeration(CustomLogger):
class ENTERPRISE_GoogleTextModeration(CustomLogger):
user_api_key_cache = None
confidence_categories = [
"toxic",
@ -125,9 +125,10 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
)
# Handle the response
return data
_ENTERPRISE_GoogleTextModeration = ENTERPRISE_GoogleTextModeration
# google_text_moderation_obj = _ENTERPRISE_GoogleTextModeration()
# google_text_moderation_obj = ENTERPRISE_GoogleTextModeration()
# asyncio.run(
# google_text_moderation_obj.async_moderation_hook(
# data={"messages": [{"role": "user", "content": "Hey, how's it going?"}]}

3
enterprise/enterprise_hooks/openai_moderation.py
View file

@ -24,7 +24,7 @@ from litellm.proxy.guardrails._content_utils import iter_message_text
from litellm.types.utils import CallTypesLiteral
class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
class ENTERPRISE_OpenAI_Moderation(CustomLogger):
@property
def model_name(self) -> str:
return litellm.openai_moderations_model_name or DEFAULT_OPENAI_MODERATIONS_MODEL
@ -55,3 +55,4 @@ class _ENTERPRISE_OpenAI_Moderation(CustomLogger):
status_code=403, detail={"error": "Violated content safety policy"}
)
pass
_ENTERPRISE_OpenAI_Moderation = ENTERPRISE_OpenAI_Moderation

35
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/endpoints.py
View file

@ -3,9 +3,10 @@ Endpoints for managing email alerts on litellm
"""
import json
from typing import Dict
from typing import Dict, Final, cast
from fastapi import APIRouter, Depends, HTTPException
from pydantic import JsonValue
from litellm_enterprise.types.enterprise_callbacks.send_emails import (
DefaultEmailSettings,
EmailEvent,
@ -38,14 +39,15 @@ async def _get_email_settings(prisma_client) -> Dict[str, bool]:
and general_settings_entry.param_value is not None
):
# Get general settings value
if isinstance(general_settings_entry.param_value, str):
general_settings = json.loads(general_settings_entry.param_value)
else:
general_settings = general_settings_entry.param_value
general_settings: Final = (
cast(Dict[str, object], json.loads(general_settings_entry.param_value))
if isinstance(general_settings_entry.param_value, str)
else cast(Dict[str, object], general_settings_entry.param_value)
)
# Extract email_settings from general settings if it exists
if general_settings and "email_settings" in general_settings:
email_settings = general_settings["email_settings"]
email_settings: Final = cast(Dict[str, bool], general_settings["email_settings"])
# Update settings_dict with values from general_settings
for event_name, enabled in email_settings.items():
settings_dict[event_name] = enabled
@ -64,7 +66,7 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
from litellm.proxy.proxy_server import proxy_config
proxy_config.reject_config_owned_writes(
section_name="general_settings", changed_keys={"email_settings": settings}
section_name="general_settings", changed_keys={"email_settings": cast(JsonValue, settings)}
)
try:
verbose_proxy_logger.debug(
@ -77,16 +79,15 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
)
# Initialize general settings dict
if (
general_settings_entry is not None
and general_settings_entry.param_value is not None
):
if isinstance(general_settings_entry.param_value, str):
general_settings = json.loads(general_settings_entry.param_value)
else:
general_settings = dict(general_settings_entry.param_value)
else:
general_settings = {}
general_settings: Final = (
(
cast(Dict[str, object], json.loads(general_settings_entry.param_value))
if isinstance(general_settings_entry.param_value, str)
else cast(Dict[str, object], dict(general_settings_entry.param_value))
)
if general_settings_entry is not None and general_settings_entry.param_value is not None
else {}
)
# Update email_settings in general_settings
general_settings["email_settings"] = settings

2
enterprise/litellm_enterprise/integrations/custom_guardrail.py
View file

@ -36,7 +36,7 @@ class EnterpriseCustomGuardrailHelper:
proxy_server_request = data.get("proxy_server_request", {})
request_tags = StandardLoggingPayloadSetup._get_request_tags(
request_tags = StandardLoggingPayloadSetup.get_request_tags(
litellm_params=data,
proxy_server_request=proxy_server_request,
)

82
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -60,6 +60,32 @@ class _ManagedObjectRow(Protocol):
@property
def file_object(self) -> object: ...
@property
def org_id(self) -> str | None: ...
@property
def api_key(self) -> str | None: ...
@property
def team_id(self) -> str | None: ...
class _ReadableFileContent(Protocol):
async def read(self) -> bytes: ...
class _HasFileContent(Protocol):
@property
def content(self) -> bytes: ...
async def _file_content_bytes(file_content: object) -> bytes:
if hasattr(file_content, "content"):
return cast(_HasFileContent, file_content).content
if hasattr(file_content, "read"):
return await cast(_ReadableFileContent, file_content).read()
return cast(bytes, file_content)
def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
return ManagedObjectRepository(prisma_client).table
@ -175,17 +201,21 @@ class CheckBatchCost:
return None
async def _get_org_id(self, job: "_ManagedObjectRow", batch_id: str) -> str | None:
org_id = getattr(job, "org_id", None)
org_id: Final = getattr(job, "org_id", None)
if org_id:
return org_id
api_key = getattr(job, "api_key", None)
team_id = getattr(job, "team_id", None)
api_key: Final = getattr(job, "api_key", None)
team_id: Final = getattr(job, "team_id", None)
if api_key:
try:
key_row: prisma_models.LiteLLM_VerificationToken | None = await _token_table(
self.prisma_client
).find_unique(where={"token": api_key})
key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
key_org_id: Final = (
cast(str | None, getattr(key_row, "organization_id", None))
if key_row is not None
else None
)
if key_org_id:
return key_org_id
except Exception as e:
@ -199,7 +229,7 @@ class CheckBatchCost:
team_row: prisma_models.LiteLLM_TeamTable | None = await _team_table(self.prisma_client).find_unique(
where={"team_id": team_id}
)
return getattr(team_row, "organization_id", None) if team_row is not None else None
return cast(str | None, getattr(team_row, "organization_id", None)) if team_row is not None else None
except Exception as e:
verbose_proxy_logger.error(f"CheckBatchCost: could not resolve the team's org for batch {batch_id}: {e}")
return None
@ -673,16 +703,17 @@ class CheckBatchCost:
from litellm.types.utils import LiteLLMBatch
file_object = job.file_object
if isinstance(file_object, str):
try:
file_object = json.loads(file_object)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(file_object, dict):
file_object: Final = job.file_object
try:
parsed_file_object: Final[object] = (
json.loads(file_object) if isinstance(file_object, str) else file_object
)
except (json.JSONDecodeError, ValueError):
return None
if not isinstance(parsed_file_object, dict):
return None
try:
return LiteLLMBatch.model_validate(file_object).input_file_id
return LiteLLMBatch.model_validate(parsed_file_object).input_file_id
except Exception:
return None
@ -704,10 +735,11 @@ class CheckBatchCost:
"""
from litellm.batches.batch_utils import (
count_error_file_failed_requests,
_get_file_content_as_dictionary,
get_file_content_as_dictionary,
calculate_batch_cost_and_usage,
)
from litellm.files.main import afile_content
from litellm.files.types import FileContentCallOptions
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
@ -745,30 +777,22 @@ class CheckBatchCost:
except (IndexError, AttributeError):
pass
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
credentials: Final = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
_file_content = await afile_content(
file_id=raw_output_file_id,
file_id=cast(str, raw_output_file_id),
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
**credentials,
**cast(FileContentCallOptions, credentials),
)
# Access content - handle both direct attribute and method call
if hasattr(_file_content, 'content'):
content_bytes = _file_content.content # type: ignore[union-attr]
elif hasattr(_file_content, 'read'):
content_bytes = await _file_content.read() # type: ignore[misc]
else:
content_bytes = _file_content # type: ignore[assignment]
content_bytes: Final = await _file_content_bytes(_file_content)
file_content_as_dict = _get_file_content_as_dictionary(
content_bytes # type: ignore[arg-type]
)
file_content_as_dict = get_file_content_as_dictionary(content_bytes)
# Record output file size
if prom_logger and content_bytes:
try:
prom_logger.record_managed_file_size(
size_bytes=len(content_bytes), # type: ignore
size_bytes=len(content_bytes),
purpose="batch",
file_type="output",
model=model_id,
@ -805,7 +829,7 @@ class CheckBatchCost:
team_id=getattr(job, "team_id", None),
)
for _file_attr in ["output_file_id", "error_file_id"]:
_raw_file_id = getattr(response, _file_attr, None)
_raw_file_id = cast(str | None, getattr(response, _file_attr, None))
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
try:
_unified_file_id = managed_files_hook.get_unified_output_file_id(

4
enterprise/litellm_enterprise/proxy/enterprise_routes.py
View file

@ -8,7 +8,7 @@ from . import ui_crud_endpoints # side-effect: registers extra UI settings
from .audit_logging_endpoints import router as audit_logging_router
from .liteadmin import router as liteadmin_router
from .management_endpoints import management_endpoints_router
from .utils import _should_block_robots
from .utils import should_block_robots
__all__ = ["router", "ui_crud_endpoints"]
@ -25,7 +25,7 @@ async def get_robots():
Block all web crawlers from indexing the proxy server endpoints
This is useful for ensuring that the API endpoints aren't indexed by search engines
"""
if _should_block_robots():
if should_block_robots():
return Response(content="User-agent: *\nDisallow: /", media_type="text/plain")
else:
return Response(status_code=404)

93
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -23,6 +23,7 @@ from uuid import NAMESPACE_URL, uuid5
import httpx
from fastapi import HTTPException
from pydantic import ValidationError
from typing_extensions import ReadOnly
import litellm
from litellm import Router, verbose_logger
@ -30,10 +31,13 @@ from litellm._internal_context import with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.constants import MAX_FILE_LIST_LIMIT
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
from openai import AsyncOpenAI
from openai.types.file_deleted import FileDeleted
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
@ -190,6 +194,19 @@ class _ManagedObjectTableActions(Protocol):
async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
class _ManagedResourceDatabase(Protocol):
@property
def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
@property
def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
class _ManagedResourcePrismaClient(Protocol):
@property
def db(self) -> _ManagedResourceDatabase: ...
class _SchedulerWithJobLookup(Protocol):
def get_job(self, job_id: str) -> object: ...
@ -199,7 +216,12 @@ class _CursorPageArgs(TypedDict, total=False):
skip: int
def _managed_file_table(prisma_client: PrismaClient) -> _ManagedFileTableActions:
class _RouterFileCallKwargs(TypedDict, total=False):
client: ReadOnly[AsyncOpenAI | None]
custom_llm_provider: ReadOnly[str | None]
def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
return prisma_client.db.litellm_managedfiletable
@ -213,7 +235,7 @@ def _iter_provider_file_id_pairs(
yield provider_file_id, row.unified_file_id
def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableActions:
def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedObjectTableActions:
return prisma_client.db.litellm_managedobjecttable
@ -233,7 +255,7 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
_MANAGED_FILES_TARGET: Final = "managed_files"
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
class PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
self.internal_usage_cache = internal_usage_cache
@ -1282,7 +1304,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
target_model_names_list=target_model_names_list,
litellm_parent_otel_span=litellm_parent_otel_span,
)
response = await _PROXY_LiteLLMManagedFiles.return_unified_file_id(
response = await PROXY_LiteLLMManagedFiles.return_unified_file_id(
file_objects=responses,
create_file_request=create_file_request,
internal_usage_cache=self.internal_usage_cache,
@ -1294,7 +1316,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings: Dict[str, str] = {}
for file_object in responses:
model_file_id_mapping = file_object._hidden_params.get("model_file_id_mapping")
file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata
dict[str, object], getattr(file_object, HIDDEN_PARAMS_ATTR)
)
model_file_id_mapping = file_hidden_params.get("model_file_id_mapping")
if model_file_id_mapping and isinstance(model_file_id_mapping, dict):
model_mappings.update(model_file_id_mapping)
@ -1344,7 +1369,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
_, file_type = extract_file_metadata(create_file_request["file"])
output_file_id = file_objects[0].id
model_id = file_objects[0]._hidden_params.get("model_id")
file_hidden_params: Final = cast(dict[str, object], getattr(file_objects[0], HIDDEN_PARAMS_ATTR))
model_id = file_hidden_params.get("model_id")
unified_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format(
file_type,
@ -1410,11 +1436,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if decoded_batch_id and is_litellm_executed_batch(decoded_batch_id):
return response
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id
is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
response_hidden_params: Final = cast(dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR))
unified_file_id = response_hidden_params.get("unified_file_id")
unified_batch_id = response_hidden_params.get("unified_batch_id")
is_batch_create: Final = response_hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True
model_id = cast(Optional[str], response_hidden_params.get("model_id"))
model_name = cast(Optional[str], response_hidden_params.get("model_name"))
resolved_model_name = resolve_managed_output_file_model_name(
unified_input_file_id=unified_file_id if isinstance(unified_file_id, str) else response.input_file_id,
@ -1459,13 +1486,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
_creds = _llm_router.get_deployment_credentials_with_provider(model_id) or {}
file_object = await litellm.afile_retrieve(
file_id=provider_file_id,
**_creds,
**cast(FileRetrieveCallOptions, _creds),
)
else:
file_object = await litellm.afile_retrieve(
custom_llm_provider=model_name.split("/")[0]
if model_name and "/" in model_name
else "openai", # type: ignore[arg-type]
custom_llm_provider=cast(
FileRetrieveProvider,
model_name.split("/")[0] if model_name and "/" in model_name else "openai",
),
file_id=provider_file_id,
)
verbose_logger.debug(
@ -1525,12 +1553,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
elif isinstance(response, LiteLLMFineTuningJob):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_finetuning_job_id = response._hidden_params.get(
"unified_finetuning_job_id"
) # managed finetuning job id
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
finetuning_response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
)
unified_file_id = finetuning_response_hidden_params.get("unified_file_id")
unified_finetuning_job_id = finetuning_response_hidden_params.get("unified_finetuning_job_id")
model_id = cast(Optional[str], finetuning_response_hidden_params.get("model_id"))
original_response_id = response.id
if (unified_file_id or unified_finetuning_job_id) and model_id:
response.id = self.get_unified_generic_response_id(model_id=model_id, generic_response_id=response.id)
@ -1608,8 +1636,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
try:
model_id, model_file_id = next(iter(stored_file_object.model_mappings.items()))
credentials = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(file_id=model_file_id, **credentials)
credentials: Final = llm_router.get_deployment_credentials_with_provider(model_id) or {}
response = await litellm.afile_retrieve(
file_id=model_file_id,
**cast(FileRetrieveCallOptions, credentials),
)
response.id = file_id # Replace with unified ID
return response
except Exception as e:
@ -1904,7 +1935,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
else {}
),
}
await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
await llm_router.afile_delete(
model=model_id,
file_id=model_file_id,
**cast(_RouterFileCallKwargs, delete_data),
)
async def afile_content(
self,
@ -1940,7 +1975,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials)))
else:
data.pop("_litellm_internal_model_credentials", None)
return await llm_router.afile_content(model=model_id, file_id=provider_file_id, **data) # type: ignore
return cast(
HttpxBinaryResponseContent,
await llm_router.afile_content(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, data),
),
)
except Exception as e:
exception_dict[model_id] = str(e)
raise Exception(
@ -2068,3 +2110,4 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
verbose_logger.debug(
f"Converted file {file_id} from storage backend to base64 with format {content_type}"
)
_PROXY_LiteLLMManagedFiles = PROXY_LiteLLMManagedFiles

21
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -2,11 +2,11 @@
## This hook is used to manage vector stores with target_model_names support
## It allows creating vector stores across multiple models and managing them with unified IDs
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Union, cast
from fastapi import HTTPException
import litellm
from litellm import Router, verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_logger import CustomLogger
@ -38,7 +38,7 @@ else:
PrismaClient = Any
class _PROXY_LiteLLMManagedVectorStores(
class PROXY_LiteLLMManagedVectorStores(
CustomLogger, BaseManagedResource[VectorStoreCreateResponse]
):
"""
@ -89,10 +89,10 @@ class _PROXY_LiteLLMManagedVectorStores(
# Model ID is stored in hidden params if the response object supports it
# For TypedDict responses, we need to check if _hidden_params was added
hidden_params: Dict[str, Any] = {}
hidden_params: Mapping[str, object] = {}
if hasattr(resource_object, "_hidden_params"):
hidden_params = getattr(resource_object, "_hidden_params", {}) or {}
model_id = hidden_params.get("model_id", "")
hidden_params = cast(Mapping[str, object], getattr(resource_object, "_hidden_params", {}) or {})
model_id: Final = cast(str, hidden_params.get("model_id", ""))
return generate_unified_id_string(
resource_type=self.resource_type,
@ -106,7 +106,7 @@ class _PROXY_LiteLLMManagedVectorStores(
self,
llm_router: Router,
model: str,
request_data: Dict[str, Any],
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
litellm_parent_otel_span: Span,
) -> VectorStoreCreateResponse:
"""
@ -122,10 +122,8 @@ class _PROXY_LiteLLMManagedVectorStores(
VectorStoreCreateResponse from the provider
"""
# Use the router to create the vector store
response = await llm_router.avector_store_create(
model=model, **request_data
)
return response
response: Final = await llm_router.avector_store_create(model=model, **request_data)
return cast(VectorStoreCreateResponse, response)
# ============================================================================
# VECTOR STORE CRUD OPERATIONS
@ -464,3 +462,4 @@ class _PROXY_LiteLLMManagedVectorStores(
parent_otel_span=parent_otel_span,
resource_id_key="vector_store_id",
)
_PROXY_LiteLLMManagedVectorStores = PROXY_LiteLLMManagedVectorStores

73
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -22,7 +22,9 @@ from litellm._uuid import uuid
from litellm.proxy._types import *
from litellm.proxy.auth.auth_checks import delete_cached_project_object
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management.teams.authz import is_team_admin
from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamAccess, is_team_admin, is_team_member
from litellm.proxy.management.teams.dependencies import get_team_access
from litellm.proxy.management.users.service import org_admin_org_ids
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
from litellm.proxy.management_helpers.utils import (
@ -118,6 +120,37 @@ async def _check_user_permission_for_project(
return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
async def _can_view_team_projects(
user_api_key_dict: UserAPIKeyAuth,
team_id: str | None,
prisma_client: PrismaClient,
team_access: TeamAccess,
) -> bool:
if user_api_key_has_admin_view(user_api_key_dict):
return True
if not team_id or not user_api_key_dict.user_id:
return False
team_row: Final = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
if team_row is None:
return False
team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
return is_team_member(user_api_key_dict, team) or await team_access.allows(
user_api_key_dict, team, TEAM_OR_ORG_ADMIN
)
def _visible_projects_where(team_ids: list[str], admin_org_ids: frozenset[str]) -> dict[str, object]:
# bounded-ok: one caller's team memberships
member_scope: Final[dict[str, object]] = {"team_id": {"in": team_ids}}
if not admin_org_ids:
return member_scope
org_scope: Final[dict[str, object]] = {
# bounded-ok: orgs one caller administers
"litellm_team_table": {"is": {"organization_id": {"in": sorted(admin_org_ids)}}}
}
return {"OR": [member_scope, org_scope]}
async def _validate_team_exists(
team_id: str,
prisma_client: PrismaClient,
@ -433,7 +466,7 @@ async def new_project(
```bash
curl --location 'http://0.0.0.0:4000/project/new' \\
--header 'Authorization: Bearer sk-1234' \\
--header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
--header 'Content-Type: application/json' \\
--data '{
"project_alias": "flight-search-assistant",
@ -460,7 +493,7 @@ async def new_project(
```bash
curl --location 'http://0.0.0.0:4000/project/new' \\
--header 'Authorization: Bearer sk-1234' \\
--header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
--header 'Content-Type: application/json' \\
--data '{
"project_alias": "hotel-recommendations",
@ -648,7 +681,7 @@ async def update_project(
Example:
```bash
curl --location 'http://0.0.0.0:4000/project/update' \\
--header 'Authorization: Bearer sk-1234' \\
--header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
--header 'Content-Type: application/json' \\
--data '{
"project_id": "project-123",
@ -876,7 +909,7 @@ async def delete_project(
Example:
```bash
curl --location --request DELETE 'http://0.0.0.0:4000/project/delete' \\
--header 'Authorization: Bearer sk-1234' \\
--header "Authorization: Bearer $LITELLM_MASTER_KEY" \\
--header 'Content-Type: application/json' \\
--data '{
"project_ids": ["project-123", "project-456"]
@ -973,6 +1006,7 @@ async def delete_project(
async def project_info(
project_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
team_access: TeamAccess = Depends(get_team_access),
):
"""
Get information about a specific project
@ -983,7 +1017,7 @@ async def project_info(
Example:
```bash
curl --location 'http://0.0.0.0:4000/project/info?project_id=project-123' \\
--header 'Authorization: Bearer sk-1234'
--header "Authorization: Bearer $LITELLM_MASTER_KEY"
```
"""
from litellm.proxy.proxy_server import prisma_client
@ -1009,21 +1043,7 @@ async def project_info(
param="project_id",
)
# Check if user has access to this project (admin or team member)
is_admin = user_api_key_has_admin_view(user_api_key_dict)
is_team_member = False
if project.team_id and user_api_key_dict.user_id:
team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
if team:
caller_user_id = user_api_key_dict.user_id
for m in team.members_with_roles or []:
m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
if m_user_id == caller_user_id:
is_team_member = True
break
if not (is_admin or is_team_member):
if not await _can_view_team_projects(user_api_key_dict, project.team_id, prisma_client, team_access):
raise HTTPException(
status_code=403,
detail={"error": "You don't have access to this project"},
@ -1052,7 +1072,7 @@ async def list_projects(
Example:
```bash
curl --location 'http://0.0.0.0:4000/project/list' \\
--header 'Authorization: Bearer sk-1234'
--header "Authorization: Bearer $LITELLM_MASTER_KEY"
```
"""
from litellm.proxy.proxy_server import prisma_client
@ -1072,16 +1092,13 @@ async def list_projects(
include={"litellm_budget_table": True, "object_permission": True}
)
else:
# Look up the user's team memberships via the reverse-index on
# LiteLLM_UserTable.teams (maintained by team_member_add alongside
# members_with_roles). This avoids a full scan of all team rows.
user_record: Final = await _user_table(prisma_client).find_unique(
where={"user_id": user_api_key_dict.user_id},
include={"organization_memberships": True},
)
user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
user: Final = None if user_record is None else LiteLLM_UserTable.model_validate(user_record.model_dump())
projects = await _project_table(prisma_client).find_many(
where={"team_id": {"in": user_team_ids}},
where=_visible_projects_where(user.teams if user is not None else [], org_admin_org_ids(user)),
include={"litellm_budget_table": True, "object_permission": True},
)

2
enterprise/litellm_enterprise/proxy/proxy_server.py
View file

@ -9,7 +9,7 @@ custom_auth_settings: Optional[CustomAuthSettings] = None
class EnterpriseProxyConfig:
async def load_custom_auth_settings(
self, general_settings: dict
) -> CustomAuthSettings:
) -> Optional[CustomAuthSettings]:
custom_auth_settings = general_settings.get("custom_auth_settings", None)
if custom_auth_settings is not None:
custom_auth_settings = CustomAuthSettings(

3
enterprise/litellm_enterprise/proxy/utils.py
View file

@ -3,7 +3,7 @@ from typing import Optional, Union
from litellm.secret_managers.main import str_to_bool
def _should_block_robots():
def should_block_robots():
"""
Returns True if the robots.txt file should block web crawlers
@ -33,3 +33,4 @@ def _should_block_robots():
)
return True
return False
_should_block_robots = should_block_robots

2
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -171,7 +171,7 @@ async def list_vector_stores(
try:
# Get vector stores from database (source of truth)
# Only return what's in the database to ensure consistency across instances
vector_stores_from_db = await VectorStoreRegistry._get_vector_stores_from_db(
vector_stores_from_db = await VectorStoreRegistry.get_vector_stores_from_db(
prisma_client=prisma_client
)

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.73"
version = "0.1.74"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.73"
version = "0.1.74"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -2,8 +2,8 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9a
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.25.2
ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin

13
litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_mcp_catalog_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,13 @@
CREATE OR REPLACE FUNCTION litellm_bump_mcp_catalog_revision() RETURNS TRIGGER AS $$
BEGIN
INSERT INTO "LiteLLM_Config" ("param_name", "reload_revision")
VALUES ('mcp_catalog', 1)
ON CONFLICT ("param_name") DO UPDATE SET "reload_revision" = "LiteLLM_Config"."reload_revision" + 1;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_mcp_catalog_revision" ON "LiteLLM_MCPServerTable";
CREATE TRIGGER "litellm_mcp_catalog_revision"
AFTER INSERT OR UPDATE OR DELETE ON "LiteLLM_MCPServerTable"
FOR EACH STATEMENT EXECUTE PROCEDURE litellm_bump_mcp_catalog_revision();

7
litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000000_lens_review_checkpoints/migration.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_LensReview" (
"lens_id" TEXT NOT NULL REFERENCES "LiteLLM_Lens"("id") ON DELETE CASCADE,
"criteria_key" TEXT NOT NULL,
"execution_id" TEXT NOT NULL,
"data" JSONB NOT NULL,
PRIMARY KEY ("lens_id", "criteria_key", "execution_id")
);

7
litellm-proxy-extras/litellm_proxy_extras/migrations/20261006000100_lens_datasets/migration.sql Normal file
View file

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_LensDataset" (
"id" TEXT NOT NULL,
"revision" INTEGER NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL,
"data" JSONB NOT NULL,
CONSTRAINT "LiteLLM_LensDataset_pkey" PRIMARY KEY ("id", "revision")
);

15
litellm-proxy-extras/litellm_proxy_extras/migrations/20261007000000_lens_review_fk_on_update_cascade/migration.sql Normal file
View file

@ -0,0 +1,15 @@
-- DropForeignKey
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_LensReview_lens_id_fkey' AND conrelid = '"LiteLLM_LensReview"'::regclass) THEN
ALTER TABLE "LiteLLM_LensReview" DROP CONSTRAINT "LiteLLM_LensReview_lens_id_fkey";
END IF;
END $$;
-- AddForeignKey
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_LensReview_lens_id_fkey' AND conrelid = '"LiteLLM_LensReview"'::regclass) THEN
ALTER TABLE "LiteLLM_LensReview" ADD CONSTRAINT "LiteLLM_LensReview_lens_id_fkey" FOREIGN KEY ("lens_id") REFERENCES "LiteLLM_Lens"("id") ON DELETE CASCADE ON UPDATE CASCADE;
END IF;
END $$;

20
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1935,6 +1935,7 @@ model LiteLLM_BackgroundInteractionSettlement {
}
model LiteLLM_Lens {
reviews LiteLLM_LensReview[]
id String @id
version Int @default(0)
data Json
@ -1949,8 +1950,27 @@ model LiteLLM_LensRun {
@@index([lens_id, created_at])
}
model LiteLLM_LensReview {
lens_id String
criteria_key String
execution_id String
data Json
lens LiteLLM_Lens @relation(fields: [lens_id], references: [id], onDelete: Cascade)
@@id([lens_id, criteria_key, execution_id])
}
model LiteLLM_LensWorker {
id String @id
token_hash String @unique
data Json
}
model LiteLLM_LensDataset {
id String
revision Int
created_at DateTime
data Json
@@id([id, revision])
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.105"
version = "0.4.106"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.105"
version = "0.4.106"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

3
litellm-rust/crates/callbacks-legacy-python/python_contract.json
View file

@ -88,7 +88,8 @@
"call_type"
],
"stream_opened": [
"logger"
"logger",
"head"
],
"stream_success": [
"logger",

7
litellm-rust/crates/callbacks-legacy-python/src/adapter.rs
View file

@ -536,7 +536,7 @@ impl LegacyLogging {
head.bind(py).set_item("cache_key", key)?;
head.bind(py).set_item("cache_hit", true)?;
}
Streaming::Opened.call(py, (self.logger()?.object(py),))?;
Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
self.stream = Some(DeliveredStream {
chunks: PyList::empty(py).unbind(),
first_chunk: None,
@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response
fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() {
Python::initialize();
Python::attach(|py| {
let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None");
let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}");
let mut logging = LegacyLogging {
operation: crate::LoggingOperation::Messages,
..logged(py, &locals, true)
};
logging
.on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind())
.on_stream_open(py, &local(&locals, "head").unbind())
.unwrap();
logging
.on_stream_chunk(py, &local(&locals, "first").unbind())
@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response
&locals,
c"
assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
assert logger.calls[0][1] is head
chunks = logger.calls[1][1]
assert len(chunks) == 2
assert chunks[0] is first

2
litellm-rust/crates/callbacks-legacy-python/src/test_support.rs
View file

@ -84,7 +84,7 @@ FAKES = {
'success', response, call_type
),
'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
'stream_opened': lambda logger: logger.record('stream_opened', None),
'stream_opened': lambda logger, head: logger.record('stream_opened', head),
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
'stream_success', list(chunks)
),

4
litellm-rust/crates/gateway-auth/tests/auth.rs
View file

@ -94,8 +94,8 @@ async fn enforces_configured_keys_without_exposing_secrets(
#[rstest]
fn hash_token_matches_python_sha256_hexdigest() {
assert_eq!(
hash_token("sk-1234"),
"88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"
hash_token("sk-9876"),
"595b23af2e99cee580245f388d3244a39247a25a3846e898e9c78841f8471a3e"
);
}

8
litellm-rust/crates/python-bridge/src/secrets/mod.rs
View file

@ -21,10 +21,10 @@ use crate::{coercion::FieldSpec, python_settings::PythonSettings};
const NATIVE: FieldSpec<bool> = FieldSpec::new("native", |field| field.schema_bool());
/// Where a Rust route reads provider secrets from. Python's `get_secret_str` until a
/// `SecretManagerRule` in `catalog.py` moves the configured system off `PYTHON_ONLY`, then the
/// native secret manager. A bare extension module without the litellm package reads the process
/// environment.
/// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings
/// projection reports no configured secret manager, otherwise the resolved manager: the native
/// backend when the configured client captures one, the Python callback when it does not. A bare
/// extension module without the litellm package reads the process environment.
pub(crate) fn source(py: Python<'_>) -> PyResult<Arc<dyn SecretSource>> {
let Some(snapshot) = PythonSettings::SecretManager.read_or_unset(py)? else {
return Ok(Arc::new(EnvironmentSecrets::python_compatible(

26
litellm-rust/crates/traces-cache/src/cache.rs
View file

@ -58,9 +58,8 @@ impl SnapshotKey {
}
}
/// How long a read result stays reusable: traces still receiving spans, or read with spend
/// unavailable, are re-read after `LIVE_TTL`; traces quiet for `SETTLED_AFTER_MS` are kept for
/// `SETTLED_TTL`.
/// Native sessions can resume without a terminal record, so their reads retain `LIVE_TTL`.
/// Other traces with known spend settle after `SETTLED_AFTER_MS` of inactivity.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Freshness {
Live,
@ -69,6 +68,12 @@ pub enum Freshness {
impl Freshness {
pub fn of(rows: &[TraceSpansRow], spend_known: bool, snapshot_ms: u64) -> Self {
if rows
.iter()
.any(|row| matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk"))
{
return Self::Live;
}
let last_end_ms = rows
.iter()
.map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns) / 1_000_000)
@ -298,6 +303,7 @@ mod tests {
fn row(span_id: &str) -> TraceSpansRow {
TraceSpansRow {
trace_id: String::new(),
original_trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: String::new(),
name: "run".into(),
@ -392,4 +398,18 @@ mod tests {
expected
);
}
#[rstest]
#[case::claude_code("claude-code")]
#[case::claude_agent_sdk("claude-agent-sdk")]
fn idle_native_sessions_remain_live(#[case] framework: &str) {
let native = TraceSpansRow {
framework: framework.into(),
..row("native")
};
assert_eq!(
Freshness::of(&[row("root"), native], true, LAST_END_MS + SETTLED_AFTER_MS),
Freshness::Live
);
}
}

1
litellm-rust/crates/traces-cache/src/spend.rs
View file

@ -48,6 +48,7 @@ pub(super) async fn spend<S: TraceStore>(
let params = SpendByResponseIdsParams {
access: access.clone(),
response_ids: lookup.response_ids,
provider_request_ids: lookup.provider_request_ids,
request_ids: lookup.request_ids,
trace_ids: lookup.trace_ids,
start_ms: window.start,

50
litellm-rust/crates/traces-cache/tests/read.rs
View file

@ -260,6 +260,7 @@ fn access() -> ReadAccessParams {
fn span(index: usize) -> TraceSpansRow {
TraceSpansRow {
trace_id: "trace".into(),
original_trace_id: String::new(),
span_id: format!("span-{index}"),
parent_span_id: if index == 0 {
String::new()
@ -523,6 +524,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow {
litellm_call_id: String::new(),
response_id: response_id.into(),
upstream_response_id: String::new(),
provider_request_id: String::new(),
trace_id: String::new(),
span_id: String::new(),
team_id: "team".into(),
@ -704,6 +706,54 @@ async fn failed_reads_are_not_cached() {
assert_eq!(store.calls(Operation::TraceSpans), 2);
}
#[rstest]
#[case::claude_code("claude-code")]
#[case::claude_agent_sdk("claude-agent-sdk")]
#[tokio::test]
async fn resumed_native_sessions_refresh_after_live_ttl(#[case] framework: &str) {
let original = TraceSpansRow {
framework: framework.into(),
..span(0)
};
let store = FakeStore::with_spans("ref", vec![original.clone()]);
let reader = TraceReader::new(usize::MAX);
let access = access();
let first = reader
.get_trace(&store, &access, "trace", "ref")
.await
.unwrap()
.unwrap();
assert_eq!(first.spans.len(), 1);
store.state.lock().unwrap().trace_spans.insert(
"ref".into(),
vec![
original,
TraceSpansRow {
start_ns: now_ns(),
..span(1)
},
],
);
let cached = reader
.get_trace(&store, &access, "trace", "ref")
.await
.unwrap()
.unwrap();
assert_eq!(cached.spans.len(), 1);
assert_eq!(store.calls(Operation::TraceSpans), 1);
tokio::time::sleep(LIVE_TTL + Duration::from_millis(200)).await;
let resumed = reader
.get_trace(&store, &access, "trace", "ref")
.await
.unwrap()
.unwrap();
assert_eq!(resumed.spans.len(), 2);
assert_eq!(store.calls(Operation::TraceSpans), 2);
assert_eq!(first.spans.len(), 1);
}
#[rstest]
#[tokio::test]
async fn listed_runs_are_read_once_until_a_live_run_expires() {

1
litellm-rust/crates/traces-cache/tests/snapshots.rs
View file

@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120);
fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
TraceSpansRow {
trace_id: String::new(),
original_trace_id: String::new(),
span_id: span_id.into(),
parent_span_id: parent.into(),
name: name.into(),

4
litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql Normal file
View file

@ -0,0 +1,4 @@
ALTER TABLE {database}.spend_logs
ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id,
ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id
TYPE bloom_filter(0.001) GRANULARITY 1

5
litellm-rust/crates/traces-clickhouse/query/spend_batch.sql
View file

@ -1,5 +1,5 @@
SELECT * FROM (
SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend,
toUnixTimestamp64Milli(start_time) AS start_ms
FROM (
SELECT *,
@ -15,7 +15,8 @@ FROM (
OR ({user_id:String} != '' AND user = {user_id:String})
OR has({team_ids:Array(String)}, team_id))
)
WHERE response_id IN {response_ids:Array(String)}
WHERE provider_request_id IN {provider_request_ids:Array(String)}
OR response_id IN {response_ids:Array(String)}
OR upstream_response_id IN {response_ids:Array(String)}
OR litellm_call_id IN {request_ids:Array(String)}
OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})

5
litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql
View file

@ -1,4 +1,4 @@
SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend,
toUnixTimestamp64Milli(start_time) AS start_ms
FROM (
SELECT *,
@ -14,7 +14,8 @@ FROM (
OR ({user_id:String} != '' AND user = {user_id:String})
OR has({team_ids:Array(String)}, team_id))
)
WHERE response_id IN {response_ids:Array(String)}
WHERE provider_request_id IN {provider_request_ids:Array(String)}
OR response_id IN {response_ids:Array(String)}
OR upstream_response_id IN {response_ids:Array(String)}
OR litellm_call_id IN {request_ids:Array(String)}
OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})

2
litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql
View file

@ -1,5 +1,5 @@
SELECT * FROM (
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,

2
litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql
View file

@ -1,4 +1,4 @@
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,

2
litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql
View file

@ -1,5 +1,5 @@
SELECT * FROM (
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,

2
litellm-rust/crates/traces-clickhouse/query/trace_spans.sql
View file

@ -1,4 +1,4 @@
SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
o.Framework AS framework, o.StatusCode AS status,
substringUTF8(o.StatusMessage, 1, 128) AS status_message,

11
litellm-rust/crates/traces-clickhouse/src/query/named.rs
View file

@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams;
struct TraceSpansRowEncoding {
#[serde(default)]
pub trace_id: String,
#[serde(default)]
pub original_trace_id: String,
pub span_id: String,
pub parent_span_id: String,
pub name: String,
@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding {
#[serde(flatten)]
pub access: contracts::ReadAccessParams,
pub response_ids: Vec<String>,
pub provider_request_ids: Vec<String>,
pub request_ids: Vec<String>,
pub trace_ids: Vec<String>,
#[serde(deserialize_with = "super::number::deserialize")]
@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding {
pub litellm_call_id: String,
pub response_id: String,
pub upstream_response_id: String,
#[serde(default)]
pub provider_request_id: String,
pub trace_id: String,
pub span_id: String,
pub team_id: String,
@ -344,7 +349,7 @@ mod tests {
quoted,
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
quoted,
);
round_trip::<SpanDetailRow>(
@ -356,7 +361,7 @@ mod tests {
quoted,
);
round_trip::<SpendByResponseIdsRow>(
json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
quoted,
);
}
@ -374,7 +379,7 @@ mod tests {
quoted,
);
round_trip::<SpendByResponseIdsParams>(
json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
quoted,
);
}

5
litellm-rust/crates/traces-clickhouse/src/span_row.rs
View file

@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str {
.flatten()
.find_map(|key| match key {
CallKey::ProviderResponse(id) => Some(id.as_str()),
CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None,
CallKey::LiteLlmRequest(_)
| CallKey::ProviderRequest(_)
| CallKey::Transport
| CallKey::GatewayAttempt => None,
})
.unwrap_or_default()
}

5
litellm-rust/crates/traces-clickhouse/tests/migrations.rs
View file

@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins(
Parameter::Strings(vec!["response-1".into()]),
),
("request_ids".into(), Parameter::Strings(Vec::new())),
(
"provider_request_ids".into(),
Parameter::Strings(Vec::new()),
),
("trace_ids".into(), Parameter::Strings(Vec::new())),
("all_teams".into(), Parameter::Integer(0)),
("user_id".into(), Parameter::Text(String::new())),
@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility(
"api_key_hash": legacy_key.unwrap_or_default(),
}))?,
response_ids: vec!["shared-response".into()],
provider_request_ids: Vec::new(),
request_ids: Vec::new(),
trace_ids: Vec::new(),
start_ms: timestamp / 1_000_000 - 1,

3
litellm-rust/crates/traces-clickhouse/tests/queries/support.rs
View file

@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap<String, Value
.find_map(|key| match key {
litellm_traces::CallKey::LiteLlmRequest(id)
| litellm_traces::CallKey::ProviderResponse(id) => Some(id.as_str()),
litellm_traces::CallKey::Transport
litellm_traces::CallKey::ProviderRequest(_)
| litellm_traces::CallKey::Transport
| litellm_traces::CallKey::GatewayAttempt => None,
})
.unwrap_or_default()

194
litellm-rust/crates/traces-clickhouse/tests/reads.rs
View file

@ -600,9 +600,17 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals(
}
#[rstest]
#[case::same_key("same-key", Some(0.25))]
#[case::other_key("other-key", None)]
#[case::foreign_team("foreign-team", None)]
#[case::call_id_other_key("call-id-other-key", None)]
#[case::call_id_foreign_team("call-id-foreign-team", None)]
#[case::transport_only("transport-only", None)]
#[tokio::test]
async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback(
async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads(
#[future(awt)] migrated_database: TestResult<SeededDatabase>,
#[case] id: &str,
#[case] expected: Option<f64>,
) -> TestResult {
let fixture = migrated_database?;
let client = &fixture.database.client;
@ -610,31 +618,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
let start_ms = 1_790_000_000_000_i64;
let cases = [
(
"gateway",
"provider-request",
"gateway",
"same-key",
"provider_response:same-key",
"team-a",
"key-a",
Some(0.25),
),
("legacy", "legacy", "", "team-a", "key-a", Some(0.25)),
("conflict", "conflict", "different", "team-a", "key-a", None),
(
"other-key",
"provider_response:other-key",
"team-a",
"key-b",
None,
),
(
"foreign-team",
"request",
"foreign-team",
"provider_response:foreign-team",
"team-b",
"key-a",
None,
),
(
"foreign-key",
"request",
"foreign-key",
"call-id-other-key",
"litellm_request:call-id-other-key",
"team-a",
"key-b",
None,
),
(
"call-id-foreign-team",
"litellm_request:call-id-foreign-team",
"team-b",
"key-a",
None,
),
("transport-only", "transport:", "team-a", "key-a", None),
];
insert_rows(
client,
@ -643,7 +661,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
InsertTable::OtelTraces,
cases
.iter()
.map(|(id, _, _, _, _, _)| {
.map(|(id, key, _, _, _)| {
BTreeMap::from([
("Timestamp".into(), json!(start_ms * 1_000_000)),
("Duration".into(), json!(1_000_000)),
@ -652,7 +670,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
("ObservationType".into(), json!("llm")),
("TeamId".into(), json!("team-a")),
("ApiKeyHash".into(), json!("key-a")),
("CallKeys".into(), json!([format!("litellm_request:{id}")])),
("CallKeys".into(), json!([key])),
("CallEvidence".into(), json!("complete")),
])
})
@ -666,11 +684,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
InsertTable::SpendLogs,
cases
.iter()
.map(|(_, request, call_id, team, key, _)| {
.map(|(id, _, team, key, _)| {
BTreeMap::from([
("request_id".into(), json!(request)),
("response_id".into(), json!("provider-response")),
("litellm_call_id".into(), json!(call_id)),
("request_id".into(), json!(format!("request-{id}"))),
("response_id".into(), json!(id)),
("litellm_call_id".into(), json!(id)),
("team_id".into(), json!(team)),
("api_key".into(), json!(key)),
("start_time".into(), json!(start_ms)),
@ -689,24 +707,138 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
let access = ReadAccessParams {
all_teams: false,
user_id: String::new(),
team_ids: vec!["team-a".into()],
team_ids: vec!["team-a".into(), "team-b".into()],
};
let page = reader
.list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
.await?;
assert_eq!(page.data.len(), cases.len());
for (id, _, _, _, _, expected) in cases {
let summary = page
.data
.iter()
.find(|summary| summary.trace_id == id)
.ok_or("missing run")?;
let detail = reader
.get_trace(&store, &access, id, &summary.trace_ref)
.await?
.ok_or("missing trace")?;
assert_eq!(detail.summary.spend, expected, "{id}");
assert_eq!(summary.spend, expected, "{id}");
}
let summary = page
.data
.iter()
.find(|summary| summary.trace_id == id)
.ok_or("missing run")?;
let detail = reader
.get_trace(&store, &access, id, &summary.trace_ref)
.await?
.ok_or("missing trace")?;
assert_eq!(detail.summary.spend, expected, "{id}");
assert_eq!(summary.spend, expected, "{id}");
assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
Ok(())
}
#[rstest]
#[case::provider_request(true)]
#[case::transport(false)]
#[tokio::test]
async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners(
#[future(awt)] migrated_database: TestResult<SeededDatabase>,
#[case] response_header: bool,
#[values(false, true)] grouped: bool,
) -> TestResult {
let fixture = migrated_database?;
let client = &fixture.database.client;
let writer = Connection::writer(&fixture.database.url)?;
let trace_id = if grouped {
"grouped-trace"
} else {
"original-trace"
};
let start_ms = 1_790_000_000_000_i64;
let keys = if response_header {
vec!["provider_response:req_native"]
} else {
Vec::new()
};
insert_rows(
client,
&writer,
DATABASE,
InsertTable::OtelTraces,
vec![BTreeMap::from([
("Timestamp".into(), json!(start_ms * 1_000_000)),
("TraceId".into(), json!(trace_id)),
("SpanId".into(), json!("native-call")),
("SpanName".into(), json!("claude_code.llm_request")),
("ObservationType".into(), json!("llm")),
("Framework".into(), json!("claude-code")),
("TeamId".into(), json!("team-a")),
("ApiKeyHash".into(), json!("key-a")),
("CallKeys".into(), json!(keys)),
(
"CallEvidence".into(),
json!(if response_header {
"complete"
} else {
"unknown"
}),
),
(
"SpanAttributes".into(),
json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}),
),
])],
)
.await?;
insert_rows(
client,
&writer,
DATABASE,
InsertTable::SpendLogs,
["key-a", "key-b"]
.into_iter()
.map(|key| {
BTreeMap::from([
("request_id".into(), json!(format!("log-{key}"))),
("response_id".into(), json!("msg_native")),
("provider_request_id".into(), json!("req_native")),
(
"trace_id".into(),
json!(if response_header {
""
} else {
"original-trace"
}),
),
(
"span_id".into(),
json!(if response_header { "" } else { "native-call" }),
),
("team_id".into(), json!("team-a")),
("api_key".into(), json!(key)),
("start_time".into(), json!(start_ms)),
("end_time".into(), json!(start_ms + 1)),
("spend".into(), json!(0.25)),
])
})
.collect(),
)
.await?;
let connection = fixture
.readers
.connection(client, &QueryScope::All, "fixture-secret")
.await?;
let (reader, store) = make_reader(client, connection);
let access = ReadAccessParams {
all_teams: true,
user_id: String::new(),
team_ids: Vec::new(),
};
let page = reader
.list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
.await?;
assert_eq!(page.data.len(), 1);
let summary = &page.data[0];
let detail = reader
.get_trace(&store, &access, trace_id, &summary.trace_ref)
.await?
.ok_or("missing trace")?;
assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1));
assert_eq!(detail.summary.spend, summary.spend);
assert_eq!(
detail.spans[0].spend_log_request_id.as_deref(),
Some("log-key-a")
);
Ok(())
}

3
litellm-rust/crates/traces/src/lib.rs
View file

@ -44,5 +44,6 @@ pub use tenant::Tenant;
pub use truncate::{truncate_messages, truncate_value};
pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content};
pub use view::{
AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary,
AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage,
TraceSummary,
};

25
litellm-rust/crates/traces/src/normalize/format/claude_code.rs
View file

@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts};
use crate::{
Error,
normalize::{
CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey,
CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence,
ObservationType, RoleEvidence, SpanContext, attr, present, tokens,
},
otlp::DecodedEvent,
@ -159,10 +159,23 @@ fn exported_tool_results(attributes: &BTreeMap<String, String>) -> String {
let Ok(body) = serde_json::from_str::<Value>(attr(attributes, "body")) else {
return json!({"warning": "Claude's API body export is missing or truncated. Some tool results may be unavailable."}).to_string();
};
let results: Vec<Value> = body.get("messages").and_then(Value::as_array)
.and_then(|messages| messages.last())
.filter(|message| message.get("role").and_then(Value::as_str) == Some("user"))
.and_then(|message| message.get("content").and_then(Value::as_array))
let message = body
.get("messages")
.and_then(Value::as_array)
.and_then(|messages| {
messages
.iter()
.rev()
.find(|message| message.get("role").and_then(Value::as_str) != Some("system"))
})
.filter(|message| message.get("role").and_then(Value::as_str) == Some("user"));
let Some(content) = message.and_then(|message| message.get("content")) else {
return json!({"warning": "Claude's API body export has an unexpected message shape. Some tool results may be unavailable."}).to_string();
};
if !content.is_array() && !content.is_string() {
return json!({"warning": "Claude's API body export has an unexpected content shape. Some tool results may be unavailable."}).to_string();
}
let results: Vec<Value> = content.as_array()
.into_iter()
.flatten()
.filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result"))
@ -265,7 +278,7 @@ impl Format for ClaudeCode {
output: llm_output(attributes),
calls: present(attributes, &["gen_ai.response.id", "request_id"])
.map_or(CallEvidence::Unknown, |id| {
CallEvidence::complete(CallKey::ProviderResponse(id))
CallEvidence::complete(crate::normalize::claude_call_key(id))
}),
..base
},

11
litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs
View file

@ -216,7 +216,16 @@ impl Instrumentation {
fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts {
let calls = [
present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse),
present(context.attributes, &["gen_ai.response.id"]).map(|id| {
if matches!(
context.scope,
crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE
) {
crate::normalize::claude_call_key(id)
} else {
CallKey::ProviderResponse(id)
}
}),
present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest),
]
.into_iter()

7
litellm-rust/crates/traces/src/normalize/messages.rs
View file

@ -422,11 +422,10 @@ pub(super) fn event_payload(events: &[(bool, Value)], output: bool) -> Option<St
(!values.is_empty()).then(|| canonical(&encode(&values)))
}
/// The latest user message with text.
/// The first user message with text.
pub(super) fn preview(messages: &[Message]) -> String {
messages
.iter()
.rev()
.find(|message| message.role == "user" && !message.content.is_empty())
.map_or("", |message| message.content.as_str())
.chars()
@ -434,7 +433,7 @@ pub(super) fn preview(messages: &[Message]) -> String {
.collect()
}
/// The latest user message when `input` is a conversation, else the input itself.
/// The first user message when `input` is a conversation, else the input itself.
pub(super) fn input_preview(input: &str) -> String {
match serde_json::from_str::<Value>(input)
.ok()
@ -597,7 +596,7 @@ mod tests {
use serde_json::Value;
#[rstest]
#[case::latest_user(r#"{"messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"last"}]}"#, Some("last"))]
#[case::first_user(r#"{"messages":[{"role":"user","content":"first"},{"role":"assistant","content":"reply"},{"role":"user","content":"last"}]}"#, Some("first"))]
#[case::malformed("not-json", None)]
#[case::missing("{}", None)]
#[case::not_messages(r#"{"messages":[{"role":"user"}]}"#, None)]

48
litellm-rust/crates/traces/src/normalize/mod.rs
View file

@ -65,16 +65,26 @@ pub enum CallKey {
LiteLlmRequest(String),
/// The provider response id returned to the caller (`spend_logs.response_id`).
ProviderResponse(String),
ProviderRequest(String),
/// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id.
Transport,
GatewayAttempt,
}
pub(crate) fn claude_call_key(id: String) -> CallKey {
if id.starts_with("msg_") {
CallKey::ProviderResponse(id)
} else {
CallKey::ProviderRequest(id)
}
}
impl fmt::Display for CallKey {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"),
Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"),
Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"),
Self::Transport => formatter.write_str("transport:"),
Self::GatewayAttempt => formatter.write_str("gateway_attempt:"),
}
@ -89,6 +99,9 @@ impl FromStr for CallKey {
Some(("provider_response", id)) if !id.is_empty() => {
Ok(Self::ProviderResponse(id.to_owned()))
}
Some(("provider_request", id)) if !id.is_empty() => {
Ok(Self::ProviderRequest(id.to_owned()))
}
Some(("litellm_request", id)) if !id.is_empty() => {
Ok(Self::LiteLlmRequest(id.to_owned()))
}
@ -127,24 +140,45 @@ pub enum CallEvidence {
impl CallEvidence {
pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet<CallKey> {
if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() {
let native = Self::native_request(row);
let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() {
BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())])
} else {
row.call_keys.iter().cloned().collect()
};
if !native {
return keys;
}
if keys.is_empty() {
return BTreeSet::from([CallKey::Transport]);
}
keys.into_iter()
.map(|key| match key {
CallKey::ProviderResponse(id) => claude_call_key(id),
key => key,
})
.collect()
}
fn native_request(row: &crate::query::named::TraceSpansRow) -> bool {
matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk")
&& row.name == "claude_code.llm_request"
}
pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self {
let kind = row
.call_evidence
.unwrap_or(if Self::row_keys(row).is_empty() {
let keys = Self::row_keys(row);
let kind = if Self::native_request(row) {
CallEvidenceKind::Complete
} else {
row.call_evidence.unwrap_or(if keys.is_empty() {
CallEvidenceKind::Unknown
} else {
CallEvidenceKind::Complete
});
})
};
match kind {
CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)),
CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)),
CallEvidenceKind::Complete => Self::Complete(keys),
CallEvidenceKind::Partial => Self::Partial(keys),
CallEvidenceKind::Unknown => Self::Unknown,
}
}

104
litellm-rust/crates/traces/src/otlp/logs.rs
View file

@ -28,16 +28,31 @@ fn text<'a>(attributes: &'a [KeyValue], key: &str) -> &'a str {
}
}
fn message(record: LogRecord) -> Span {
fn absent_id(id: &[u8], length: usize) -> bool {
id.is_empty() || (id.len() == length && id.iter().all(|byte| *byte == 0))
}
struct LogContext {
missing_trace: bool,
missing_parent: bool,
}
fn message(record: LogRecord) -> (Span, LogContext) {
let timestamp = if record.time_unix_nano == 0 {
record.observed_time_unix_nano
} else {
record.time_unix_nano
};
let missing_trace = absent_id(&record.trace_id, 16);
let missing_parent = absent_id(&record.span_id, 8);
let mut hash = Sha256::new();
hash.update(b"litellm.claude.message.v1\0");
hash.update(&record.trace_id);
hash.update(&record.span_id);
if !missing_trace {
hash.update(&record.trace_id);
}
if !missing_parent {
hash.update(&record.span_id);
}
hash.update(text(&record.attributes, "event.name"));
let uuid = text(&record.attributes, "message.uuid");
if uuid.is_empty() {
@ -50,28 +65,53 @@ fn message(record: LogRecord) -> Span {
} else {
hash.update(uuid);
}
if missing_trace {
hash.update(b"\0unassigned\0");
hash.update(text(&record.attributes, "session.id"));
}
let identity = hash.finalize();
let trace_id = if missing_trace {
let session = text(&record.attributes, "session.id");
if session.is_empty() {
identity[..16].to_vec()
} else {
span::session_trace_id(session)
}
} else {
record.trace_id
};
let failed = match value(&record.attributes, "success") {
Some(Value::BoolValue(success)) => !success,
Some(Value::StringValue(success)) => success == "false",
_ => false,
};
Span {
trace_id: record.trace_id,
span_id: hash.finalize()[..8].to_vec(),
parent_span_id: record.span_id,
name: format!("claude_code.{}", text(&record.attributes, "event.name")),
kind: 1,
start_time_unix_nano: timestamp,
end_time_unix_nano: timestamp,
status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(|| {
Status {
code: 2,
message: text(&record.attributes, "error").to_owned(),
}
}),
attributes: record.attributes,
..Span::default()
}
(
Span {
trace_id,
span_id: identity[..8].to_vec(),
parent_span_id: if missing_trace || missing_parent {
Vec::new()
} else {
record.span_id
},
name: format!("claude_code.{}", text(&record.attributes, "event.name")),
kind: 1,
start_time_unix_nano: timestamp,
end_time_unix_nano: timestamp,
status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(
|| Status {
code: 2,
message: text(&record.attributes, "error").to_owned(),
},
),
attributes: record.attributes,
..Span::default()
},
LogContext {
missing_trace,
missing_parent,
},
)
}
pub(super) fn flatten(
@ -79,6 +119,7 @@ pub(super) fn flatten(
limits: DecodeLimits,
) -> Result<Vec<DecodedSpan>, Error> {
let mut count = 0usize;
let mut contexts = Vec::new();
let mut resources = Vec::new();
for resource in request.resource_logs {
let mut scopes = Vec::new();
@ -111,7 +152,11 @@ pub(super) fn flatten(
text(&record.attributes, "query_source"),
)))
})
.map(message)
.map(|record| {
let (span, context) = message(record);
contexts.push(context);
span
})
.collect();
scopes.push(ScopeSpans {
scope: scope.scope,
@ -125,10 +170,23 @@ pub(super) fn flatten(
schema_url: resource.schema_url,
});
}
span::flatten(
let (mut spans, mut budget) = span::flatten_with_budget(
ExportTraceServiceRequest {
resource_spans: resources,
},
limits,
)
)?;
budget.consume(contexts.len() * size_of::<LogContext>())?;
for (span, context) in spans.iter_mut().zip(contexts) {
if context.missing_trace {
span.attributes.remove("lens.original_trace_id");
}
if context.missing_trace || context.missing_parent {
let key = "lens.capture.warning";
let warning = "This native log has incomplete trace context. Its content is retained, but its execution parent is unconfirmed.";
budget.consume(key.len() + warning.len() + 96)?;
span.attributes.insert(key.to_owned(), warning.to_owned());
}
}
Ok(spans)
}

22
litellm-rust/crates/traces/src/otlp/span.rs
View file

@ -20,12 +20,19 @@ pub(super) fn flatten(
request: ExportTraceServiceRequest,
limits: DecodeLimits,
) -> Result<Vec<DecodedSpan>, Error> {
flatten_with_budget(request, limits).map(|(spans, _)| spans)
}
pub(super) fn flatten_with_budget(
request: ExportTraceServiceRequest,
limits: DecodeLimits,
) -> Result<(Vec<DecodedSpan>, Budget), Error> {
let mut budget = Budget::new(limits);
let mut spans = Vec::new();
for resource in request.resource_spans {
append_resource(resource, &mut budget, &mut spans)?;
}
Ok(spans)
Ok((spans, budget))
}
fn append_resource(
@ -124,6 +131,10 @@ fn hex_bytes(bytes: &[u8]) -> String {
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}
pub(super) fn session_trace_id(session: &str) -> Vec<u8> {
Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16].to_vec()
}
fn decoded_span(
span: Span,
resource_attributes: &Shared<BTreeMap<String, String>>,
@ -145,8 +156,7 @@ fn decoded_span(
.get("session.id")
.filter(|value| !value.is_empty())
{
let trace_id =
hex_bytes(&Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16]);
let trace_id = hex_bytes(&session_trace_id(session));
let actor = span_attributes.get("agent_id").unwrap_or(session).clone();
budget.consume(original_trace_id.len() + actor.len() + 256)?;
span_attributes.insert("lens.original_trace_id".to_owned(), original_trace_id);
@ -193,9 +203,9 @@ fn decoded_span(
.into_iter()
.flatten()
.map(|key| match key {
crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => {
id.len() + size_of::<crate::CallKey>()
}
crate::CallKey::LiteLlmRequest(id)
| crate::CallKey::ProviderResponse(id)
| crate::CallKey::ProviderRequest(id) => id.len() + size_of::<crate::CallKey>(),
crate::CallKey::Transport | crate::CallKey::GatewayAttempt => {
size_of::<crate::CallKey>()
}

15
litellm-rust/crates/traces/src/query/named.rs
View file

@ -68,6 +68,8 @@ pub struct TraceSpansParams {
pub struct TraceSpansRow {
#[serde(default)]
pub trace_id: String,
#[serde(default)]
pub original_trace_id: String,
pub span_id: String,
pub parent_span_id: String,
pub name: String,
@ -113,6 +115,16 @@ pub struct TraceSpansRow {
pub user_id: String,
}
impl TraceSpansRow {
pub(crate) fn transport_trace_id(&self) -> &str {
if self.original_trace_id.is_empty() {
&self.trace_id
} else {
&self.original_trace_id
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct TracePageSpansParams {
#[serde(flatten)]
@ -163,6 +175,7 @@ pub struct SpendByResponseIdsParams {
#[serde(flatten)]
pub access: ReadAccessParams,
pub response_ids: Vec<String>,
pub provider_request_ids: Vec<String>,
pub request_ids: Vec<String>,
pub trace_ids: Vec<String>,
pub start_ms: i64,
@ -175,6 +188,8 @@ pub struct SpendByResponseIdsRow {
pub litellm_call_id: String,
pub response_id: String,
pub upstream_response_id: String,
#[serde(default)]
pub provider_request_id: String,
pub trace_id: String,
pub span_id: String,
pub team_id: String,

35
litellm-rust/crates/traces/src/resolve/resolution.rs
View file

@ -3,6 +3,7 @@ use std::collections::HashMap;
use indexmap::IndexMap;
use crate::{
SpendMatch,
normalize::{CallKey, ObservationType},
query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow},
};
@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> {
types: HashMap<&'a str, ObservationType>,
tool_failures: HashMap<&'a str, &'a TraceSpansRow>,
pub(super) model_calls: Vec<usize>,
call_matches: HashMap<usize, CallMatch<'a>>,
}
pub(super) type CallMatch<'a> = (Option<Requests<'a>>, SpendMatch);
impl<'a> Resolution<'a> {
pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self {
let graph = Graph::new(rows);
@ -36,7 +40,7 @@ impl<'a> Resolution<'a> {
let types: HashMap<&str, ObservationType> = (0..rows.len())
.map(|index| (graph.id(index), resolved_type(&graph, index, named_agents)))
.collect();
let model_calls = (0..rows.len())
let model_calls: Vec<usize> = (0..rows.len())
.filter(|index| {
types[graph.id(*index)] == ObservationType::Llm
&& !graph
@ -45,7 +49,7 @@ impl<'a> Resolution<'a> {
.any(|descendant| types[graph.id(descendant)] == ObservationType::Llm)
})
.collect();
Self {
let resolution = Self {
ownership: Ownership {
team_id: &rows[0].team_id,
api_key_hash: &rows[0].api_key_hash,
@ -65,6 +69,16 @@ impl<'a> Resolution<'a> {
.map(|row| (row.tool_call_id.as_str(), row))
.collect(),
model_calls,
call_matches: HashMap::new(),
};
let call_matches = resolution
.model_calls
.iter()
.map(|call| (*call, resolution.resolve_call_match(*call)))
.collect();
Self {
call_matches,
..resolution
}
}
@ -118,7 +132,24 @@ impl<'a> Resolution<'a> {
spend::requests(self.row(index), &self.ownership, self.spend)
}
pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> {
self.call_matches.get(&call)
}
pub(super) fn call_requests(&self, call: usize) -> Option<Requests<'a>> {
self.call_match(call)
.and_then(|(requests, _)| requests.clone())
}
fn resolve_call_match(&self, call: usize) -> CallMatch<'a> {
if let Some(requests) = self.resolve_call_requests(call) {
return (Some(requests), SpendMatch::Matched);
}
let evidence = self.requests(call);
(None, evidence.unmatched_reason())
}
fn resolve_call_requests(&self, call: usize) -> Option<Requests<'a>> {
let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| {
self.kind(*ancestor) == ObservationType::Llm
&& self

106
litellm-rust/crates/traces/src/resolve/spend.rs
View file

@ -3,7 +3,7 @@ use std::collections::BTreeSet;
use indexmap::IndexMap;
use crate::{
CallEvidence, CallEvidenceKind, CallKey,
CallEvidence, CallEvidenceKind, CallKey, SpendMatch,
query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow},
};
@ -12,6 +12,7 @@ use crate::{
pub struct SpendLookup {
pub response_ids: Vec<String>,
pub request_ids: Vec<String>,
pub provider_request_ids: Vec<String>,
/// Traces whose transport spans LiteLLM logged by `traceparent`.
pub trace_ids: Vec<String>,
}
@ -45,13 +46,21 @@ impl SpendLookup {
})
.collect(),
),
provider_request_ids: sorted(
keys()
.filter_map(|(_, key)| match key {
CallKey::ProviderRequest(id) => Some(id.clone()),
_ => None,
})
.collect(),
),
trace_ids: sorted(
keys()
.filter_map(|(row, key)| match key {
CallKey::Transport | CallKey::GatewayAttempt
if !row.trace_id.is_empty() =>
if !row.transport_trace_id().is_empty() =>
{
Some(row.trace_id.clone())
Some(row.transport_trace_id().to_owned())
}
_ => None,
})
@ -61,7 +70,10 @@ impl SpendLookup {
}
pub fn is_empty(&self) -> bool {
self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty()
self.response_ids.is_empty()
&& self.request_ids.is_empty()
&& self.provider_request_ids.is_empty()
&& self.trace_ids.is_empty()
}
}
@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>;
enum KeyFamily {
GatewayCall,
ProviderResponse,
ProviderRequest,
Transport,
}
@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily {
match key {
CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall,
CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse,
CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest,
CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport,
}
}
pub(super) enum KeyMatch<'a> {
Missing,
Conflicting,
Unique(&'a SpendRow),
Ambiguous(Requests<'a>),
}
@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> {
fn unique(&self) -> Option<&'a SpendRow> {
match self {
Self::Unique(request) => Some(request),
Self::Missing | Self::Ambiguous(_) => None,
Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None,
}
}
fn agrees_with(&self, selected: &[&SpendRow]) -> bool {
match self {
Self::Missing => false,
Self::Missing | Self::Conflicting => false,
Self::Unique(request) => selected
.iter()
.any(|row| row.identity() == request.identity()),
@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> {
}
impl<'a> SpendEvidence<'a> {
pub(super) fn unmatched_reason(&self) -> SpendMatch {
match self {
Self::Unknown => SpendMatch::NoCallId,
Self::Partial(_) => SpendMatch::IncompleteEvidence,
Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId,
Self::Complete(matches)
if matches.iter().any(|evidence| {
matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_))
}) =>
{
SpendMatch::Ambiguous
}
Self::Complete(matches)
if matches
.iter()
.any(|evidence| matches!(evidence, KeyMatch::Missing)) =>
{
SpendMatch::NoSpendLog
}
Self::Complete(_) => SpendMatch::Ambiguous,
}
}
pub(super) fn complete_requests(&self) -> Option<Requests<'a>> {
match self {
Self::Complete(matches) if !matches.is_empty() => {
@ -186,15 +224,16 @@ fn matches<'a>(
CallKey::ProviderResponse(id) => {
!id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id)
}
CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id,
CallKey::LiteLlmRequest(id) => {
!id.is_empty()
&& (spend.litellm_call_id == *id
|| (spend.litellm_call_id.is_empty() && spend.request_id == *id))
}
CallKey::Transport | CallKey::GatewayAttempt => {
!row.trace_id.is_empty()
!row.transport_trace_id().is_empty()
&& !row.span_id.is_empty()
&& spend.trace_id == row.trace_id
&& spend.trace_id == row.transport_trace_id()
&& spend.span_id == row.span_id
}
};
@ -253,7 +292,8 @@ pub(super) fn requests<'a>(
let matches = aliases
.into_iter()
.map(|(_, requests)| {
KeyMatch::new(
let had_candidates = !requests.is_empty();
let matched = KeyMatch::new(
requests
.into_iter()
.filter(|request| {
@ -262,7 +302,12 @@ pub(super) fn requests<'a>(
.all(|family| family.contains(&request.identity()))
})
.collect(),
)
);
if had_candidates && matches!(matched, KeyMatch::Missing) {
KeyMatch::Conflicting
} else {
matched
}
})
.collect();
match evidence.kind() {
@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option<f64> {
})
}
pub(super) fn total(calls: &[Option<Requests<'_>>]) -> Option<f64> {
if calls.is_empty() {
return None;
}
let requests: Option<Vec<&SpendRow>> = calls
.iter()
.map(|requests| requests.as_ref())
.collect::<Option<Vec<_>>>()
.map(|calls| calls.into_iter().flatten().copied().collect());
let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests?
pub(super) fn unique<'a>(requests: impl IntoIterator<Item = &'a SpendRow>) -> Requests<'a> {
requests
.into_iter()
.map(|request| (request.identity(), request))
.collect();
request_cost(&unique.into_values().collect::<Vec<_>>())
.collect::<IndexMap<_, _>>()
.into_values()
.collect()
}
pub(super) struct Priced {
pub(super) spend: Option<f64>,
pub(super) priced_calls: u64,
}
pub(super) fn total(calls: &[Option<Requests<'_>>]) -> Priced {
let priced: Vec<&Requests<'_>> = calls
.iter()
.flatten()
.filter(|requests| request_cost(requests).is_some())
.collect();
let spend = if priced.is_empty() {
None
} else {
request_cost(&unique(
priced.iter().flat_map(|requests| requests.iter().copied()),
))
};
Priced {
spend,
priced_calls: priced.len() as u64,
}
}

40
litellm-rust/crates/traces/src/resolve/view.rs
View file

@ -6,7 +6,7 @@ use time::OffsetDateTime;
use crate::{
normalize::ObservationType,
query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow},
view::{AgentNode, Span, SpanStatus, Trace, TraceSummary},
view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary},
};
use super::{
@ -23,7 +23,18 @@ fn optional(value: &str) -> Option<String> {
fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span {
let row = resolution.row(index);
let status = resolution.status_source(index);
let requests = resolution.requests(index).complete_requests();
let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) {
(requests.clone(), Some(*matched))
} else {
(resolution.requests(index).complete_requests(), None)
};
let spend = requests
.as_ref()
.and_then(|requests| request_cost(requests));
let spend_log_request_id = match (spend_match, requests.as_deref()) {
(Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()),
_ => None,
};
Span {
span_id: row.span_id.clone(),
parent_span_id: optional(&row.parent_span_id),
@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span
input_tokens: row.input_tokens,
output_tokens: row.output_tokens,
litellm_request_id: optional(&row.litellm_request_id),
spend: requests
.as_ref()
.and_then(|requests| request_cost(requests)),
spend,
spend_log_request_id,
spend_match,
}
}
@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
.filter(|(owner, _)| *owner == name)
.map(|(_, requests)| requests.clone())
.collect();
let priced = total(&owned_calls);
AgentNode {
name: name.to_owned(),
parent_agent,
@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
.map(|span| graph.rows[*span].duration_ns)
.sum::<u64>() as f64
/ NANOS_PER_MS,
spend: total(&owned_calls),
spend: priced.spend,
priced_calls: priced.priced_calls,
}
})
.collect()
@ -160,6 +173,12 @@ pub fn resolve_trace(
} else {
calls.iter().map(|call| &rows[*call]).collect()
};
let priced = total(
&calls
.iter()
.map(|call| resolution.call_requests(*call))
.collect::<Vec<_>>(),
);
let first_input = spans
.iter()
.zip(rows)
@ -208,12 +227,8 @@ pub fn resolve_trace(
input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(),
output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(),
models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())),
spend: total(
&calls
.iter()
.map(|call| resolution.call_requests(*call))
.collect::<Vec<_>>(),
),
spend: priced.spend,
priced_calls: priced.priced_calls,
};
Some(Trace {
summary,
@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary {
output_tokens: row.output_tokens,
models: row.models.clone(),
spend: None,
priced_calls: 0,
}
}

68
litellm-rust/crates/traces/src/ui.rs
View file

@ -85,6 +85,32 @@ struct RawMessage {
kwargs: Option<Box<RawMessage>>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct AssistantSummary {
content: Option<String>,
tool_names: Vec<String>,
}
impl AssistantSummary {
fn into_ui(self) -> UiMessage {
let calls: Vec<UiToolCall> = self
.tool_names
.into_iter()
.map(|name| UiToolCall {
name,
arguments: "{}".to_owned(),
})
.collect();
UiMessage {
role: ChatRole::Assistant,
content: self.content.unwrap_or_default(),
name: None,
tool_calls: (!calls.is_empty()).then_some(calls),
}
}
}
#[derive(Deserialize)]
struct ContentBlock {
#[serde(rename = "type", default)]
@ -211,6 +237,19 @@ fn messages(parsed: &Value) -> Option<Vec<UiMessage>> {
Some(unwrapped.into_iter().map(RawMessage::into_ui).collect())
}
fn assistant_summaries(parsed: &Value) -> Option<Vec<UiMessage>> {
let summaries = Vec::<AssistantSummary>::deserialize(parsed).ok()?;
if summaries.is_empty() {
return None;
}
Some(
summaries
.into_iter()
.map(AssistantSummary::into_ui)
.collect(),
)
}
pub fn to_ui_content(raw: &str) -> UiContent {
let text = || UiContent::Text {
text: raw.to_owned(),
@ -223,7 +262,7 @@ pub fn to_ui_content(raw: &str) -> UiContent {
Ok(parsed @ (Value::Array(_) | Value::Object(_))) => parsed,
_ => return text(),
};
if let Some(messages) = messages(&parsed) {
if let Some(messages) = messages(&parsed).or_else(|| assistant_summaries(&parsed)) {
return UiContent::Messages { messages };
}
match parsed {
@ -356,6 +395,33 @@ mod tests {
);
}
#[rstest]
fn assistant_summaries_become_assistant_messages() {
let raw = json!([
{"content": "`/etc/hosts` has 11 lines.", "tool_names": []},
{"content": null, "tool_names": ["terminal"]},
]);
assert_eq!(
to_ui_content(&raw.to_string()),
UiContent::Messages {
messages: vec![
message("assistant", "`/etc/hosts` has 11 lines."),
UiMessage {
tool_calls: Some(vec![call("terminal", "{}")]),
..message("assistant", "")
},
]
}
);
}
#[rstest]
#[case::extra_field(r#"[{"content": "x", "tool_names": [], "score": 1}]"#)]
#[case::missing_tool_names(r#"[{"content": "x"}]"#)]
fn near_summaries_stay_text(#[case] raw: &str) {
assert!(matches!(to_ui_content(raw), UiContent::Text { .. }));
}
#[rstest]
fn plain_objects_become_fields_in_key_order() {
let raw = r#"{"zeta": "plain", "alpha": {"nested": [1, 2]}, "count": 3, "missing": null}"#;

15
litellm-rust/crates/traces/src/view.rs
View file

@ -37,6 +37,19 @@ pub struct Span {
pub output_tokens: u32,
pub litellm_request_id: Option<String>,
pub spend: Option<f64>,
pub spend_log_request_id: Option<String>,
pub spend_match: Option<SpendMatch>,
}
#[macro_rules_attribute::apply(response_type)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum SpendMatch {
Matched,
NoCallId,
NoSpendLog,
Ambiguous,
IncompleteEvidence,
}
/// One distinct agent in a trace: 200 invocations of `researcher` are one node.
@ -50,6 +63,7 @@ pub struct AgentNode {
pub tool_calls: u64,
pub duration_ms: f64,
pub spend: Option<f64>,
pub priced_calls: u64,
}
#[macro_rules_attribute::apply(response_type)]
@ -80,6 +94,7 @@ pub struct TraceSummary {
pub output_tokens: u64,
pub models: Vec<String>,
pub spend: Option<f64>,
pub priced_calls: u64,
}
#[macro_rules_attribute::apply(response_type)]

9
litellm-rust/crates/traces/tests/captures.rs
View file

@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec<SpendByResponse
litellm_call_id: record.litellm_call_id,
response_id: record.response_id,
upstream_response_id,
provider_request_id: String::new(),
trace_id: record.trace_id,
span_id: record.span_id,
team_id: record.team_id,
@ -171,12 +172,13 @@ fn trace_span(span: DecodedSpan) -> TraceSpansRow {
.into_iter()
.flatten()
.find_map(|key| match key {
CallKey::ProviderResponse(id) => Some(id.clone()),
CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()),
CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None,
})
.unwrap_or_default();
TraceSpansRow {
trace_id: span.trace_id,
original_trace_id: String::new(),
span_id: span.span_id,
parent_span_id: span.parent_span_id,
name: span.name,
@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow {
.expect("valid unrelated transport timestamp");
TraceSpansRow {
trace_id: call.trace_id.clone(),
original_trace_id: call.original_trace_id.clone(),
span_id: format!("unrelated-transport-{}", call.span_id),
parent_span_id: call.parent_span_id.clone(),
name: "unrelated-http".into(),
@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence(
let response_ids: Vec<_> = keys
.iter()
.filter_map(|key| match key {
CallKey::ProviderResponse(id) => Some(id.clone()),
CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => {
Some(id.clone())
}
CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => {
None
}

44
litellm-rust/crates/traces/tests/normalization_formats.rs
View file

@ -502,6 +502,25 @@ fn existing_formats_win_over_new_formats(span: Span, #[case] kind: &str) {
assert_eq!(decoded.normalized.observation_type, ObservationType::Llm);
}
#[rstest]
fn input_preview_is_the_first_user_message(span: Span) {
let conversation = json!([
{"role": "system", "content": "sys"},
{"role": "user", "content": "initial question"},
{"role": "assistant", "content": "answer"},
{"role": "user", "content": "follow up"},
])
.to_string();
let decoded = decode(
span,
"custom",
&[("gen_ai.input.messages", &conversation)],
vec![],
)
.unwrap();
assert_eq!(decoded.normalized.input_preview, "initial question");
}
#[rstest]
#[case::messages(&[("gen_ai.input.messages", r#"[{"role":"user","content":"modern"}]"#)], "modern")]
#[case::indexed(&[("gen_ai.prompt.0.role", "user"), ("gen_ai.prompt.0.content", "indexed")], "indexed")]
@ -928,3 +947,28 @@ fn convention_markers_keep_genai_call_evidence(
let marked = decode(span, scope, &marked_attributes, vec![]).unwrap();
assert_eq!(marked.normalized.calls, plain.normalized.calls);
}
#[rstest]
#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))]
#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))]
fn native_claude_preserves_the_provider_id_family(
span: Span,
#[case] id: &str,
#[case] key: CallKey,
) {
let native = Span {
name: "claude_code.llm_request".into(),
..span
};
let decoded = decode(
native,
"com.anthropic.claude_code.tracing",
&[("gen_ai.response.id", id)],
Vec::new(),
)
.unwrap();
assert_eq!(
decoded.normalized.calls,
CallEvidence::Complete(std::collections::BTreeSet::from([key]))
);
}

4
litellm-rust/crates/traces/tests/normalize.rs
View file

@ -82,7 +82,7 @@ fn assert_invariants(span: &DecodedSpan) {
}
assert!(normalized.input_preview.chars().count() <= 240);
if let Ok(Value::Array(messages)) = serde_json::from_str(&normalized.input) {
let user = messages.iter().rev().find_map(|message| {
let user = messages.iter().find_map(|message| {
(message.get("role")?.as_str()? == "user")
.then(|| {
message
@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) {
#[rstest]
#[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))]
#[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))]
#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))]
#[case::transport(litellm_traces::CallKey::Transport)]
#[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)]
fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) {
@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) {
#[rstest]
#[case::missing_separator("provider_response")]
#[case::missing_response("provider_response:")]
#[case::missing_provider_request("provider_request:")]
#[case::missing_request("litellm_request:")]
#[case::transport_id("transport:unexpected")]
#[case::gateway_attempt_separator("gateway_attempt")]

150
litellm-rust/crates/traces/tests/otlp.rs
View file

@ -1673,7 +1673,7 @@ fn session_capture_joins_native_logs_and_traces_across_turns_without_changing_sp
#[rstest]
#[case::short_trace(vec![1;15], vec![2;8], 1)]
#[case::zero_parent(vec![1;16], vec![0;8], 1)]
#[case::short_parent(vec![1;16], vec![2;7], 1)]
#[case::timestamp(vec![1;16], vec![2;8], i64::MAX as u64 + 1)]
fn native_logs_reject_invalid_context(
#[case] trace: Vec<u8>,
@ -1692,6 +1692,106 @@ fn native_logs_reject_invalid_context(
));
}
#[rstest]
#[case::json_empty(true, false, true, true)]
#[case::protobuf_empty(false, false, true, true)]
#[case::json_zero(true, true, true, true)]
#[case::protobuf_zero(false, true, true, true)]
#[case::trace_only(true, false, true, false)]
#[case::parent_only(false, false, false, true)]
fn contextless_native_logs_preserve_the_entire_batch(
#[case] json: bool,
#[case] zero: bool,
#[case] missing_trace: bool,
#[case] missing_parent: bool,
) {
use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
use prost::Message;
let mut request = log_request("repl_main_thread");
let valid = request.resource_logs[0].scope_logs[0].log_records[0].clone();
let mut uncorrelated = valid.clone();
if missing_trace {
uncorrelated.trace_id = if zero { vec![0; 16] } else { Vec::new() };
}
if missing_parent {
uncorrelated.span_id = if zero { vec![0; 8] } else { Vec::new() };
}
let mut standalone = uncorrelated.clone();
standalone
.attributes
.retain(|attribute| attribute.key != "session.id");
request.resource_logs[0].scope_logs[0].log_records = vec![valid, uncorrelated, standalone];
request.resource_logs[0].resource = Some(opentelemetry_proto::tonic::resource::v1::Resource {
attributes: vec![KeyValue {
key: "lens.session.capture".into(),
value: Some(AnyValue {
value: Some(Value::StringValue("true".into())),
}),
..Default::default()
}],
..Default::default()
});
let bytes = if json {
serde_json::to_vec(&request).unwrap()
} else {
request.encode_to_vec()
};
let media = json.then_some("application/json");
let limits = litellm_traces::DecodeLimits {
attributes: 6,
..Default::default()
};
let spans = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
let replayed = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
assert_eq!(spans.len(), 3);
assert_eq!(
serde_json::to_value(&spans).unwrap(),
serde_json::to_value(&replayed).unwrap()
);
assert_eq!(spans[0].trace_id, spans[1].trace_id);
assert_ne!(spans[1].trace_id, spans[2].trace_id);
assert_ne!(spans[0].span_id, spans[1].span_id);
if missing_trace {
assert_ne!(spans[1].span_id, spans[2].span_id);
assert!(!spans[1].attributes.contains_key("lens.original_trace_id"));
}
assert_eq!(spans[0].parent_span_id, "02".repeat(8));
assert!(spans[1].parent_span_id.is_empty());
assert!(spans[2].parent_span_id.is_empty());
assert!(!spans[0].attributes.contains_key("lens.capture.warning"));
assert!(spans[1].attributes["lens.capture.warning"].contains("unconfirmed"));
assert!(spans[2].attributes["lens.capture.warning"].contains("unconfirmed"));
assert_eq!(spans[0].normalized.output, spans[1].normalized.output);
assert_eq!(spans[0].normalized.output, spans[2].normalized.output);
assert_eq!(spans[1].normalized.observation_type, ObservationType::Chain);
}
#[rstest]
#[case::session(true)]
#[case::standalone(false)]
fn absent_native_log_context_has_encoding_independent_identity(#[case] session: bool) {
use prost::Message;
let mut request = log_request("repl_main_thread");
let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
record.trace_id.clear();
record.span_id.clear();
if !session {
record.attributes.retain(|entry| entry.key != "session.id");
}
let omitted = litellm_traces::decode_otlp_logs(
&serde_json::to_vec(&request).unwrap(),
Some("application/json"),
)
.unwrap();
let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
record.trace_id = vec![0; 16];
record.span_id = vec![0; 8];
let zeroed = litellm_traces::decode_otlp_logs(&request.encode_to_vec(), None).unwrap();
assert_eq!(omitted[0].trace_id, zeroed[0].trace_id);
assert_eq!(omitted[0].span_id, zeroed[0].span_id);
assert_eq!(omitted[0].normalized.output, zeroed[0].normalized.output);
}
#[rstest]
#[case::nodes(litellm_traces::DecodeLimits { nodes: 4, ..Default::default() })]
#[case::depth(litellm_traces::DecodeLimits { depth: 2, ..Default::default() })]
@ -1778,6 +1878,18 @@ fn interactive_claude_exports_join_replies_with_native_child_execution_context()
#[rstest]
#[case::tool_result("tool_result", "", false)]
#[case::complete_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":[{"type":"text","text":"exit 3 output"},{"type":"image","source":{"data":"PRIVATE_IMAGE"}}]}]}],"system":"PRIVATE_SYSTEM"}"#, false)]
#[case::headless_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":"exit 3 output"}]},{"role":"system","content":"PRIVATE_SYSTEM"}]}"#, false)]
#[case::missing_messages("api_request_body", r#"{}"#, true)]
#[case::wrong_content(
"api_request_body",
r#"{"messages":[{"role":"user","content":{}}]}"#,
true
)]
#[case::unexpected_last_message(
"api_request_body",
r#"{"messages":[{"role":"assistant","content":"unexpected"}]}"#,
true
)]
#[case::truncated_body("api_request_body", "{truncated", true)]
fn native_tool_logs_supply_arguments_and_results_without_fake_calls(
#[case] event: &str,
@ -1849,3 +1961,39 @@ fn interactive_claude_body_export_retains_failed_command_stdout() {
.unwrap();
assert_eq!(failed["content"], "Exit code 3\nRAW-EXPECTED");
}
#[rstest]
#[case::new_prompt(serde_json::json!({"role":"user", "content":"Continue"}))]
#[case::new_blocks(serde_json::json!({"role":"user", "content":[{"type":"text", "text":"Continue"}]}))]
fn native_body_exports_do_not_replay_old_tool_results(#[case] final_message: serde_json::Value) {
use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
let mut request = log_request("repl_main_thread");
let body = serde_json::json!({"messages": [
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "old-call", "content": "OLD_RESULT"}]},
{"role": "assistant", "content": "Done"}, final_message,
{"role": "system", "content": "PRIVATE_SYSTEM"}
]}).to_string();
request.resource_logs[0].scope_logs[0].log_records[0].attributes = [
("event.name", "api_request_body"),
("query_source", "repl_main_thread"),
("body", body.as_str()),
]
.into_iter()
.map(|(key, value)| KeyValue {
key: key.into(),
value: Some(AnyValue {
value: Some(Value::StringValue(value.into())),
}),
..Default::default()
})
.collect();
let spans = litellm_traces::decode_otlp_logs(
&serde_json::to_vec(&request).unwrap(),
Some("application/json"),
)
.unwrap();
assert_eq!(
serde_json::from_str::<serde_json::Value>(&spans[0].normalized.output).unwrap(),
serde_json::json!({"tool_results":[]})
);
}

6
litellm-rust/crates/traces/tests/query/named.rs
View file

@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases(
json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}),
));
round_trip::<SpendByResponseIdsParams>(request(
json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
));
}
@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() {
json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
);
round_trip::<TraceSpansRow>(
json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
);
round_trip::<SpanDetailRow>(
json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}),
@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() {
json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}),
);
round_trip::<SpendByResponseIdsRow>(
json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
);
}

Some files were not shown because too many files have changed in this diff Show more