mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
Merge branch 'litellm_internal_staging' into feature/improve-gigachat-provider
This commit is contained in:
KnyazSh
commit
2d75d24c34
1535 changed files with 67930 additions and 15672 deletions
|
|
@ -5,6 +5,16 @@ orbs:
|
|||
win: circleci/windows@5.0 # Add Windows orb
|
||||
|
||||
commands:
|
||||
skip_if_unrelated_changes:
|
||||
parameters:
|
||||
category:
|
||||
type: enum
|
||||
enum: ["backend", "client"]
|
||||
default: "backend"
|
||||
steps:
|
||||
- run:
|
||||
name: "Skip job when no << parameters.category >>-relevant files changed"
|
||||
command: bash .circleci/scripts/path_filter.sh << parameters.category >>
|
||||
setup_google_dns:
|
||||
steps:
|
||||
- run:
|
||||
|
|
@ -282,6 +292,7 @@ jobs:
|
|||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -354,6 +365,7 @@ jobs:
|
|||
parallelism: 4
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -427,6 +439,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -480,6 +493,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -545,6 +559,7 @@ jobs:
|
|||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -584,6 +599,7 @@ jobs:
|
|||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -624,6 +640,7 @@ jobs:
|
|||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -656,6 +673,7 @@ jobs:
|
|||
FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -705,6 +723,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -755,6 +774,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -787,6 +807,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -832,6 +853,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -877,6 +899,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -918,6 +941,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -963,6 +987,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1007,6 +1032,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -1045,6 +1071,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1089,6 +1116,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1132,6 +1160,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1163,6 +1192,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1205,6 +1235,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1248,6 +1279,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1291,6 +1323,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1321,6 +1354,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1366,6 +1400,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1407,6 +1442,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -1459,6 +1495,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1482,6 +1519,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1507,6 +1545,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1531,6 +1570,7 @@ jobs:
|
|||
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- setup_google_dns
|
||||
|
|
@ -1570,14 +1610,14 @@ jobs:
|
|||
- run:
|
||||
name: Run helm lint
|
||||
command: |
|
||||
helm lint ./deploy/charts/litellm-helm
|
||||
helm lint ./helm/litellm-helm
|
||||
|
||||
# Run helm tests
|
||||
- run:
|
||||
name: Run helm tests
|
||||
command: |
|
||||
IMAGE_TAG=${CIRCLE_SHA1:-ci}
|
||||
helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
|
||||
helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \
|
||||
--set image.repository=litellm-ci \
|
||||
--set image.tag=${IMAGE_TAG} \
|
||||
--set image.pullPolicy=Never
|
||||
|
|
@ -1606,6 +1646,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1698,6 +1739,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- setup_google_dns
|
||||
|
|
@ -1746,13 +1788,13 @@ jobs:
|
|||
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
||||
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
||||
-e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -1787,6 +1829,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1832,13 +1875,13 @@ jobs:
|
|||
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
||||
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
||||
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -1869,6 +1912,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -1911,14 +1955,14 @@ jobs:
|
|||
-e COHERE_API_KEY=$COHERE_API_KEY \
|
||||
-e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \
|
||||
-e GCS_FLUSH_INTERVAL="1" \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -1960,13 +2004,13 @@ jobs:
|
|||
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
||||
-e LITELLM_LICENSE="bad-license" \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app-3 \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug
|
||||
--port 4000
|
||||
|
||||
- run:
|
||||
name: Start outputting logs for second container
|
||||
|
|
@ -2000,6 +2044,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -2041,13 +2086,13 @@ jobs:
|
|||
-e DD_SITE=$DD_SITE \
|
||||
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
||||
-e PROXY_BATCH_WRITE_AT=2 \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2085,6 +2130,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -2117,13 +2163,13 @@ jobs:
|
|||
-e USE_DDTRACE=True \
|
||||
-e DD_API_KEY=$DD_API_KEY \
|
||||
-e DD_SITE=$DD_SITE \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Run Docker container 2
|
||||
command: |
|
||||
|
|
@ -2139,13 +2185,13 @@ jobs:
|
|||
-e USE_DDTRACE=True \
|
||||
-e DD_API_KEY=$DD_API_KEY \
|
||||
-e DD_SITE=$DD_SITE \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app-2 \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4001 \
|
||||
--detailed_debug
|
||||
--port 4001
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2180,6 +2226,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -2201,19 +2248,20 @@ jobs:
|
|||
# the OTEL test - should get this as a trace
|
||||
command: |
|
||||
docker run -d \
|
||||
--restart on-failure \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
||||
-e STORE_MODEL_IN_DB="True" \
|
||||
-e LITELLM_MASTER_KEY="sk-1234" \
|
||||
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
||||
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2252,6 +2300,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
# Remove Docker CLI installation since it's already available in machine executor
|
||||
- install_uv
|
||||
|
|
@ -2289,13 +2338,13 @@ jobs:
|
|||
-e DD_API_KEY=$DD_API_KEY \
|
||||
-e DD_SITE=$DD_SITE \
|
||||
-e GCS_FLUSH_INTERVAL="1" \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/docker/build_from_pip/litellm_config.yaml:/app/config.yaml \
|
||||
my-app:latest \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2333,6 +2382,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -2365,14 +2415,14 @@ jobs:
|
|||
-e DD_SITE=$DD_SITE \
|
||||
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||
-e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
||||
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug \
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2471,6 +2521,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- run:
|
||||
|
|
@ -2499,13 +2550,13 @@ jobs:
|
|||
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
||||
-e AWS_REGION_NAME="us-east-1" \
|
||||
-e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name my-app \
|
||||
-v $(pwd)/tests/proxy_e2e_anthropic_messages_tests/test_config.yaml:/app/config.yaml \
|
||||
litellm-docker-database:ci \
|
||||
--config /app/config.yaml \
|
||||
--port 4000 \
|
||||
--detailed_debug
|
||||
--port 4000
|
||||
- run:
|
||||
name: Start outputting logs
|
||||
command: docker logs -f my-app
|
||||
|
|
@ -2537,6 +2588,7 @@ jobs:
|
|||
- *python312_image
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: .
|
||||
# Check file locations
|
||||
|
|
@ -2567,6 +2619,8 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -2609,6 +2663,8 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
- restore_cache:
|
||||
keys:
|
||||
|
|
@ -2629,7 +2685,7 @@ jobs:
|
|||
cd ui/litellm-dashboard
|
||||
|
||||
CI=true npm run test -- --run \
|
||||
--pool forks --poolOptions.forks.maxForks=8
|
||||
--pool forks --poolOptions.forks.maxForks=6
|
||||
|
||||
e2e_ui_testing:
|
||||
docker:
|
||||
|
|
@ -2654,6 +2710,8 @@ jobs:
|
|||
PROXY_LOGOUT_URL: "https://www.example.com"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -2791,6 +2849,8 @@ jobs:
|
|||
SERVER_ROOT_PATH: "/litellm"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- restore_cache:
|
||||
|
|
@ -2892,6 +2952,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
|
||||
- run:
|
||||
name: Build Docker image
|
||||
|
|
@ -2917,6 +2978,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
- setup_google_dns
|
||||
|
|
|
|||
27
.circleci/scripts/classify_changes.sh
Executable file
27
.circleci/scripts/classify_changes.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: classify_changes.sh <backend|client>}"
|
||||
|
||||
has_client=false
|
||||
has_backend=false
|
||||
while IFS= read -r file || [ -n "$file" ]; do
|
||||
[ -n "$file" ] || continue
|
||||
case "$file" in
|
||||
ui/*) has_client=true ;;
|
||||
docs/* | *.md | *.mdx) : ;;
|
||||
*) has_backend=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$category" in
|
||||
backend)
|
||||
[ "$has_backend" = true ] && echo run || echo skip
|
||||
;;
|
||||
client)
|
||||
{ [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
|
||||
;;
|
||||
*)
|
||||
echo run
|
||||
;;
|
||||
esac
|
||||
40
.circleci/scripts/path_filter.sh
Executable file
40
.circleci/scripts/path_filter.sh
Executable file
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: path_filter.sh <backend|client>}"
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
run_full() {
|
||||
echo "path-filter[$category]: running job ($1)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
|
||||
|
||||
candidate_bases="main litellm_internal_staging litellm_oss_staging"
|
||||
merge_base=""
|
||||
for base in $candidate_bases; do
|
||||
git fetch --quiet origin "$base" 2>/dev/null || continue
|
||||
candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue
|
||||
[ -n "$candidate" ] || continue
|
||||
if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then
|
||||
merge_base="$candidate"
|
||||
fi
|
||||
done
|
||||
|
||||
[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases"
|
||||
|
||||
changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed"
|
||||
[ -n "$changed" ] || run_full "no files changed vs $merge_base"
|
||||
|
||||
echo "path-filter[$category]: changed files vs ${merge_base}:"
|
||||
printf '%s\n' "$changed" | sed 's/^/ /' || true
|
||||
|
||||
decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed"
|
||||
|
||||
if [ "$decision" = run ]; then
|
||||
run_full "$category-relevant changes detected"
|
||||
fi
|
||||
|
||||
echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful"
|
||||
circleci-agent step halt
|
||||
5
.github/pull_request_template.md
vendored
5
.github/pull_request_template.md
vendored
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
## Linear ticket
|
||||
|
||||
<!-- if you are an internal contributor (e.g., your username is postfixed with -berri or -berriai), add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
|
||||
<!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
|
||||
|
||||
## Pre-Submission checklist
|
||||
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
- [ ] I have added meaningful tests
|
||||
- [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests)
|
||||
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
|
||||
- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
|
||||
- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
|
||||
|
||||
## Delays in PR merge?
|
||||
|
||||
|
|
@ -24,6 +24,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
|
|||
<!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
|
||||
The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
|
||||
For bug fixes: show reproduction before the fix and passing behavior after
|
||||
Include the commit hash each proof was captured at, for both the before and the after runs
|
||||
For new features: show the feature working end-to-end
|
||||
For UI changes: include before/after screenshots -->
|
||||
|
||||
|
|
|
|||
4
.github/workflows/codspeed.yml
vendored
4
.github/workflows/codspeed.yml
vendored
|
|
@ -4,9 +4,11 @@ on:
|
|||
push:
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
# Allow CodSpeed to trigger backtest performance analysis
|
||||
# in order to generate initial data
|
||||
workflow_dispatch:
|
||||
|
|
@ -22,7 +24,7 @@ concurrency:
|
|||
jobs:
|
||||
benchmarks:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
|
|||
32
.github/workflows/create-release.yml
vendored
32
.github/workflows/create-release.yml
vendored
|
|
@ -122,10 +122,28 @@ jobs:
|
|||
makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
|
||||
}
|
||||
|
||||
try {
|
||||
await github.rest.git.createRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: `refs/tags/${tag}`,
|
||||
sha: commitHash,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error.status !== 422) throw error;
|
||||
const existing = await github.rest.git.getRef({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
ref: `tags/${tag}`,
|
||||
});
|
||||
if (existing.data.object.sha !== commitHash) {
|
||||
throw new Error(`Tag ${tag} already exists at ${existing.data.object.sha}, expected ${commitHash}`);
|
||||
}
|
||||
}
|
||||
|
||||
const response = await github.rest.repos.createRelease({
|
||||
draft: true,
|
||||
generate_release_notes: true,
|
||||
target_commitish: commitHash,
|
||||
name: tag,
|
||||
owner: context.repo.owner,
|
||||
prerelease: isPrerelease,
|
||||
|
|
@ -138,11 +156,21 @@ jobs:
|
|||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
release_id: response.data.id,
|
||||
tag_name: tag,
|
||||
body: updatedBody,
|
||||
draft: false,
|
||||
make_latest: makeLatest,
|
||||
});
|
||||
|
||||
if (!isPrerelease) {
|
||||
await github.rest.repos.updateRelease({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
release_id: response.data.id,
|
||||
tag_name: tag,
|
||||
make_latest: makeLatest,
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
core.setFailed(error.message);
|
||||
}
|
||||
|
|
|
|||
4
.github/workflows/helm_unit_test.yml
vendored
4
.github/workflows/helm_unit_test.yml
vendored
|
|
@ -38,4 +38,6 @@ jobs:
|
|||
echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
|
||||
|
||||
- name: Run unit tests
|
||||
run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
||||
run: |
|
||||
helm unittest -f 'tests/*.yaml' helm/litellm-helm
|
||||
helm unittest -f 'tests/*.yaml' helm/litellm
|
||||
|
|
|
|||
2
.github/workflows/test-linting.yml
vendored
2
.github/workflows/test-linting.yml
vendored
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
git diff --name-only "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
|
||||
git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
|
||||
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
|
||||
echo "No changed litellm Python files to check with ruff format."
|
||||
exit 0
|
||||
|
|
|
|||
113
.github/workflows/test-terraform-provider.yml
vendored
Normal file
113
.github/workflows/test-terraform-provider.yml
vendored
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
name: Terraform Provider
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "terraform/provider/**"
|
||||
- ".github/workflows/test-terraform-provider.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- litellm_internal_staging
|
||||
- litellm_oss_staging
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- "terraform/provider/**"
|
||||
- "litellm/proxy/**"
|
||||
- ".github/workflows/test-terraform-provider.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
provider-checks:
|
||||
name: gofmt, vet, build, test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
defaults:
|
||||
run:
|
||||
working-directory: terraform/provider
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
|
||||
with:
|
||||
go-version-file: terraform/provider/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: terraform/provider/go.sum
|
||||
|
||||
- name: gofmt
|
||||
run: |
|
||||
UNFORMATTED=$(gofmt -l .)
|
||||
if [ -n "${UNFORMATTED}" ]; then
|
||||
echo "::error::gofmt required for: ${UNFORMATTED}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: Build
|
||||
run: go build ./...
|
||||
|
||||
- name: Test
|
||||
run: go test -timeout 120s ./...
|
||||
|
||||
endpoint-drift:
|
||||
name: Provider endpoints vs proxy OpenAPI schema
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
||||
|
||||
- name: Generate Prisma client
|
||||
env:
|
||||
PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
|
||||
run: |
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Generate proxy OpenAPI schema
|
||||
run: |
|
||||
uv run --no-sync python terraform/provider/tools/dump_openapi.py "${RUNNER_TEMP}/openapi.json"
|
||||
|
||||
- uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
|
||||
with:
|
||||
go-version-file: terraform/provider/go.mod
|
||||
cache: true
|
||||
cache-dependency-path: terraform/provider/go.sum
|
||||
|
||||
- name: Audit provider endpoints against the schema
|
||||
working-directory: terraform/provider
|
||||
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -52,9 +52,8 @@ ui/litellm-dashboard/node_modules
|
|||
ui/litellm-dashboard/next-env.d.ts
|
||||
ui/litellm-dashboard/package.json
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
deploy/charts/litellm/*.tgz
|
||||
deploy/charts/litellm/charts/*
|
||||
deploy/charts/*.tgz
|
||||
helm/litellm-helm/*.tgz
|
||||
helm/*.tgz
|
||||
litellm/proxy/vertex_key.json
|
||||
**/.vim/
|
||||
**/node_modules
|
||||
|
|
@ -130,3 +129,5 @@ crash.*.log
|
|||
|
||||
# pytest coverage data
|
||||
.coverage
|
||||
|
||||
ui/litellm-dashboard/out/
|
||||
|
|
|
|||
12
CLAUDE.md
12
CLAUDE.md
|
|
@ -17,11 +17,15 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
|
|||
|
||||
`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`
|
||||
|
||||
When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
|
||||
|
||||
Always use @.github/pull_request_template.md as a guide for your PR body
|
||||
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout
|
||||
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
|
||||
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
|
||||
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
|
||||
|
||||
If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
|
|
@ -43,9 +47,11 @@ If you're trying to create a new function that relies on untyped stuff, instead
|
|||
|
||||
If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
|
||||
|
||||
Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
|
||||
|
||||
Commit and push your work when you're done without asking
|
||||
|
||||
When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
|
||||
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
|
||||
|
||||
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
|
||||
|
|
|
|||
35
Makefile
35
Makefile
|
|
@ -4,7 +4,7 @@
|
|||
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
|
||||
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
|
||||
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
|
||||
info lint lint-dev format \
|
||||
info lint lint-dev lint-checks format \
|
||||
lint-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
|
||||
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
|
||||
install-dev install-proxy-dev install-test-deps install-hooks \
|
||||
|
|
@ -53,6 +53,11 @@ help:
|
|||
UV := uv
|
||||
UV_RUN := $(UV) run --no-sync
|
||||
|
||||
LINT_DEP_INSTALL ?= install-dev
|
||||
LINT_DEP_BASE ?= lint-fetch-base
|
||||
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
|
||||
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)
|
||||
|
||||
# Show info
|
||||
info:
|
||||
@echo "UV: $(UV)"
|
||||
|
|
@ -107,12 +112,12 @@ lint-fetch-base:
|
|||
# running proxy need.
|
||||
lint-install:
|
||||
$(UV) sync --inexact --frozen --group proxy-dev
|
||||
$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
|
||||
$(UV_RUN) python scripts/prisma_generate_if_needed.py
|
||||
|
||||
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
|
||||
# only the litellm Python files changed vs the base are checked, so a pre-existing
|
||||
# format issue elsewhere doesn't block an unrelated commit.
|
||||
lint-format-check-changed: install-dev lint-fetch-base
|
||||
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
@files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \
|
||||
if [ -z "$$files" ]; then \
|
||||
echo "No changed litellm Python files to format-check."; \
|
||||
|
|
@ -121,7 +126,7 @@ lint-format-check-changed: install-dev lint-fetch-base
|
|||
fi
|
||||
|
||||
# Linting targets
|
||||
lint-ruff: install-dev
|
||||
lint-ruff: $(LINT_DEP_INSTALL)
|
||||
cd litellm && $(UV_RUN) ruff check . && cd ..
|
||||
|
||||
# faster linter for developing ...
|
||||
|
|
@ -156,12 +161,12 @@ lint-ruff-FULL-dev: install-dev
|
|||
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
|
||||
else echo "No changed .py files to check."; fi
|
||||
|
||||
lint-basedpyright: install-dev lint-fetch-base
|
||||
lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --base origin/litellm_internal_staging
|
||||
|
||||
# Type-discipline budget (mutable collections / casts / type guards / kwargs /
|
||||
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
|
||||
lint-type-discipline: install-dev lint-fetch-base
|
||||
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
|
||||
|
||||
# --update lowers each limit by what this branch fixed since its branch point, so
|
||||
|
|
@ -176,7 +181,7 @@ lint-ruff-budget: install-dev
|
|||
|
||||
# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
|
||||
# means the CI check will pass too.
|
||||
lint-gate: install-dev lint-fetch-base
|
||||
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
|
||||
$(UV_RUN) python scripts/ruff_strict_gate.py --base origin/litellm_internal_staging
|
||||
|
||||
lint-ruff-budget-update: install-dev lint-fetch-base
|
||||
|
|
@ -188,10 +193,10 @@ lint-type-discipline-budget-update: install-dev lint-fetch-base
|
|||
# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
|
||||
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
|
||||
|
||||
check-circular-imports: install-dev
|
||||
check-circular-imports: $(LINT_DEP_INSTALL)
|
||||
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
|
||||
|
||||
check-import-safety: install-dev
|
||||
check-import-safety: $(LINT_DEP_INSTALL)
|
||||
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
||||
|
||||
# Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a
|
||||
|
|
@ -199,9 +204,13 @@ check-import-safety: install-dev
|
|||
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
|
||||
# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import
|
||||
# and import-safety checks. Steps that compare against the base resolve it the same way CI
|
||||
# does (merge-base with origin/litellm_internal_staging). lint-install is first so the
|
||||
# Prisma client exists before basedpyright runs.
|
||||
lint: lint-install lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
|
||||
# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
|
||||
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
|
||||
# fans them out with -j and the fast ones finish under basedpyright's shadow.
|
||||
lint: lint-install lint-fetch-base
|
||||
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_DEP_BASE= lint-checks
|
||||
|
||||
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright check-circular-imports check-import-safety
|
||||
|
||||
# Faster linting for local development (only checks changed code)
|
||||
lint-dev: lint-format-changed check-circular-imports check-import-safety
|
||||
|
|
@ -256,7 +265,7 @@ test-integration: install-test-deps
|
|||
$(UV_RUN) pytest tests/ -k "not test_litellm"
|
||||
|
||||
test-unit-helm: install-helm-unittest
|
||||
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
||||
helm unittest -f 'tests/*.yaml' helm/litellm-helm
|
||||
|
||||
# LLM Translation testing targets
|
||||
test-llm-translation: install-test-deps
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
|
|
@ -3,16 +3,16 @@
|
|||
"limit": 37484
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2721
|
||||
"limit": 2704
|
||||
},
|
||||
"reportAssignmentType": {
|
||||
"limit": 330
|
||||
},
|
||||
"reportAttributeAccessIssue": {
|
||||
"limit": 519
|
||||
"limit": 516
|
||||
},
|
||||
"reportCallIssue": {
|
||||
"limit": 131
|
||||
"limit": 124
|
||||
},
|
||||
"reportConstantRedefinition": {
|
||||
"limit": 59
|
||||
|
|
@ -42,7 +42,7 @@
|
|||
"limit": 18
|
||||
},
|
||||
"reportIndexIssue": {
|
||||
"limit": 39
|
||||
"limit": 37
|
||||
},
|
||||
"reportInvalidTypeForm": {
|
||||
"limit": 35
|
||||
|
|
@ -51,7 +51,7 @@
|
|||
"limit": 5
|
||||
},
|
||||
"reportMatchNotExhaustive": {
|
||||
"limit": 2
|
||||
"limit": 0
|
||||
},
|
||||
"reportMissingParameterType": {
|
||||
"limit": 5900
|
||||
|
|
@ -63,25 +63,25 @@
|
|||
"limit": 41
|
||||
},
|
||||
"reportOperatorIssue": {
|
||||
"limit": 9
|
||||
"limit": 0
|
||||
},
|
||||
"reportOptionalCall": {
|
||||
"limit": 7
|
||||
"limit": 0
|
||||
},
|
||||
"reportOptionalIterable": {
|
||||
"limit": 6
|
||||
"limit": 0
|
||||
},
|
||||
"reportOptionalMemberAccess": {
|
||||
"limit": 1086
|
||||
"limit": 1085
|
||||
},
|
||||
"reportOptionalOperand": {
|
||||
"limit": 6
|
||||
"limit": 0
|
||||
},
|
||||
"reportOptionalSubscript": {
|
||||
"limit": 17
|
||||
"limit": 0
|
||||
},
|
||||
"reportPossiblyUnboundVariable": {
|
||||
"limit": 78
|
||||
"limit": 77
|
||||
},
|
||||
"reportPrivateUsage": {
|
||||
"limit": 2438
|
||||
|
|
@ -90,28 +90,28 @@
|
|||
"limit": 12
|
||||
},
|
||||
"reportReturnType": {
|
||||
"limit": 226
|
||||
"limit": 225
|
||||
},
|
||||
"reportTypedDictNotRequiredAccess": {
|
||||
"limit": 30
|
||||
"limit": 27
|
||||
},
|
||||
"reportUndefinedVariable": {
|
||||
"limit": 5
|
||||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 45905
|
||||
"limit": 45894
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 113
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 40556
|
||||
"limit": 40541
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 20418
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 32168
|
||||
"limit": 32151
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 177
|
||||
|
|
@ -141,6 +141,6 @@
|
|||
"limit": 1005
|
||||
},
|
||||
"reportUnusedVariable": {
|
||||
"limit": 1298
|
||||
"limit": 1297
|
||||
}
|
||||
}
|
||||
|
|
|
|||
10
codecov.yaml
10
codecov.yaml
|
|
@ -15,6 +15,16 @@ ignore:
|
|||
flag_management:
|
||||
default_rules:
|
||||
carryforward: true
|
||||
# Dead flags no CI job uploads anymore: their carried-forward sessions were
|
||||
# measured against old revisions, and the stale line maps mark comment lines
|
||||
# of since-edited files as missed, sinking patch coverage on unrelated PRs.
|
||||
individual_flags:
|
||||
- name: proxy-mgmt-behavior
|
||||
carryforward: false
|
||||
- name: security
|
||||
carryforward: false
|
||||
- name: proxy-db-schema-migration
|
||||
carryforward: false
|
||||
|
||||
component_management:
|
||||
individual_components:
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1,15 +0,0 @@
|
|||
{
|
||||
"$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#",
|
||||
"handler": "Microsoft.Azure.CreateUIDef",
|
||||
"version": "0.1.2-preview",
|
||||
"parameters": {
|
||||
"config": {
|
||||
"isWizard": false,
|
||||
"basics": { }
|
||||
},
|
||||
"basics": [ ],
|
||||
"steps": [ ],
|
||||
"outputs": { },
|
||||
"resourceTypes": [ ]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
{
|
||||
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"imageName": {
|
||||
"type": "string",
|
||||
"defaultValue": "ghcr.io/berriai/litellm:main-latest"
|
||||
},
|
||||
"containerName": {
|
||||
"type": "string",
|
||||
"defaultValue": "litellm-container"
|
||||
},
|
||||
"dnsLabelName": {
|
||||
"type": "string",
|
||||
"defaultValue": "litellm"
|
||||
},
|
||||
"portNumber": {
|
||||
"type": "int",
|
||||
"defaultValue": 4000
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"type": "Microsoft.ContainerInstance/containerGroups",
|
||||
"apiVersion": "2021-03-01",
|
||||
"name": "[parameters('containerName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"properties": {
|
||||
"containers": [
|
||||
{
|
||||
"name": "[parameters('containerName')]",
|
||||
"properties": {
|
||||
"image": "[parameters('imageName')]",
|
||||
"resources": {
|
||||
"requests": {
|
||||
"cpu": 1,
|
||||
"memoryInGB": 2
|
||||
}
|
||||
},
|
||||
"ports": [
|
||||
{
|
||||
"port": "[parameters('portNumber')]"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"osType": "Linux",
|
||||
"restartPolicy": "Always",
|
||||
"ipAddress": {
|
||||
"type": "Public",
|
||||
"ports": [
|
||||
{
|
||||
"protocol": "tcp",
|
||||
"port": "[parameters('portNumber')]"
|
||||
}
|
||||
],
|
||||
"dnsNameLabel": "[parameters('dnsLabelName')]"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
param imageName string = 'ghcr.io/berriai/litellm:main-latest'
|
||||
param containerName string = 'litellm-container'
|
||||
param dnsLabelName string = 'litellm'
|
||||
param portNumber int = 4000
|
||||
|
||||
resource containerGroupName 'Microsoft.ContainerInstance/containerGroups@2021-03-01' = {
|
||||
name: containerName
|
||||
location: resourceGroup().location
|
||||
properties: {
|
||||
containers: [
|
||||
{
|
||||
name: containerName
|
||||
properties: {
|
||||
image: imageName
|
||||
resources: {
|
||||
requests: {
|
||||
cpu: 1
|
||||
memoryInGB: 2
|
||||
}
|
||||
}
|
||||
ports: [
|
||||
{
|
||||
port: portNumber
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
osType: 'Linux'
|
||||
restartPolicy: 'Always'
|
||||
ipAddress: {
|
||||
type: 'Public'
|
||||
ports: [
|
||||
{
|
||||
protocol: 'tcp'
|
||||
port: portNumber
|
||||
}
|
||||
]
|
||||
dnsNameLabel: dnsLabelName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG PROXY_EXTRAS_SOURCE=published
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
|
|
|
|||
|
|
@ -57,8 +57,6 @@ source ~/.nvm/nvm.sh
|
|||
nvm install v18.17.0
|
||||
nvm use v18.17.0
|
||||
|
||||
# copy _enterprise.json from this directory to /ui/litellm-dashboard, and rename it to ui_colors.json
|
||||
cp enterprise/enterprise_ui/enterprise_colors.json ui/litellm-dashboard/ui_colors.json
|
||||
|
||||
# cd in to /ui/litellm-dashboard
|
||||
cd ui/litellm-dashboard
|
||||
|
|
|
|||
|
|
@ -477,9 +477,12 @@ class BaseEmailLogger(CustomLogger):
|
|||
_id = user_info.token or user_info.user_id or "default_id"
|
||||
_cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}"
|
||||
|
||||
# Check if we've already sent this alert
|
||||
result = await _cache.async_get_cache(key=_cache_key)
|
||||
if result is None:
|
||||
send_count = await _cache.async_increment_cache(
|
||||
key=_cache_key,
|
||||
value=1,
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
if send_count is None or send_count <= 1:
|
||||
# Create WebhookEvent for soft budget alert
|
||||
event_message = f"Soft Budget Crossed - Total Soft Budget: ${user_info.soft_budget}"
|
||||
webhook_event = WebhookEvent(
|
||||
|
|
@ -508,18 +511,12 @@ class BaseEmailLogger(CustomLogger):
|
|||
await self.send_team_soft_budget_alert_email(webhook_event)
|
||||
else:
|
||||
await self.send_soft_budget_alert_email(webhook_event)
|
||||
|
||||
# Cache the alert to prevent duplicate sends
|
||||
await _cache.async_set_cache(
|
||||
key=_cache_key,
|
||||
value="SENT",
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"Error sending soft budget alert email: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
await self._release_budget_alert_claim(_cache, _cache_key)
|
||||
return
|
||||
|
||||
# For max_budget_alert, check if we've already sent an alert
|
||||
|
|
@ -545,9 +542,12 @@ class BaseEmailLogger(CustomLogger):
|
|||
_id = user_info.token or user_info.user_id or "default_id"
|
||||
_cache_key = f"email_budget_alerts:max_budget_alert:{_id}"
|
||||
|
||||
# Check if we've already sent this alert
|
||||
result = await _cache.async_get_cache(key=_cache_key)
|
||||
if result is None:
|
||||
send_count = await _cache.async_increment_cache(
|
||||
key=_cache_key,
|
||||
value=1,
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
if send_count is None or send_count <= 1:
|
||||
# Calculate percentage
|
||||
percentage = int(
|
||||
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100
|
||||
|
|
@ -576,18 +576,12 @@ class BaseEmailLogger(CustomLogger):
|
|||
|
||||
try:
|
||||
await self.send_max_budget_alert_email(webhook_event)
|
||||
|
||||
# Cache the alert to prevent duplicate sends
|
||||
await _cache.async_set_cache(
|
||||
key=_cache_key,
|
||||
value="SENT",
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"Error sending max budget alert email: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
await self._release_budget_alert_claim(_cache, _cache_key)
|
||||
return
|
||||
|
||||
async def _handle_multi_threshold_max_budget_alert(
|
||||
|
|
@ -617,10 +611,6 @@ class BaseEmailLogger(CustomLogger):
|
|||
f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
|
||||
)
|
||||
|
||||
result = await _cache.async_get_cache(key=_cache_key)
|
||||
if result is not None:
|
||||
continue
|
||||
|
||||
# Parse emails + auto-include owner
|
||||
emails = _parse_email_list(raw_emails)
|
||||
if user_info.user_email:
|
||||
|
|
@ -634,6 +624,14 @@ class BaseEmailLogger(CustomLogger):
|
|||
continue
|
||||
recipient_emails = list(set(emails))
|
||||
|
||||
send_count = await _cache.async_increment_cache(
|
||||
key=_cache_key,
|
||||
value=1,
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
if send_count is not None and send_count > 1:
|
||||
continue
|
||||
|
||||
event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
|
||||
webhook_event = WebhookEvent(
|
||||
event="max_budget_alert",
|
||||
|
|
@ -660,16 +658,21 @@ class BaseEmailLogger(CustomLogger):
|
|||
threshold_pct=threshold_pct,
|
||||
recipient_emails=recipient_emails,
|
||||
)
|
||||
await _cache.async_set_cache(
|
||||
key=_cache_key,
|
||||
value="SENT",
|
||||
ttl=EMAIL_BUDGET_ALERT_TTL,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(
|
||||
f"Error sending multi-threshold max budget alert email for {threshold_pct}%: {e}",
|
||||
exc_info=True,
|
||||
)
|
||||
await self._release_budget_alert_claim(_cache, _cache_key)
|
||||
|
||||
async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None:
|
||||
try:
|
||||
await cache.async_delete_cache(key=cache_key)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"Failed to release budget alert claim for %s; it expires with the TTL",
|
||||
cache_key,
|
||||
)
|
||||
|
||||
async def _get_email_params(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy._types import LiteLLM_ManagedObjectTable
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
|
||||
|
|
@ -277,13 +278,20 @@ class CheckBatchCost:
|
|||
except Exception:
|
||||
return None
|
||||
|
||||
async def check_batch_cost(self):
|
||||
async def _track_completed_batch_cost(
|
||||
self,
|
||||
job: "LiteLLM_ManagedObjectTable",
|
||||
response: "LiteLLMBatch",
|
||||
model_id: str,
|
||||
batch_id: str,
|
||||
prom_logger: Optional["PrometheusLogger"],
|
||||
) -> Optional[Tuple[Optional[str], Optional[str]]]:
|
||||
"""
|
||||
Check if the batch JOB has been tracked.
|
||||
- get all status="validating" and file_purpose="batch" jobs
|
||||
- check if batch is now complete
|
||||
- if not, return False
|
||||
- if so, return True
|
||||
Fetch a completed batch's results, compute cost/usage, and emit the
|
||||
aretrieve_batch spend log. Returns (model_name, llm_provider) on
|
||||
success, None when the job can't be routed to a deployment. Raises on
|
||||
results-fetch or cost-computation failures so the caller can leave the
|
||||
job unprocessed and retry it on a later poll.
|
||||
"""
|
||||
from litellm.batches.batch_utils import (
|
||||
_get_file_content_as_dictionary,
|
||||
|
|
@ -296,6 +304,184 @@ class CheckBatchCost:
|
|||
_is_base64_encoded_unified_file_id,
|
||||
)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
f"Batch ID: {batch_id} is complete, tracking cost and usage"
|
||||
)
|
||||
|
||||
# aretrieve_batch is called with the raw provider batch ID, so response.id
|
||||
# is the raw provider value (e.g. "batch_20260223-0518.234"). We need the
|
||||
# unified base64 ID in the S3 log so downstream consumers can correlate it
|
||||
# back to the batch they submitted via the proxy.
|
||||
#
|
||||
# CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and
|
||||
# calls async_success_handler(result=response) directly. That handler calls
|
||||
# _build_standard_logging_payload(response, ...) which reads response.id at
|
||||
# that point — so setting response.id here is sufficient.
|
||||
#
|
||||
# The HTTP endpoint does this substitution via the managed files hook
|
||||
# (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely,
|
||||
# so we do it explicitly here.
|
||||
response.id = job.unified_object_id
|
||||
|
||||
# This background job runs as default_user_id, so going through the HTTP endpoint
|
||||
# would trigger check_managed_file_id_access and get 403. Instead, extract the raw
|
||||
# provider file ID and call afile_content directly with deployment credentials.
|
||||
raw_output_file_id = response.output_file_id
|
||||
decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
|
||||
if decoded:
|
||||
try:
|
||||
raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
|
||||
except (IndexError, AttributeError):
|
||||
pass
|
||||
|
||||
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=raw_output_file_id,
|
||||
**credentials,
|
||||
)
|
||||
|
||||
# Access content - handle both direct attribute and method call
|
||||
if hasattr(_file_content, 'content'):
|
||||
content_bytes = _file_content.content # type: ignore[union-attr]
|
||||
elif hasattr(_file_content, 'read'):
|
||||
content_bytes = await _file_content.read() # type: ignore[misc]
|
||||
else:
|
||||
content_bytes = _file_content # type: ignore[assignment]
|
||||
|
||||
file_content_as_dict = _get_file_content_as_dictionary(
|
||||
content_bytes # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
# Record output file size
|
||||
if prom_logger and content_bytes:
|
||||
try:
|
||||
prom_logger.record_managed_file_size(
|
||||
size_bytes=len(content_bytes), # type: ignore
|
||||
purpose="batch",
|
||||
file_type="output",
|
||||
model=model_id,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
deployment_info = self.llm_router.get_deployment(model_id=model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.info(
|
||||
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
|
||||
)
|
||||
self._record_error(prom_logger, "deployment_not_found")
|
||||
return None
|
||||
custom_llm_provider = deployment_info.litellm_params.custom_llm_provider
|
||||
litellm_model_name = deployment_info.litellm_params.model
|
||||
|
||||
model_name, llm_provider, _, _ = get_llm_provider(
|
||||
model=litellm_model_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
|
||||
# output/error file IDs to managed base64 IDs before the DB write here.
|
||||
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
|
||||
if managed_files_hook is not None:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
_minimal_auth = UserAPIKeyAuth(
|
||||
user_id=job.created_by or "default-user-id",
|
||||
team_id=getattr(job, "team_id", None),
|
||||
)
|
||||
for _file_attr in ["output_file_id", "error_file_id"]:
|
||||
_raw_file_id = getattr(response, _file_attr, None)
|
||||
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
|
||||
try:
|
||||
_unified_file_id = managed_files_hook.get_unified_output_file_id(
|
||||
output_file_id=_raw_file_id,
|
||||
model_id=model_id,
|
||||
model_name=str(model_name) if model_name else deployment_info.model_name or None,
|
||||
)
|
||||
await managed_files_hook.store_unified_file_id(
|
||||
file_id=_unified_file_id,
|
||||
file_object=None,
|
||||
litellm_parent_otel_span=None,
|
||||
model_mappings={model_id: _raw_file_id},
|
||||
user_api_key_dict=_minimal_auth,
|
||||
)
|
||||
setattr(response, _file_attr, _unified_file_id)
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: converted {_file_attr} "
|
||||
f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
|
||||
)
|
||||
except Exception as _e:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: failed to create managed file ID for "
|
||||
f"{_file_attr}={_raw_file_id!r}: {_e}"
|
||||
)
|
||||
|
||||
# Pass deployment model_info so custom batch pricing
|
||||
# (input_cost_per_token_batches etc.) is used for cost calc
|
||||
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
|
||||
batch_cost, batch_usage, batch_models = (
|
||||
await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=file_content_as_dict,
|
||||
custom_llm_provider=llm_provider, # type: ignore
|
||||
model_name=model_name,
|
||||
model_info=deployment_model_info, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
logging_obj = LiteLLMLogging(
|
||||
model=batch_models[0],
|
||||
messages=[{"role": "user", "content": "<retrieve_batch>"}],
|
||||
stream=False,
|
||||
call_type="aretrieve_batch",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
function_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
creator_user_id = job.created_by
|
||||
user_info = await self._get_user_info(batch_id, job.created_by)
|
||||
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={
|
||||
# set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
|
||||
"proxy_server_request": {
|
||||
"headers": {
|
||||
"user-agent": CHECK_BATCH_COST_USER_AGENT,
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"user_api_key_user_id": creator_user_id,
|
||||
**user_info,
|
||||
},
|
||||
},
|
||||
optional_params={},
|
||||
)
|
||||
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_cost,
|
||||
batch_usage=batch_usage,
|
||||
batch_models=batch_models,
|
||||
)
|
||||
|
||||
# Record batch duration (completed_at - created_at)
|
||||
if prom_logger and response.completed_at and response.created_at:
|
||||
duration_seconds = float(response.completed_at - response.created_at)
|
||||
if duration_seconds >= 0:
|
||||
prom_logger.record_managed_batch_duration(
|
||||
duration_seconds=duration_seconds,
|
||||
model=model_name,
|
||||
api_provider=str(llm_provider) if llm_provider else None,
|
||||
)
|
||||
|
||||
return model_name, str(llm_provider) if llm_provider else None
|
||||
|
||||
async def check_batch_cost(self):
|
||||
"""
|
||||
Check if the batch JOB has been tracked.
|
||||
- get all status="validating" and file_purpose="batch" jobs
|
||||
- check if batch is now complete
|
||||
- if not, return False
|
||||
- if so, return True
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
prom_logger = PrometheusLogger.get_instance()
|
||||
|
|
@ -381,177 +567,26 @@ class CheckBatchCost:
|
|||
response.status == "completed"
|
||||
and response.output_file_id is not None
|
||||
):
|
||||
verbose_proxy_logger.info(
|
||||
f"Batch ID: {batch_id} is complete, tracking cost and usage"
|
||||
)
|
||||
|
||||
# aretrieve_batch is called with the raw provider batch ID, so response.id
|
||||
# is the raw provider value (e.g. "batch_20260223-0518.234"). We need the
|
||||
# unified base64 ID in the S3 log so downstream consumers can correlate it
|
||||
# back to the batch they submitted via the proxy.
|
||||
#
|
||||
# CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and
|
||||
# calls async_success_handler(result=response) directly. That handler calls
|
||||
# _build_standard_logging_payload(response, ...) which reads response.id at
|
||||
# that point — so setting response.id here is sufficient.
|
||||
#
|
||||
# The HTTP endpoint does this substitution via the managed files hook
|
||||
# (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely,
|
||||
# so we do it explicitly here.
|
||||
response.id = job.unified_object_id
|
||||
|
||||
# This background job runs as default_user_id, so going through the HTTP endpoint
|
||||
# would trigger check_managed_file_id_access and get 403. Instead, extract the raw
|
||||
# provider file ID and call afile_content directly with deployment credentials.
|
||||
raw_output_file_id = response.output_file_id
|
||||
decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
|
||||
if decoded:
|
||||
try:
|
||||
raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
|
||||
except (IndexError, AttributeError):
|
||||
pass
|
||||
|
||||
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
|
||||
_file_content = await afile_content(
|
||||
file_id=raw_output_file_id,
|
||||
**credentials,
|
||||
)
|
||||
|
||||
# Access content - handle both direct attribute and method call
|
||||
if hasattr(_file_content, 'content'):
|
||||
content_bytes = _file_content.content # type: ignore[union-attr]
|
||||
elif hasattr(_file_content, 'read'):
|
||||
content_bytes = await _file_content.read() # type: ignore[misc]
|
||||
else:
|
||||
content_bytes = _file_content # type: ignore[assignment]
|
||||
|
||||
file_content_as_dict = _get_file_content_as_dictionary(
|
||||
content_bytes # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
# Record output file size
|
||||
if prom_logger and content_bytes:
|
||||
try:
|
||||
prom_logger.record_managed_file_size(
|
||||
size_bytes=len(content_bytes), # type: ignore
|
||||
purpose="batch",
|
||||
file_type="output",
|
||||
model=model_id,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
deployment_info = self.llm_router.get_deployment(model_id=model_id)
|
||||
if deployment_info is None:
|
||||
verbose_proxy_logger.info(
|
||||
f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
|
||||
try:
|
||||
tracked = await self._track_completed_batch_cost(
|
||||
job=job,
|
||||
response=response,
|
||||
model_id=model_id,
|
||||
batch_id=batch_id,
|
||||
prom_logger=prom_logger,
|
||||
)
|
||||
if prom_logger:
|
||||
prom_logger.record_check_batch_cost_error("deployment_not_found")
|
||||
except Exception as tracking_err:
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: failed to track cost for batch {batch_id} "
|
||||
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
|
||||
)
|
||||
self._record_error(prom_logger, "cost_tracking_error")
|
||||
continue
|
||||
if tracked is None:
|
||||
continue
|
||||
custom_llm_provider = deployment_info.litellm_params.custom_llm_provider
|
||||
litellm_model_name = deployment_info.litellm_params.model
|
||||
|
||||
model_name, llm_provider, _, _ = get_llm_provider(
|
||||
model=litellm_model_name,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
# CheckBatchCost bypasses async_post_call_success_hook, so convert raw
|
||||
# output/error file IDs to managed base64 IDs before the DB write here.
|
||||
managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
|
||||
if managed_files_hook is not None:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
_minimal_auth = UserAPIKeyAuth(
|
||||
user_id=job.created_by or "default-user-id",
|
||||
team_id=getattr(job, "team_id", None),
|
||||
)
|
||||
for _file_attr in ["output_file_id", "error_file_id"]:
|
||||
_raw_file_id = getattr(response, _file_attr, None)
|
||||
if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
|
||||
try:
|
||||
_unified_file_id = managed_files_hook.get_unified_output_file_id(
|
||||
output_file_id=_raw_file_id,
|
||||
model_id=model_id,
|
||||
model_name=str(model_name) if model_name else deployment_info.model_name or None,
|
||||
)
|
||||
await managed_files_hook.store_unified_file_id(
|
||||
file_id=_unified_file_id,
|
||||
file_object=None,
|
||||
litellm_parent_otel_span=None,
|
||||
model_mappings={model_id: _raw_file_id},
|
||||
user_api_key_dict=_minimal_auth,
|
||||
)
|
||||
setattr(response, _file_attr, _unified_file_id)
|
||||
verbose_proxy_logger.info(
|
||||
f"CheckBatchCost: converted {_file_attr} "
|
||||
f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
|
||||
)
|
||||
except Exception as _e:
|
||||
verbose_proxy_logger.warning(
|
||||
f"CheckBatchCost: failed to create managed file ID for "
|
||||
f"{_file_attr}={_raw_file_id!r}: {_e}"
|
||||
)
|
||||
|
||||
# Pass deployment model_info so custom batch pricing
|
||||
# (input_cost_per_token_batches etc.) is used for cost calc
|
||||
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
|
||||
batch_cost, batch_usage, batch_models = (
|
||||
await calculate_batch_cost_and_usage(
|
||||
file_content_dictionary=file_content_as_dict,
|
||||
custom_llm_provider=llm_provider, # type: ignore
|
||||
model_name=model_name,
|
||||
model_info=deployment_model_info, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
logging_obj = LiteLLMLogging(
|
||||
model=batch_models[0],
|
||||
messages=[{"role": "user", "content": "<retrieve_batch>"}],
|
||||
stream=False,
|
||||
call_type="aretrieve_batch",
|
||||
start_time=datetime.now(),
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
function_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
creator_user_id = job.created_by
|
||||
user_info = await self._get_user_info(batch_id, job.created_by)
|
||||
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={
|
||||
# set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
|
||||
"proxy_server_request": {
|
||||
"headers": {
|
||||
"user-agent": CHECK_BATCH_COST_USER_AGENT,
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"user_api_key_user_id": creator_user_id,
|
||||
**user_info,
|
||||
},
|
||||
},
|
||||
optional_params={},
|
||||
)
|
||||
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_cost,
|
||||
batch_usage=batch_usage,
|
||||
batch_models=batch_models,
|
||||
)
|
||||
|
||||
# Record batch duration (completed_at - created_at)
|
||||
if prom_logger and response.completed_at and response.created_at:
|
||||
duration_seconds = float(response.completed_at - response.created_at)
|
||||
if duration_seconds >= 0:
|
||||
prom_logger.record_managed_batch_duration(
|
||||
duration_seconds=duration_seconds,
|
||||
model=model_name,
|
||||
api_provider=str(llm_provider) if llm_provider else None,
|
||||
)
|
||||
|
||||
# Track this job for the final metrics summary
|
||||
processed_models.append((model_name, str(llm_provider) if llm_provider else None))
|
||||
processed_models.append(tracked)
|
||||
|
||||
# mark the job as complete
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.45"
|
||||
version = "0.1.48"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.45"
|
||||
version = "0.1.48"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
|
|
@ -25,17 +25,25 @@ DatabaseURLSettings.from_env().apply_to_env()
|
|||
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
|
||||
from gateway.routes.allowlist import (
|
||||
GATEWAY_EXACT_PATHS,
|
||||
GATEWAY_MOUNT_PATHS,
|
||||
GATEWAY_PATH_PREFIXES,
|
||||
)
|
||||
|
||||
|
||||
def _is_gateway_route(route) -> bool:
|
||||
"""Keep the route on the gateway if its path is in the LLM data-plane surface."""
|
||||
"""Keep the route on the gateway if its path is in the LLM data-plane surface.
|
||||
|
||||
Prometheus registers /metrics as a Mount (``app.mount("/metrics", make_asgi_app())``),
|
||||
so Mounts are matched against GATEWAY_MOUNT_PATHS instead of being dropped with
|
||||
the UI static mounts.
|
||||
"""
|
||||
path = getattr(route, "path", None)
|
||||
if path is None:
|
||||
return False
|
||||
if isinstance(route, Mount):
|
||||
# Gateway never serves the static UI or its asset bundles.
|
||||
return False
|
||||
return path in GATEWAY_MOUNT_PATHS
|
||||
if path in GATEWAY_EXACT_PATHS:
|
||||
return True
|
||||
return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
# Health & ops
|
||||
"/health",
|
||||
"/metrics",
|
||||
"/watsonx"
|
||||
"/watsonx",
|
||||
)
|
||||
|
||||
GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
||||
|
|
@ -121,3 +121,9 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
|
|||
"/test",
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_MOUNT_PATHS: frozenset[str] = frozenset(
|
||||
{
|
||||
"/metrics",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -45,11 +45,16 @@ spec:
|
|||
value: /app/config/config.yaml
|
||||
{{- end }}
|
||||
{{- include "litellm.envFrom" .Values.backend | nindent 10 }}
|
||||
{{- if .Values.gateway.config.create }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
|
||||
volumeMounts:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
mountPath: /app/config/config.yaml
|
||||
subPath: config.yaml
|
||||
{{- end }}
|
||||
{{- with .Values.backend.volumeMounts }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with .Values.backend.livenessProbe }}
|
||||
livenessProbe:
|
||||
|
|
@ -61,11 +66,16 @@ spec:
|
|||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.backend.resources | nindent 12 }}
|
||||
{{- if .Values.gateway.config.create }}
|
||||
{{- if or .Values.gateway.config.create .Values.backend.volumes }}
|
||||
volumes:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
configMap:
|
||||
name: {{ include "litellm.gateway.fullname" . }}-config
|
||||
{{- end }}
|
||||
{{- with .Values.backend.volumes }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with .Values.backend.nodeSelector }}
|
||||
nodeSelector:
|
||||
|
|
|
|||
|
|
@ -47,11 +47,16 @@ spec:
|
|||
value: {{ .Values.gateway.numWorkers | quote }}
|
||||
{{- end }}
|
||||
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
|
||||
{{- if .Values.gateway.config.create }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }}
|
||||
volumeMounts:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
mountPath: /app/config/config.yaml
|
||||
subPath: config.yaml
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.volumeMounts }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.livenessProbe }}
|
||||
livenessProbe:
|
||||
|
|
@ -63,11 +68,16 @@ spec:
|
|||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.gateway.resources | nindent 12 }}
|
||||
{{- if .Values.gateway.config.create }}
|
||||
{{- if or .Values.gateway.config.create .Values.gateway.volumes }}
|
||||
volumes:
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: gateway-config
|
||||
configMap:
|
||||
name: {{ include "litellm.gateway.fullname" . }}-config
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.volumes }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with .Values.gateway.nodeSelector }}
|
||||
nodeSelector:
|
||||
|
|
|
|||
|
|
@ -46,6 +46,10 @@ spec:
|
|||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- include "litellm.envFrom" .Values.ui | nindent 10 }}
|
||||
{{- with .Values.ui.volumeMounts }}
|
||||
volumeMounts:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.ui.livenessProbe }}
|
||||
livenessProbe:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
|
|
@ -56,6 +60,10 @@ spec:
|
|||
{{- end }}
|
||||
resources:
|
||||
{{- toYaml .Values.ui.resources | nindent 12 }}
|
||||
{{- with .Values.ui.volumes }}
|
||||
volumes:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.ui.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
|||
172
helm/litellm/tests/deployment_volumes_tests.yaml
Normal file
172
helm/litellm/tests/deployment_volumes_tests.yaml
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
suite: test deployment volumes and volumeMounts
|
||||
templates:
|
||||
- gateway/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
- backend/deployment.yaml
|
||||
- ui/deployment.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: gateway renders only the config volume by default
|
||||
template: gateway/deployment.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes
|
||||
value:
|
||||
- name: gateway-config
|
||||
configMap:
|
||||
name: RELEASE-NAME-litellm-gateway-config
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
value:
|
||||
- name: gateway-config
|
||||
mountPath: /app/config/config.yaml
|
||||
subPath: config.yaml
|
||||
|
||||
- it: gateway merges user volumes and volumeMounts with the config volume
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
gateway.volumes:
|
||||
- name: custom-callbacks
|
||||
configMap:
|
||||
name: custom-callbacks
|
||||
gateway.volumeMounts:
|
||||
- name: custom-callbacks
|
||||
mountPath: /app/custom_callbacks.py
|
||||
subPath: custom_callbacks.py
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].name
|
||||
value: gateway-config
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[1]
|
||||
value:
|
||||
name: custom-callbacks
|
||||
configMap:
|
||||
name: custom-callbacks
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts[0].name
|
||||
value: gateway-config
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts[1]
|
||||
value:
|
||||
name: custom-callbacks
|
||||
mountPath: /app/custom_callbacks.py
|
||||
subPath: custom_callbacks.py
|
||||
|
||||
- it: gateway renders user volumes even when config creation is disabled
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
gateway.config.create: false
|
||||
gateway.volumes:
|
||||
- name: certs
|
||||
secret:
|
||||
secretName: tls-certs
|
||||
gateway.volumeMounts:
|
||||
- name: certs
|
||||
mountPath: /etc/certs
|
||||
readOnly: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes
|
||||
value:
|
||||
- name: certs
|
||||
secret:
|
||||
secretName: tls-certs
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
value:
|
||||
- name: certs
|
||||
mountPath: /etc/certs
|
||||
readOnly: true
|
||||
|
||||
- it: gateway omits volumes when config creation is disabled and no user volumes are set
|
||||
template: gateway/deployment.yaml
|
||||
set:
|
||||
gateway.config.create: false
|
||||
asserts:
|
||||
- isNull:
|
||||
path: spec.template.spec.volumes
|
||||
- isNull:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
|
||||
- it: backend merges user volumes and volumeMounts with the shared config volume
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
backend.volumes:
|
||||
- name: sso-handler
|
||||
configMap:
|
||||
name: sso-handler
|
||||
backend.volumeMounts:
|
||||
- name: sso-handler
|
||||
mountPath: /app/custom_sso.py
|
||||
subPath: custom_sso.py
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].name
|
||||
value: gateway-config
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[1]
|
||||
value:
|
||||
name: sso-handler
|
||||
configMap:
|
||||
name: sso-handler
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts[1]
|
||||
value:
|
||||
name: sso-handler
|
||||
mountPath: /app/custom_sso.py
|
||||
subPath: custom_sso.py
|
||||
|
||||
- it: backend renders user volumes even when config creation is disabled
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
gateway.config.create: false
|
||||
backend.volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
backend.volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes
|
||||
value:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
value:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
|
||||
- it: ui renders no volumes by default
|
||||
template: ui/deployment.yaml
|
||||
asserts:
|
||||
- isNull:
|
||||
path: spec.template.spec.volumes
|
||||
- isNull:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
|
||||
- it: ui renders user volumes and volumeMounts
|
||||
template: ui/deployment.yaml
|
||||
set:
|
||||
ui.volumes:
|
||||
- name: nginx-config
|
||||
configMap:
|
||||
name: custom-nginx
|
||||
ui.volumeMounts:
|
||||
- name: nginx-config
|
||||
mountPath: /etc/nginx/conf.d
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.volumes
|
||||
value:
|
||||
- name: nginx-config
|
||||
configMap:
|
||||
name: custom-nginx
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
value:
|
||||
- name: nginx-config
|
||||
mountPath: /etc/nginx/conf.d
|
||||
4
helm/litellm/tests/values/required.yaml
Normal file
4
helm/litellm/tests/values/required.yaml
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
database:
|
||||
writer:
|
||||
host: postgres.example.com
|
||||
dbname: litellm
|
||||
|
|
@ -124,6 +124,11 @@ gateway:
|
|||
extraEnv: [] # Add extra environment variables to the gateway
|
||||
envConfigMaps: [] # Add extra environment variables to the gateway from config maps
|
||||
envSecrets: [] # Add extra environment variables to the gateway from secrets
|
||||
# Additional volumes on the gateway Deployment (e.g. a ConfigMap holding
|
||||
# custom callback / SSO handler code, mounted next to the proxy config).
|
||||
volumes: []
|
||||
# Additional volumeMounts on the gateway container.
|
||||
volumeMounts: []
|
||||
config:
|
||||
create: true
|
||||
proxy_config: {}
|
||||
|
|
@ -167,6 +172,10 @@ backend:
|
|||
extraEnv: []
|
||||
envConfigMaps: []
|
||||
envSecrets: []
|
||||
# Additional volumes on the backend Deployment.
|
||||
volumes: []
|
||||
# Additional volumeMounts on the backend container.
|
||||
volumeMounts: []
|
||||
image:
|
||||
repository: ghcr.io/berriai/litellm-backend
|
||||
tag: ""
|
||||
|
|
@ -206,6 +215,10 @@ ui:
|
|||
extraEnv: []
|
||||
envConfigMaps: []
|
||||
envSecrets: []
|
||||
# Additional volumes on the ui Deployment.
|
||||
volumes: []
|
||||
# Additional volumeMounts on the ui container.
|
||||
volumeMounts: []
|
||||
image:
|
||||
repository: ghcr.io/berriai/litellm-ui
|
||||
tag: ""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "max_concurrent_requests" INTEGER;
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
|
||||
|
||||
-- AlterTable
|
||||
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "budget_fallbacks" JSONB NOT NULL DEFAULT '{}';
|
||||
|
|
@ -338,6 +338,7 @@ model LiteLLM_MCPServerTable {
|
|||
byok_api_key_help_url String?
|
||||
source_url String?
|
||||
timeout Float?
|
||||
max_concurrent_requests Int?
|
||||
// BYOM submission lifecycle
|
||||
approval_status String? @default("active")
|
||||
submitted_by String?
|
||||
|
|
@ -418,6 +419,7 @@ model LiteLLM_VerificationToken {
|
|||
access_group_ids String[] @default([])
|
||||
model_spend Json @default("{}")
|
||||
model_max_budget Json @default("{}")
|
||||
budget_fallbacks Json @default("{}")
|
||||
budget_id String?
|
||||
organization_id String?
|
||||
object_permission_id String?
|
||||
|
|
@ -511,6 +513,7 @@ model LiteLLM_DeletedVerificationToken {
|
|||
access_group_ids String[] @default([])
|
||||
model_spend Json @default("{}")
|
||||
model_max_budget Json @default("{}")
|
||||
budget_fallbacks Json @default("{}")
|
||||
router_settings Json? @default("{}")
|
||||
budget_id String?
|
||||
organization_id String?
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.74"
|
||||
version = "0.4.75"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.74"
|
||||
version = "0.4.75"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -379,6 +379,7 @@ budget_duration: Optional[str] = (
|
|||
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
|
||||
)
|
||||
default_soft_budget: float = DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
|
||||
budget_exceeded_throttle_percentage: Optional[float] = None
|
||||
forward_traceparent_to_llm_provider: bool = False
|
||||
|
||||
|
||||
|
|
@ -588,6 +589,7 @@ gemini_models: Set = set()
|
|||
xai_models: Set = set()
|
||||
zai_models: Set = set()
|
||||
deepseek_models: Set = set()
|
||||
tencent_models: Set = set()
|
||||
runwayml_models: Set = set()
|
||||
azure_ai_models: Set = set()
|
||||
jina_ai_models: Set = set()
|
||||
|
|
@ -801,6 +803,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
|
|||
fal_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepseek":
|
||||
deepseek_models.add(key)
|
||||
elif value.get("litellm_provider") == "tencent":
|
||||
tencent_models.add(key)
|
||||
elif value.get("litellm_provider") == "runwayml":
|
||||
runwayml_models.add(key)
|
||||
elif value.get("litellm_provider") == "meta_llama":
|
||||
|
|
@ -1093,6 +1097,7 @@ models_by_provider: dict = {
|
|||
"zai": zai_models,
|
||||
"fal_ai": fal_ai_models,
|
||||
"deepseek": deepseek_models,
|
||||
"tencent": tencent_models,
|
||||
"runwayml": runwayml_models,
|
||||
"mistral": mistral_chat_models,
|
||||
"azure_ai": azure_ai_models,
|
||||
|
|
@ -1804,6 +1809,9 @@ if TYPE_CHECKING:
|
|||
from .llms.deepseek.chat.transformation import (
|
||||
DeepSeekChatConfig as _DeepSeekChatConfig,
|
||||
)
|
||||
from .llms.tencent.chat.transformation import (
|
||||
TencentChatConfig as _TencentChatConfig,
|
||||
)
|
||||
from .llms.sap.chat.transformation import (
|
||||
GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig,
|
||||
)
|
||||
|
|
@ -1846,6 +1854,7 @@ if TYPE_CHECKING:
|
|||
# Type stubs for lazy-loaded config classes (to help mypy understand types)
|
||||
VLLMConfig: Type[_VLLMConfig]
|
||||
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
|
||||
TencentChatConfig: Type[_TencentChatConfig]
|
||||
GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig]
|
||||
GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig]
|
||||
AzureOpenAIO1Config: Type[_AzureOpenAIO1Config]
|
||||
|
|
|
|||
|
|
@ -284,6 +284,7 @@ LLM_CONFIG_NAMES = (
|
|||
"LiteLLMProxyChatConfig",
|
||||
"VLLMConfig",
|
||||
"DeepSeekChatConfig",
|
||||
"TencentChatConfig",
|
||||
"LMStudioChatConfig",
|
||||
"LmStudioEmbeddingConfig",
|
||||
"NscaleConfig",
|
||||
|
|
@ -1096,6 +1097,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
|
|||
),
|
||||
"VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"),
|
||||
"DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"),
|
||||
"TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"),
|
||||
"LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"),
|
||||
"LmStudioEmbeddingConfig": (
|
||||
".llms.lm_studio.embed.transformation",
|
||||
|
|
|
|||
|
|
@ -129,6 +129,33 @@ def _set_agent_id_on_logging_obj(
|
|||
litellm_logging_obj.model_call_details["agent_id"] = agent_id
|
||||
|
||||
|
||||
_A2A_COST_PARAM_KEYS = ("cost_per_query", "input_cost_per_token", "output_cost_per_token")
|
||||
|
||||
|
||||
def _set_litellm_params_on_logging_obj(
|
||||
kwargs: dict[str, Any],
|
||||
litellm_params: dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Merge the agent's pricing params into model_call_details["litellm_params"]
|
||||
so A2ACostCalculator can read them.
|
||||
|
||||
The non-streaming path reuses the proxy-built logging object, whose
|
||||
litellm_params already carries metadata / proxy_server_request / user-key
|
||||
context, so merge the pricing keys in rather than replacing the dict.
|
||||
"""
|
||||
logging_obj = kwargs.get("litellm_logging_obj")
|
||||
if logging_obj is None:
|
||||
return
|
||||
|
||||
cost_params = {key: litellm_params[key] for key in _A2A_COST_PARAM_KEYS if litellm_params.get(key) is not None}
|
||||
if not cost_params:
|
||||
return
|
||||
|
||||
existing = logging_obj.model_call_details.get("litellm_params") or {}
|
||||
logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params}
|
||||
|
||||
|
||||
def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Extract agent info and set model/custom_llm_provider for cost tracking.
|
||||
|
|
@ -477,6 +504,9 @@ async def asend_message(
|
|||
completion_tokens=completion_tokens,
|
||||
)
|
||||
|
||||
# Merge agent pricing params into the logging obj so cost is calculated
|
||||
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
|
||||
|
||||
# Set agent_id on logging obj for SpendLogs tracking
|
||||
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from litellm._logging import verbose_logger
|
|||
from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
|
||||
from litellm.a2a_protocol.utils import A2ARequestUtils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from a2a.types import SendStreamingMessageRequest, SendStreamingMessageResponse
|
||||
|
|
@ -128,22 +127,15 @@ class A2AStreamingIterator:
|
|||
|
||||
# Call success handlers - they will build standard_logging_object
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
result=result,
|
||||
self.logging_obj.dispatch_success_handlers(
|
||||
result,
|
||||
start_time=self.start_time,
|
||||
end_time=end_time,
|
||||
cache_hit=None,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
result=result,
|
||||
cache_hit=None,
|
||||
start_time=self.start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
||||
verbose_logger.info(
|
||||
f"A2A streaming completed: prompt_tokens={prompt_tokens}, "
|
||||
f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, "
|
||||
|
|
|
|||
|
|
@ -121,8 +121,13 @@ class A2ARequestUtils:
|
|||
Returns:
|
||||
Tuple of (prompt_tokens, completion_tokens, total_tokens)
|
||||
"""
|
||||
# Count input tokens
|
||||
# Count input tokens. Dump the message to a dict first so extraction hits
|
||||
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
|
||||
# kind/text live on part.root, which the object branch cannot read. This
|
||||
# mirrors how the response side already works (it operates on model_dump).
|
||||
input_message = A2ARequestUtils.get_input_message_from_request(request)
|
||||
if input_message is not None and hasattr(input_message, "model_dump"):
|
||||
input_message = input_message.model_dump(mode="json")
|
||||
input_text = A2ARequestUtils.extract_text_from_message(input_message)
|
||||
prompt_tokens = A2ARequestUtils.count_tokens(input_text)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from typing import Any, Iterator, List, Literal, Optional, Tuple
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_prompt_tokens_details
|
||||
from litellm.types.llms.openai import Batch
|
||||
from litellm.types.utils import CallTypes, ModelInfo, Usage
|
||||
from litellm.utils import token_counter
|
||||
|
|
@ -34,7 +35,7 @@ async def calculate_batch_cost_and_usage(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
model_name=model_name,
|
||||
)
|
||||
batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name)
|
||||
batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider)
|
||||
|
||||
return batch_cost, batch_usage, batch_models
|
||||
|
||||
|
|
@ -70,7 +71,7 @@ async def _handle_completed_batch(
|
|||
model_name=model_name,
|
||||
)
|
||||
|
||||
batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name)
|
||||
batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider)
|
||||
|
||||
return batch_cost, batch_usage, batch_models
|
||||
|
||||
|
|
@ -78,6 +79,7 @@ async def _handle_completed_batch(
|
|||
def _get_batch_models_from_file_content(
|
||||
file_content_dictionary: List[dict],
|
||||
model_name: Optional[str] = None,
|
||||
custom_llm_provider: str = "openai",
|
||||
) -> List[str]:
|
||||
"""
|
||||
Get the models from the file content
|
||||
|
|
@ -86,8 +88,8 @@ def _get_batch_models_from_file_content(
|
|||
return [model_name]
|
||||
batch_models = []
|
||||
for _item in file_content_dictionary:
|
||||
if _batch_response_was_successful(_item):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item)
|
||||
if _batch_response_was_successful(_item, custom_llm_provider):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
|
||||
_model = _response_body.get("model")
|
||||
if _model:
|
||||
batch_models.append(_model)
|
||||
|
|
@ -373,10 +375,10 @@ def _get_batch_job_cost_from_file_content(
|
|||
# parse the file content as json
|
||||
verbose_logger.debug("file_content_dictionary=%s", json.dumps(file_content_dictionary, indent=4))
|
||||
for _item in file_content_dictionary:
|
||||
if _batch_response_was_successful(_item):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item)
|
||||
if model_info is not None:
|
||||
usage = _get_batch_job_usage_from_response_body(_response_body)
|
||||
if _batch_response_was_successful(_item, custom_llm_provider):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
|
||||
if model_info is not None or custom_llm_provider == "anthropic":
|
||||
usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider)
|
||||
model = _response_body.get("model", "")
|
||||
prompt_cost, completion_cost = batch_cost_calculator(
|
||||
usage=usage,
|
||||
|
|
@ -418,17 +420,31 @@ def _get_batch_job_total_usage_from_file_content(
|
|||
total_tokens: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
for _item in file_content_dictionary:
|
||||
if _batch_response_was_successful(_item):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item)
|
||||
usage: Usage = _get_batch_job_usage_from_response_body(_response_body)
|
||||
if _batch_response_was_successful(_item, custom_llm_provider):
|
||||
_response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
|
||||
usage: Usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider)
|
||||
total_tokens += usage.total_tokens
|
||||
prompt_tokens += usage.prompt_tokens
|
||||
completion_tokens += usage.completion_tokens
|
||||
prompt_details = _parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens += prompt_details["cache_hit_tokens"]
|
||||
cache_creation_tokens += prompt_details["cache_creation_tokens"]
|
||||
cache_token_params = {
|
||||
key: tokens
|
||||
for key, tokens in (
|
||||
("cache_read_input_tokens", cache_read_tokens),
|
||||
("cache_creation_input_tokens", cache_creation_tokens),
|
||||
)
|
||||
if tokens > 0
|
||||
}
|
||||
return Usage(
|
||||
total_tokens=total_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
**cache_token_params,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -465,27 +481,51 @@ def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
|
||||
def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_provider: str = "openai") -> Usage:
|
||||
"""
|
||||
Get the tokens of a batch job from the response body
|
||||
"""
|
||||
if custom_llm_provider == "anthropic":
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
return AnthropicConfig().calculate_usage(
|
||||
usage_object=response_body.get("usage", None) or {},
|
||||
reasoning_content=None,
|
||||
)
|
||||
_usage_dict = response_body.get("usage", None) or {}
|
||||
usage: Usage = Usage(**_usage_dict)
|
||||
return usage
|
||||
|
||||
|
||||
def _get_response_from_batch_job_output_file(batch_job_output_file: dict) -> Any:
|
||||
def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> dict:
|
||||
"""
|
||||
Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
|
||||
|
||||
Anthropic batch results lines look like:
|
||||
``{"custom_id": ..., "result": {"type": "succeeded", "message": {..., "usage": {...}}}}``
|
||||
"""
|
||||
return batch_results_line.get("result", None) or {}
|
||||
|
||||
|
||||
def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> Any:
|
||||
"""
|
||||
Get the response from the batch job output file
|
||||
"""
|
||||
if custom_llm_provider == "anthropic":
|
||||
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {}
|
||||
_response: dict = batch_job_output_file.get("response", None) or {}
|
||||
_response_body = _response.get("body", None) or {}
|
||||
return _response_body
|
||||
|
||||
|
||||
def _batch_response_was_successful(batch_job_output_file: dict) -> bool:
|
||||
def _batch_response_was_successful(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> bool:
|
||||
"""
|
||||
Check if the batch job response status == 200
|
||||
Check if the batch job response was successful
|
||||
|
||||
OpenAI-shaped output rows report ``response.status_code == 200``; Anthropic
|
||||
message batch results lines report ``result.type == "succeeded"``.
|
||||
"""
|
||||
if custom_llm_provider == "anthropic":
|
||||
return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
|
||||
_response: dict = batch_job_output_file.get("response", None) or {}
|
||||
return _response.get("status_code", None) == 200
|
||||
|
|
|
|||
|
|
@ -59,8 +59,9 @@ class DiskCache(BaseCache):
|
|||
|
||||
def increment_cache(self, key, value: int, **kwargs) -> int:
|
||||
# get the value
|
||||
init_value = self.get_cache(key=key) or 0
|
||||
value = init_value + value # type: ignore
|
||||
cached_value = self.get_cache(key=key)
|
||||
init_value = cached_value if isinstance(cached_value, int) else 0
|
||||
value = init_value + value
|
||||
self.set_cache(key, value, **kwargs)
|
||||
return value
|
||||
|
||||
|
|
@ -76,8 +77,9 @@ class DiskCache(BaseCache):
|
|||
|
||||
async def async_increment(self, key, value: int, **kwargs) -> int:
|
||||
# get the value
|
||||
init_value = await self.async_get_cache(key=key) or 0
|
||||
value = init_value + value # type: ignore
|
||||
cached_value = await self.async_get_cache(key=key)
|
||||
init_value = cached_value if isinstance(cached_value, int) else 0
|
||||
value = init_value + value
|
||||
await self.async_set_cache(key, value, **kwargs)
|
||||
return value
|
||||
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ class ValkeySemanticCache(RedisSemanticCache):
|
|||
print_verbose("No prompt provided for semantic caching")
|
||||
return
|
||||
|
||||
embedding = await self._get_async_embedding(prompt, **kwargs)
|
||||
embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
|
||||
await self._ensure_index_async(len(embedding))
|
||||
|
||||
doc_key = self._doc_key(key)
|
||||
|
|
@ -298,7 +298,7 @@ class ValkeySemanticCache(RedisSemanticCache):
|
|||
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
|
||||
return None
|
||||
|
||||
embedding = await self._get_async_embedding(prompt, **kwargs)
|
||||
embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
|
||||
await self._ensure_index_async(len(embedding))
|
||||
|
||||
search_result = await self.async_client.ft(self.index_name).search(
|
||||
|
|
|
|||
|
|
@ -508,6 +508,7 @@ LITELLM_CHAT_PROVIDERS = [
|
|||
"text-completion-codestral",
|
||||
"text-completion-inception",
|
||||
"deepseek",
|
||||
"tencent",
|
||||
"sambanova",
|
||||
"maritalk",
|
||||
"cloudflare",
|
||||
|
|
@ -729,6 +730,7 @@ openai_compatible_providers: List = [
|
|||
"volcengine",
|
||||
"codestral",
|
||||
"deepseek",
|
||||
"tencent",
|
||||
"deepinfra",
|
||||
"perplexity",
|
||||
"xinference",
|
||||
|
|
@ -1502,6 +1504,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
|
|||
"public_model_groups_links",
|
||||
"cost_discount_config",
|
||||
"cost_margin_config",
|
||||
"budget_exceeded_throttle_percentage",
|
||||
]
|
||||
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
|
||||
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
|
||||
|
|
|
|||
|
|
@ -52,6 +52,9 @@ from litellm.llms.databricks.cost_calculator import (
|
|||
from litellm.llms.deepseek.cost_calculator import (
|
||||
cost_per_token as deepseek_cost_per_token,
|
||||
)
|
||||
from litellm.llms.tencent.cost_calculator import (
|
||||
cost_per_token as tencent_cost_per_token,
|
||||
)
|
||||
from litellm.llms.fireworks_ai.cost_calculator import (
|
||||
cost_per_token as fireworks_ai_cost_per_token,
|
||||
)
|
||||
|
|
@ -219,7 +222,7 @@ def _cost_per_token_custom_pricing_helper(
|
|||
output_cost = completion_tokens * output_cost_per_token
|
||||
return input_cost, output_cost
|
||||
elif custom_cost_per_second is not None:
|
||||
output_cost = custom_cost_per_second * response_time_ms / 1000 # type: ignore
|
||||
output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
|
||||
return 0, output_cost
|
||||
|
||||
return None
|
||||
|
|
@ -625,6 +628,8 @@ def cost_per_token(
|
|||
return gemini_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
|
||||
elif custom_llm_provider == "deepseek":
|
||||
return deepseek_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "tencent":
|
||||
return tencent_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
return perplexity_cost_per_token(model=model, usage=usage_block)
|
||||
elif custom_llm_provider == "xai":
|
||||
|
|
@ -657,29 +662,27 @@ def cost_per_token(
|
|||
data_residency=data_residency,
|
||||
)
|
||||
|
||||
if model_info.get("input_cost_per_second", None) is not None and response_time_ms is not None:
|
||||
input_cost_per_second = model_info.get("input_cost_per_second")
|
||||
if input_cost_per_second is not None and response_time_ms is not None:
|
||||
verbose_logger.debug(
|
||||
"For model=%s - input_cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
model_info.get("input_cost_per_second", None),
|
||||
input_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
## COST PER SECOND ##
|
||||
prompt_tokens_cost_usd_dollar = (
|
||||
model_info["input_cost_per_second"] * response_time_ms / 1000 # type: ignore
|
||||
)
|
||||
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
|
||||
|
||||
if model_info.get("output_cost_per_second", None) is not None and response_time_ms is not None:
|
||||
output_cost_per_second = model_info.get("output_cost_per_second")
|
||||
if output_cost_per_second is not None and response_time_ms is not None:
|
||||
verbose_logger.debug(
|
||||
"For model=%s - output_cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
model_info.get("output_cost_per_second", None),
|
||||
output_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
## COST PER SECOND ##
|
||||
completion_tokens_cost_usd_dollar = (
|
||||
model_info["output_cost_per_second"] * response_time_ms / 1000 # type: ignore
|
||||
)
|
||||
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
|
||||
|
||||
verbose_logger.debug(
|
||||
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
|
||||
|
|
@ -1495,6 +1498,7 @@ def completion_cost(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_model_name=model,
|
||||
data_residency=data_residency,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
elif call_type == _MCP_CALL_TYPE:
|
||||
from litellm.proxy._experimental.mcp_server.cost_calculator import (
|
||||
|
|
@ -2151,17 +2155,23 @@ def batch_cost_calculator(
|
|||
if input_cost_per_token_batches:
|
||||
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
|
||||
elif input_cost_per_token:
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens = details["cache_hit_tokens"]
|
||||
cache_creation_tokens = details["cache_creation_tokens"]
|
||||
|
||||
# Subtract cached tokens from prompt_tokens before calculating cost
|
||||
# Fixes issue where cached tokens are being charged again
|
||||
base_input_tokens = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
total_prompt_cost = (
|
||||
get_billable_input_tokens(usage) * (input_cost_per_token) / 2
|
||||
base_input_tokens * (input_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
details = _parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens = details["cache_hit_tokens"]
|
||||
cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
|
||||
|
||||
cache_creation_cost = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
|
||||
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
|
||||
if output_cost_per_token_batches:
|
||||
total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
|
||||
elif output_cost_per_token:
|
||||
|
|
@ -2297,6 +2307,7 @@ def handle_realtime_stream_cost_calculation(
|
|||
custom_llm_provider: str,
|
||||
litellm_model_name: str,
|
||||
data_residency: Optional[str] = None,
|
||||
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
|
||||
) -> float:
|
||||
"""
|
||||
Handles the cost calculation for realtime stream responses.
|
||||
|
|
@ -2332,14 +2343,25 @@ def handle_realtime_stream_cost_calculation(
|
|||
input_cost_per_token += _input_cost_per_token
|
||||
output_cost_per_token += _output_cost_per_token
|
||||
break # exit if we find a valid model
|
||||
total_cost = input_cost_per_token + output_cost_per_token
|
||||
|
||||
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results):
|
||||
total_cost += handle_realtime_transcription_cost_calculation(
|
||||
transcription_cost = (
|
||||
handle_realtime_transcription_cost_calculation(
|
||||
results=results,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_model_name=litellm_model_name,
|
||||
)
|
||||
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
|
||||
else 0.0
|
||||
)
|
||||
total_cost = input_cost_per_token + output_cost_per_token + transcription_cost
|
||||
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=input_cost_per_token,
|
||||
completion_tokens_cost_usd_dollar=output_cost_per_token,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=total_cost,
|
||||
additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None,
|
||||
)
|
||||
|
||||
return total_cost
|
||||
|
||||
|
|
|
|||
|
|
@ -1165,12 +1165,18 @@ class ModifyResponseException(Exception):
|
|||
request_data: Dict[str, Any],
|
||||
guardrail_name: Optional[str] = None,
|
||||
detection_info: Optional[Dict[str, Any]] = None,
|
||||
original_response: Optional[Any] = None,
|
||||
):
|
||||
self.message = message
|
||||
self.model = model
|
||||
self.request_data = request_data
|
||||
self.guardrail_name = guardrail_name
|
||||
self.detection_info = detection_info or {}
|
||||
# The LLM response that was blocked (post-call). Carries the real token
|
||||
# usage the upstream call consumed, so the synthetic block response can
|
||||
# report it instead of discarding it. None for pre-call blocks (the LLM
|
||||
# was never invoked).
|
||||
self.original_response = original_response
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -520,13 +520,28 @@ class MCPClient:
|
|||
# Return empty list instead of raising to allow graceful degradation
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def error_tool_result(exc: Exception) -> MCPCallToolResult:
|
||||
"""The error result ``call_tool`` returns when it swallows a failure (no re-execution)."""
|
||||
return MCPCallToolResult(
|
||||
content=[TextContent(type="text", text=f"{type(exc).__name__}: {str(exc)}")],
|
||||
isError=True,
|
||||
)
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
call_tool_request_params: MCPCallToolRequestParams,
|
||||
host_progress_callback: Optional[Callable] = None,
|
||||
raise_on_error: bool = False,
|
||||
) -> MCPCallToolResult:
|
||||
"""
|
||||
Call an MCP Tool.
|
||||
|
||||
Args:
|
||||
raise_on_error: When True, re-raise the underlying exception instead of returning an
|
||||
``isError=True`` result. The token-exchange (OBO) tool-call path uses this to detect
|
||||
an upstream 401 so it can re-mint the exchanged token and retry once; every other
|
||||
caller keeps the default and gets graceful ``isError`` degradation.
|
||||
"""
|
||||
verbose_logger.info(f"MCP client calling tool '{call_tool_request_params.name}'")
|
||||
|
||||
|
|
@ -579,11 +594,10 @@ class MCPClient:
|
|||
"MCP client detected broken connection/stream - "
|
||||
"the MCP server may have crashed, disconnected, or timed out."
|
||||
)
|
||||
if raise_on_error:
|
||||
raise
|
||||
# Return a default error result instead of raising
|
||||
return MCPCallToolResult(
|
||||
content=[TextContent(type="text", text=f"{error_type}: {str(e)}")], # Empty content for error case
|
||||
isError=True,
|
||||
)
|
||||
return self.error_tool_result(e)
|
||||
|
||||
async def list_prompts(self) -> List[Prompt]:
|
||||
"""List available prompts from the server."""
|
||||
|
|
|
|||
|
|
@ -256,8 +256,6 @@ def create_fine_tuning_job(
|
|||
extra_body = optional_params.get("extra_body", {})
|
||||
if extra_body is not None:
|
||||
extra_body.pop("azure_ad_token", None)
|
||||
else:
|
||||
get_secret_str("AZURE_AD_TOKEN") # type: ignore
|
||||
|
||||
# Prepare Azure-specific parameters for extra_body
|
||||
extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
|
||||
|
|
@ -442,7 +440,7 @@ def cancel_fine_tuning_job(
|
|||
)
|
||||
# Azure OpenAI
|
||||
elif custom_llm_provider == "azure":
|
||||
api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore
|
||||
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
|
||||
|
||||
api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") # type: ignore
|
||||
|
||||
|
|
@ -457,8 +455,6 @@ def cancel_fine_tuning_job(
|
|||
extra_body = optional_params.get("extra_body", {})
|
||||
if extra_body is not None:
|
||||
extra_body.pop("azure_ad_token", None)
|
||||
else:
|
||||
get_secret_str("AZURE_AD_TOKEN") # type: ignore
|
||||
|
||||
response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job(
|
||||
api_base=api_base,
|
||||
|
|
@ -616,8 +612,6 @@ def list_fine_tuning_jobs(
|
|||
extra_body = optional_params.get("extra_body", {})
|
||||
if extra_body is not None:
|
||||
extra_body.pop("azure_ad_token", None)
|
||||
else:
|
||||
get_secret("AZURE_AD_TOKEN") # type: ignore
|
||||
|
||||
response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs(
|
||||
api_base=api_base,
|
||||
|
|
@ -759,8 +753,6 @@ def retrieve_fine_tuning_job(
|
|||
extra_body = optional_params.get("extra_body", {})
|
||||
if extra_body is not None:
|
||||
extra_body.pop("azure_ad_token", None)
|
||||
else:
|
||||
get_secret_str("AZURE_AD_TOKEN") # type: ignore
|
||||
|
||||
response = azure_fine_tuning_apis_instance.retrieve_fine_tuning_job(
|
||||
api_base=api_base,
|
||||
|
|
|
|||
|
|
@ -61,13 +61,15 @@ class AzureSentinelLogger(CustomBatchLogger):
|
|||
client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
|
||||
If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
|
||||
audit_stream_name (str, optional): Stream name from DCR for audit logs.
|
||||
If not provided, audit logs use the standard stream name.
|
||||
If not provided, will use AZURE_SENTINEL_AUDIT_STREAM_NAME env var or the standard stream name.
|
||||
"""
|
||||
self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
|
||||
|
||||
resolved_dcr_immutable_id = dcr_immutable_id or os.getenv("AZURE_SENTINEL_DCR_IMMUTABLE_ID")
|
||||
resolved_stream_name = stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
|
||||
resolved_audit_stream_name = audit_stream_name or resolved_stream_name
|
||||
resolved_audit_stream_name = (
|
||||
audit_stream_name or os.getenv("AZURE_SENTINEL_AUDIT_STREAM_NAME") or resolved_stream_name
|
||||
)
|
||||
resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
|
||||
resolved_tenant_id = tenant_id or os.getenv("AZURE_SENTINEL_TENANT_ID") or os.getenv("AZURE_TENANT_ID")
|
||||
resolved_client_id = client_id or os.getenv("AZURE_SENTINEL_CLIENT_ID") or os.getenv("AZURE_CLIENT_ID")
|
||||
|
|
|
|||
|
|
@ -354,14 +354,14 @@ class DataDogLogger(
|
|||
Raises:
|
||||
Raises a NON Blocking verbose_logger.exception if an error occurs
|
||||
"""
|
||||
if not self.log_queue:
|
||||
verbose_logger.exception("Datadog: log_queue does not exist")
|
||||
return
|
||||
|
||||
batch_to_send = self.log_queue[:]
|
||||
self.log_queue = []
|
||||
|
||||
try:
|
||||
if not self.log_queue:
|
||||
verbose_logger.exception("Datadog: log_queue does not exist")
|
||||
return
|
||||
|
||||
batch_to_send = self.log_queue[:]
|
||||
self.log_queue = []
|
||||
|
||||
verbose_logger.debug(
|
||||
"Datadog - about to flush %s events on %s",
|
||||
len(batch_to_send),
|
||||
|
|
|
|||
|
|
@ -368,7 +368,11 @@ class OpenTelemetryV2(CustomLogger):
|
|||
# it (named provisionally) so it isn't leaked as an open span.
|
||||
carrier.span.end(end_time=to_ns(end_time))
|
||||
return None
|
||||
data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content)
|
||||
data = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload,
|
||||
capture_content=self.config.capture_span_content,
|
||||
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
|
||||
)
|
||||
end_time_ns = to_ns(end_time)
|
||||
if carrier.span is not None:
|
||||
# Born at the boundary: stamp attributes from the typed payload, set
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ class GenAIMapper:
|
|||
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
|
||||
GenAI.RESPONSE_ID: lambda d: d.response_id,
|
||||
GenAI.RESPONSE_FINISH_REASONS: lambda d: list(d.finish_reasons) if d.finish_reasons else None,
|
||||
GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
|
||||
GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
|
||||
GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
|
||||
Error.TYPE: lambda d: d.error.error_type if d.error else None,
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ from typing import TYPE_CHECKING, Any, Mapping, cast
|
|||
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.integrations.otel.model.semconv import resolve_operation
|
||||
from litellm.integrations.otel.model.utils import as_str
|
||||
from litellm.integrations.otel.model.utils import as_str, to_seconds
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import StandardLoggingPayload
|
||||
|
|
@ -201,6 +201,7 @@ class LLMCallEvent:
|
|||
# span is renamed from the typed payload at close (``finish_span``); this only
|
||||
# needs to be reasonable for a span that never gets closed (a leak).
|
||||
provisional_span_name: str
|
||||
time_to_first_chunk_seconds: float | None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent":
|
||||
|
|
@ -214,9 +215,25 @@ class LLMCallEvent:
|
|||
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
|
||||
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
|
||||
provisional_span_name=f"{operation.value} {model}".strip(),
|
||||
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
|
||||
)
|
||||
|
||||
|
||||
def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
|
||||
"""Seconds from the upstream request being issued (``api_call_start_time``)
|
||||
to the first streamed chunk (``completion_start_time``); ``None`` for
|
||||
non-streaming calls, where ``completion_start_time`` is backfilled with the
|
||||
end time and would not measure first-chunk latency."""
|
||||
optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {})
|
||||
if not optional_params.get("stream"):
|
||||
return None
|
||||
api_call_start = to_seconds(kwargs.get("api_call_start_time"))
|
||||
completion_start = to_seconds(kwargs.get("completion_start_time"))
|
||||
if api_call_start is None or completion_start is None:
|
||||
return None
|
||||
return completion_start - api_call_start
|
||||
|
||||
|
||||
def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]) -> str | None:
|
||||
"""The call id from the payload (when closed) or the bare kwargs (at pre_call)."""
|
||||
if payload is not None:
|
||||
|
|
|
|||
|
|
@ -305,10 +305,14 @@ class LLMCallSpanData:
|
|||
messages_in: tuple[Mapping[str, object], ...] = ()
|
||||
choices_out: tuple[Mapping[str, object], ...] = ()
|
||||
system_fingerprint: str | None = None
|
||||
time_to_first_chunk_seconds: float | None = None
|
||||
|
||||
@classmethod
|
||||
def from_standard_logging_payload(
|
||||
cls, payload: "StandardLoggingPayload", capture_content: bool = False
|
||||
cls,
|
||||
payload: "StandardLoggingPayload",
|
||||
capture_content: bool = False,
|
||||
time_to_first_chunk_seconds: float | None = None,
|
||||
) -> "LLMCallSpanData":
|
||||
params = cast(Mapping[str, object], payload.get("model_parameters") or {})
|
||||
# The single parse of the request's metadata — the request-vs-provider
|
||||
|
|
@ -349,6 +353,7 @@ class LLMCallSpanData:
|
|||
messages_in=_dicts(payload.get("messages")) if capture_content else (),
|
||||
choices_out=choices_out if capture_content else (),
|
||||
system_fingerprint=as_str(response.get("system_fingerprint")),
|
||||
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ class GenAI:
|
|||
RESPONSE_ID: Final = "gen_ai.response.id"
|
||||
RESPONSE_MODEL: Final = "gen_ai.response.model"
|
||||
RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons"
|
||||
RESPONSE_TIME_TO_FIRST_CHUNK: Final = "gen_ai.response.time_to_first_chunk"
|
||||
# usage
|
||||
USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
|
||||
USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.integrations.opentelemetry import (
|
|||
_build_metric_attribute_filter,
|
||||
_resolve_metric_attribute_filter,
|
||||
)
|
||||
from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
|
||||
from litellm.integrations.otel.model.semconv import Metric, resolve_operation
|
||||
from litellm.integrations.otel.model.utils import to_seconds
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
|
@ -181,13 +182,10 @@ class GenAIMetricRecorder:
|
|||
self._metrics.token_usage.record(usage.get("completion_tokens", 0), attributes=out_attrs)
|
||||
|
||||
def _record_time_to_first_token(self, kwargs: Mapping[str, Any], common_attrs: dict) -> None:
|
||||
if not kwargs.get("optional_params", {}).get("stream", False):
|
||||
time_to_first_chunk = time_to_first_chunk_seconds(kwargs)
|
||||
if time_to_first_chunk is None:
|
||||
return
|
||||
api_call_start = to_seconds(kwargs.get("api_call_start_time"))
|
||||
completion_start = to_seconds(kwargs.get("completion_start_time"))
|
||||
if api_call_start is None or completion_start is None:
|
||||
return
|
||||
self._metrics.time_to_first_token.record(completion_start - api_call_start, attributes=common_attrs)
|
||||
self._metrics.time_to_first_token.record(time_to_first_chunk, attributes=common_attrs)
|
||||
|
||||
def _record_time_per_output_token(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta
|
||||
|
|
@ -65,6 +66,26 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
AsyncIOScheduler = Any
|
||||
|
||||
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT = 5.0
|
||||
|
||||
|
||||
def _get_budget_metrics_per_request_timeout() -> float:
|
||||
raw = os.getenv("PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT")
|
||||
if raw is None:
|
||||
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
|
||||
try:
|
||||
parsed = float(raw)
|
||||
except ValueError:
|
||||
parsed = None
|
||||
if parsed is None or not math.isfinite(parsed) or parsed <= 0:
|
||||
verbose_logger.debug(
|
||||
"[Non-Blocking] Prometheus: invalid PROMETHEUS_BUDGET_METRICS_PER_REQUEST_TIMEOUT=%r; using default %ss.",
|
||||
raw,
|
||||
_DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT,
|
||||
)
|
||||
return _DEFAULT_BUDGET_METRICS_PER_REQUEST_TIMEOUT
|
||||
return parsed
|
||||
|
||||
|
||||
class PrometheusLogger(CustomLogger):
|
||||
# Class variables or attributes
|
||||
|
|
@ -1607,7 +1628,15 @@ class PrometheusLogger(CustomLogger):
|
|||
_user_spend = _metadata.get("user_api_key_user_spend", None)
|
||||
_user_max_budget = _metadata.get("user_api_key_user_max_budget", None)
|
||||
|
||||
results = await asyncio.gather(
|
||||
# Bound the per-request budget-metric emission so that slow Redis/DB
|
||||
# lookups under load cannot consume the whole LoggingWorker watchdog
|
||||
# (LOGGING_WORKER_MAX_TIME_PER_COROUTINE, default 20s) and get the entire
|
||||
# success-logging event cancelled. Budget gauges are also refreshed by the
|
||||
# periodic cron every PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES,
|
||||
# so dropping one slow per-request emission only loses sub-cron real-time
|
||||
# detail, not correctness.
|
||||
budget_metrics_timeout = _get_budget_metrics_per_request_timeout()
|
||||
gather_coro = asyncio.gather(
|
||||
self._set_api_key_budget_metrics_after_api_request(
|
||||
user_api_key=user_api_key,
|
||||
user_api_key_alias=user_api_key_alias,
|
||||
|
|
@ -1634,6 +1663,16 @@ class PrometheusLogger(CustomLogger):
|
|||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
try:
|
||||
results = await asyncio.wait_for(gather_coro, timeout=budget_metrics_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
verbose_logger.debug(
|
||||
"[Non-Blocking] Prometheus: per-request budget metric emission "
|
||||
"exceeded %ss under load; skipping (values are refreshed by the "
|
||||
"periodic budget-metrics cron job).",
|
||||
budget_metrics_timeout,
|
||||
)
|
||||
return
|
||||
for i, r in enumerate(results):
|
||||
if isinstance(r, Exception):
|
||||
verbose_logger.debug(
|
||||
|
|
@ -2004,6 +2043,43 @@ class PrometheusLogger(CustomLogger):
|
|||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _extract_api_provider_from_request_data(request_data: dict) -> Optional[str]:
|
||||
"""
|
||||
Best-effort provider for the client-side failure path.
|
||||
|
||||
A request can fail before a deployment is resolved, so the provider is
|
||||
not always known. Prefer the resolved ``custom_llm_provider`` on
|
||||
``litellm_params``, then any provider recovered onto a partial
|
||||
``standard_logging_object`` (e.g. a stream that broke mid-flight), and
|
||||
finally infer it from the requested model name (e.g. ``gpt-4o-mini`` ->
|
||||
``openai``) since the proxy's failure ``request_data`` usually carries
|
||||
only the client-supplied model. Return ``None`` when it cannot be
|
||||
determined so the label emits empty rather than a guess.
|
||||
"""
|
||||
litellm_params = request_data.get("litellm_params") or {}
|
||||
provider = litellm_params.get("custom_llm_provider")
|
||||
if provider:
|
||||
return provider
|
||||
standard_logging_object = request_data.get("standard_logging_object") or {}
|
||||
provider = standard_logging_object.get("custom_llm_provider")
|
||||
if provider:
|
||||
return provider
|
||||
model = litellm_params.get("model") or request_data.get("model")
|
||||
if not model:
|
||||
return None
|
||||
try:
|
||||
return litellm.get_llm_provider(model=model)[1] or None
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return None
|
||||
except Exception as e: # noqa: BLE001 - metrics labeling must never break request/failure handling
|
||||
verbose_logger.debug(
|
||||
"prometheus: unexpected error inferring api_provider from model=%s: %s",
|
||||
model,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
|
|
@ -2039,6 +2115,7 @@ class PrometheusLogger(CustomLogger):
|
|||
_metadata = request_data.get("metadata", {}) or {}
|
||||
model_id = _metadata.get("model_info", {}).get("id") or request_data.get("model_info", {}).get("id")
|
||||
rate_limit_category, rate_limit_type = self._extract_rate_limit_labels(original_exception)
|
||||
api_provider = self._extract_api_provider_from_request_data(request_data)
|
||||
enum_values = UserAPIKeyLabelValues(
|
||||
end_user=user_api_key_dict.end_user_id,
|
||||
user=user_api_key_dict.user_id,
|
||||
|
|
@ -2060,6 +2137,7 @@ class PrometheusLogger(CustomLogger):
|
|||
client_ip=_metadata.get("requester_ip_address"),
|
||||
user_agent=_metadata.get("user_agent"),
|
||||
model_id=model_id,
|
||||
api_provider=api_provider,
|
||||
stream=(str(request_data.get("stream")) if litellm.prometheus_emit_stream_label else None),
|
||||
)
|
||||
_label_ctx = PrometheusLabelFactoryContext(enum_values)
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
s3_strip_base64_files: bool = False,
|
||||
s3_use_key_prefix: bool = False,
|
||||
s3_use_virtual_hosted_style: bool = False,
|
||||
s3_server_side_encryption: Optional[str] = None,
|
||||
s3_callback_params_override: Optional[dict] = None,
|
||||
**kwargs,
|
||||
):
|
||||
|
|
@ -92,6 +93,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
s3_strip_base64_files=s3_strip_base64_files,
|
||||
s3_use_key_prefix=s3_use_key_prefix,
|
||||
s3_use_virtual_hosted_style=s3_use_virtual_hosted_style,
|
||||
s3_server_side_encryption=s3_server_side_encryption,
|
||||
)
|
||||
verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}")
|
||||
|
||||
|
|
@ -145,6 +147,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
s3_strip_base64_files: bool = False,
|
||||
s3_use_key_prefix: bool = False,
|
||||
s3_use_virtual_hosted_style: bool = False,
|
||||
s3_server_side_encryption: Optional[str] = None,
|
||||
params_source: Optional[dict] = None,
|
||||
):
|
||||
"""
|
||||
|
|
@ -194,6 +197,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
bool(params.get("s3_use_virtual_hosted_style", False)) or s3_use_virtual_hosted_style
|
||||
)
|
||||
|
||||
self.s3_server_side_encryption = params.get("s3_server_side_encryption") or s3_server_side_encryption
|
||||
|
||||
return
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
|
|
@ -273,6 +278,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
|
||||
async def async_upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
|
||||
try:
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
import requests
|
||||
|
|
@ -317,14 +323,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
|
||||
# Calculate SHA256 hash of the content
|
||||
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
|
||||
content_md5 = base64.b64encode(
|
||||
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
|
||||
).decode()
|
||||
|
||||
# Prepare the request
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Content-MD5": content_md5,
|
||||
"x-amz-content-sha256": content_hash,
|
||||
"Content-Language": "en",
|
||||
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
|
||||
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
|
||||
**(
|
||||
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
|
||||
if self.s3_server_side_encryption
|
||||
else {}
|
||||
),
|
||||
}
|
||||
req = requests.Request("PUT", url, data=json_string, headers=headers)
|
||||
prepped = req.prepare()
|
||||
|
|
@ -447,6 +462,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
|
||||
def upload_data_to_s3(self, batch_logging_element: s3BatchLoggingElement):
|
||||
try:
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
import requests
|
||||
|
|
@ -482,14 +498,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
|
|||
|
||||
# Calculate SHA256 hash of the content
|
||||
content_hash = hashlib.sha256(json_string.encode("utf-8")).hexdigest()
|
||||
content_md5 = base64.b64encode(
|
||||
hashlib.md5(json_string.encode("utf-8"), usedforsecurity=False).digest()
|
||||
).decode()
|
||||
|
||||
# Prepare the request
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Content-MD5": content_md5,
|
||||
"x-amz-content-sha256": content_hash,
|
||||
"Content-Language": "en",
|
||||
"Content-Disposition": f'inline; filename="{batch_logging_element.s3_object_download_filename}"',
|
||||
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
|
||||
**(
|
||||
{"x-amz-server-side-encryption": self.s3_server_side_encryption}
|
||||
if self.s3_server_side_encryption
|
||||
else {}
|
||||
),
|
||||
}
|
||||
req = requests.Request("PUT", url, data=json_string, headers=headers)
|
||||
prepped = req.prepare()
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
messages: List[Dict],
|
||||
tools: Optional[List[Dict]],
|
||||
custom_llm_provider: Optional[str],
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Short-circuit web-search-only requests by executing the search directly.
|
||||
|
|
@ -176,7 +177,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
# Execute search — keep the structured SearchResponse so the native
|
||||
# block can carry per-result url/title/page_age.
|
||||
try:
|
||||
search_result_text, structured = await self._execute_search(query)
|
||||
if kwargs is None:
|
||||
search_result_text, structured = await self._execute_search(query)
|
||||
else:
|
||||
search_result_text, structured = await self._execute_search(query, kwargs=kwargs)
|
||||
except Exception as e:
|
||||
verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}")
|
||||
search_result_text, structured = f"Search failed: {e}", None
|
||||
|
|
@ -936,7 +940,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
query = tool_call["input"].get("query")
|
||||
if query:
|
||||
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
|
||||
search_tasks.append(self._execute_search(query))
|
||||
search_tasks.append(self._execute_search(query, kwargs=kwargs))
|
||||
else:
|
||||
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query")
|
||||
# Add empty result for tools without query
|
||||
|
|
@ -1009,7 +1013,9 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
)
|
||||
return patch, structured_results
|
||||
|
||||
async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
|
||||
async def _execute_search(
|
||||
self, query: str, kwargs: Optional[dict[str, Any]] = None
|
||||
) -> Tuple[str, Optional[SearchResponse]]:
|
||||
"""
|
||||
Execute a single web search using router's search tools.
|
||||
|
||||
|
|
@ -1031,36 +1037,13 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
)
|
||||
llm_router = None
|
||||
|
||||
# Determine search provider from router's search_tools
|
||||
search_tool = self._select_search_tool_from_router(llm_router=llm_router)
|
||||
search_provider: Optional[str] = None
|
||||
if llm_router is not None and hasattr(llm_router, "search_tools"):
|
||||
if self.search_tool_name:
|
||||
# Find specific search tool by name
|
||||
matching_tools = [
|
||||
tool
|
||||
for tool in llm_router.search_tools
|
||||
if tool.get("search_tool_name") == self.search_tool_name
|
||||
]
|
||||
if matching_tools:
|
||||
search_tool = matching_tools[0]
|
||||
search_provider = search_tool.get("litellm_params", {}).get("search_provider")
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
|
||||
f"with provider '{search_provider}'"
|
||||
)
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
|
||||
"falling back to first available or perplexity"
|
||||
)
|
||||
|
||||
# If no specific tool or not found, use first available
|
||||
if not search_provider and llm_router.search_tools:
|
||||
first_tool = llm_router.search_tools[0]
|
||||
search_provider = first_tool.get("litellm_params", {}).get("search_provider")
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
|
||||
)
|
||||
search_litellm_params: dict[str, Any] = {}
|
||||
if search_tool is not None:
|
||||
await self._authorize_search_tool(search_tool=search_tool, kwargs=kwargs)
|
||||
search_litellm_params = dict(search_tool.get("litellm_params", {}) or {})
|
||||
search_provider = search_litellm_params.get("search_provider")
|
||||
|
||||
# Fallback to perplexity if no router or no search tools configured
|
||||
if not search_provider:
|
||||
|
|
@ -1073,7 +1056,12 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
|
||||
)
|
||||
result = await litellm.asearch(query=query, search_provider=search_provider)
|
||||
search_kwargs = {
|
||||
key: value
|
||||
for key, value in search_litellm_params.items()
|
||||
if key != "search_provider" and value is not None
|
||||
}
|
||||
result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs)
|
||||
|
||||
# Format using transformation function
|
||||
search_result_text = WebSearchTransformation.format_search_response(result)
|
||||
|
|
@ -1086,6 +1074,107 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}")
|
||||
raise
|
||||
|
||||
async def _authorize_search_tool(
|
||||
self,
|
||||
search_tool: dict[str, Any],
|
||||
kwargs: Optional[dict[str, Any]],
|
||||
) -> None:
|
||||
search_tool_name = search_tool.get("search_tool_name")
|
||||
if not isinstance(search_tool_name, str) or not search_tool_name:
|
||||
return
|
||||
|
||||
user_api_key_auth = self._get_user_api_key_auth_from_kwargs(kwargs)
|
||||
if user_api_key_auth is None:
|
||||
return
|
||||
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
can_key_call_search_tool,
|
||||
can_team_call_search_tool,
|
||||
get_team_object,
|
||||
)
|
||||
|
||||
await can_key_call_search_tool(
|
||||
search_tool_name=search_tool_name,
|
||||
valid_token=user_api_key_auth,
|
||||
)
|
||||
|
||||
team_id = getattr(user_api_key_auth, "team_id", None)
|
||||
if team_id:
|
||||
from litellm.proxy.proxy_server import (
|
||||
prisma_client,
|
||||
proxy_logging_obj,
|
||||
user_api_key_cache,
|
||||
)
|
||||
|
||||
team_object = await get_team_object(
|
||||
team_id=team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await can_team_call_search_tool(
|
||||
search_tool_name=search_tool_name,
|
||||
team_object=team_object,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any:
|
||||
if not kwargs:
|
||||
return None
|
||||
|
||||
for metadata_key in ("metadata", "litellm_metadata"):
|
||||
metadata = kwargs.get(metadata_key)
|
||||
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
|
||||
return metadata["user_api_key_auth"]
|
||||
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not isinstance(litellm_params, dict):
|
||||
return None
|
||||
|
||||
for metadata_key in ("metadata", "litellm_metadata"):
|
||||
metadata = litellm_params.get(metadata_key)
|
||||
if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
|
||||
return metadata["user_api_key_auth"]
|
||||
|
||||
return None
|
||||
|
||||
def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]:
|
||||
if llm_router is None or not hasattr(llm_router, "search_tools"):
|
||||
return None
|
||||
search_tools = list(getattr(llm_router, "search_tools") or [])
|
||||
return self._select_search_tool_from_list(search_tools=search_tools, source="router")
|
||||
|
||||
def _select_search_tool_from_list(
|
||||
self,
|
||||
search_tools: list[dict[str, Any]],
|
||||
source: str,
|
||||
) -> Optional[dict[str, Any]]:
|
||||
if self.search_tool_name:
|
||||
matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
|
||||
if matching_tools:
|
||||
search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
|
||||
f"from {source} with provider '{search_provider}'"
|
||||
)
|
||||
return matching_tools[0]
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, "
|
||||
"falling back to first available or perplexity"
|
||||
)
|
||||
|
||||
if search_tools:
|
||||
first_tool = search_tools[0]
|
||||
search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider")
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Using first available search tool from {source} "
|
||||
f"with provider '{search_provider}'"
|
||||
)
|
||||
return first_tool
|
||||
|
||||
return None
|
||||
|
||||
async def _execute_chat_completion_agentic_loop(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1145,7 +1234,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
if query:
|
||||
verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
|
||||
search_tasks.append(self._execute_search(query))
|
||||
search_tasks.append(self._execute_search(query, kwargs=kwargs))
|
||||
else:
|
||||
verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query")
|
||||
# Add empty result for tools without query
|
||||
|
|
|
|||
|
|
@ -174,22 +174,15 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
|||
logging_response = copy.deepcopy(self.completed_response)
|
||||
|
||||
asyncio.create_task(
|
||||
self.logging_obj.async_success_handler(
|
||||
result=logging_response,
|
||||
self.logging_obj.dispatch_success_handlers(
|
||||
logging_response,
|
||||
start_time=self.start_time,
|
||||
end_time=datetime.now(),
|
||||
cache_hit=None,
|
||||
prefer_async_handlers=True,
|
||||
)
|
||||
)
|
||||
|
||||
executor.submit(
|
||||
self.logging_obj.success_handler,
|
||||
result=logging_response,
|
||||
cache_hit=None,
|
||||
start_time=self.start_time,
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -123,6 +123,34 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
|
|||
return ProcessedAudioFile(file_content=file_content, filename=filename, content_type=content_type)
|
||||
|
||||
|
||||
BARE_ISO_639_1_TO_BCP47 = {
|
||||
"en": "en-US",
|
||||
"es": "es-ES",
|
||||
"de": "de-DE",
|
||||
"fr": "fr-FR",
|
||||
"it": "it-IT",
|
||||
"pt": "pt-BR",
|
||||
"ja": "ja-JP",
|
||||
"ko": "ko-KR",
|
||||
"zh": "zh-CN",
|
||||
"ru": "ru-RU",
|
||||
"hi": "hi-IN",
|
||||
"ar": "ar-SA",
|
||||
}
|
||||
|
||||
|
||||
def normalize_transcription_language_to_bcp47(language: str) -> str:
|
||||
"""
|
||||
OpenAI's transcription `language` param accepts bare ISO-639-1 codes like
|
||||
``en``; speech APIs such as Google Speech-to-Text and NVIDIA Riva require
|
||||
BCP-47 like ``en-US``. Map the most common bare codes and pass through
|
||||
anything already region-qualified (or unknown, for a clear provider error).
|
||||
"""
|
||||
if "-" in language:
|
||||
return language
|
||||
return BARE_ISO_639_1_TO_BCP47.get(language.lower(), language)
|
||||
|
||||
|
||||
def get_audio_file_name(file_obj: FileTypes) -> str:
|
||||
"""
|
||||
Safely get the name of a file-like object or return its string representation.
|
||||
|
|
|
|||
|
|
@ -1944,7 +1944,7 @@ def _map_azure_exception(
|
|||
response=getattr(original_exception, "response", None),
|
||||
body=getattr(original_exception, "body", None),
|
||||
)
|
||||
elif "invalid_request_error" in error_str:
|
||||
elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
|
||||
raise BadRequestError(
|
||||
message=f"AzureException BadRequestError - {message}",
|
||||
llm_provider="azure",
|
||||
|
|
@ -1986,6 +1986,14 @@ def _map_azure_exception(
|
|||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif original_exception.status_code == 404:
|
||||
raise NotFoundError(
|
||||
message=f"AzureException NotFoundError - {message}",
|
||||
llm_provider="azure",
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif original_exception.status_code == 408:
|
||||
raise Timeout(
|
||||
message=f"AzureException Timeout - {message}",
|
||||
|
|
@ -2173,7 +2181,7 @@ def exception_type( # type: ignore
|
|||
litellm_response_headers = _get_response_headers(original_exception=original_exception)
|
||||
try:
|
||||
error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
|
||||
if model:
|
||||
if model or custom_llm_provider:
|
||||
if hasattr(original_exception, "message"):
|
||||
error_str = (
|
||||
redact_string(str(original_exception.message))
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue