Merge branch 'BerriAI:main' into feat/sambanova-update-2026-10
|
|
@ -38,7 +38,7 @@ commands:
|
|||
parameters:
|
||||
category:
|
||||
type: enum
|
||||
enum: ["backend", "client", "provider-harness"]
|
||||
enum: ["backend", "client", "provider-harness", "redis-compat"]
|
||||
default: "backend"
|
||||
steps:
|
||||
- run:
|
||||
|
|
@ -229,6 +229,19 @@ commands:
|
|||
- wait_for_service:
|
||||
url: tcp://localhost:6379
|
||||
timeout: "60"
|
||||
install_codecov_cli:
|
||||
steps:
|
||||
- run:
|
||||
name: Install Codecov CLI (pinned v11.3.1)
|
||||
when: always
|
||||
command: |
|
||||
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
|
||||
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
|
||||
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
|
||||
(cd /tmp && sha256sum -c codecov.SHA256SUM)
|
||||
chmod +x /tmp/codecov
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
mv /tmp/codecov "$HOME/.local/bin/codecov"
|
||||
start_openai_record_replay_proxy:
|
||||
description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
|
||||
steps:
|
||||
|
|
@ -3279,30 +3292,164 @@ jobs:
|
|||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
unit:
|
||||
postgres_suite:
|
||||
parameters:
|
||||
test_path:
|
||||
type: string
|
||||
seed:
|
||||
type: boolean
|
||||
coverage_flag:
|
||||
type: string
|
||||
default: ""
|
||||
timeout_minutes:
|
||||
type: integer
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
parallelism: 4
|
||||
environment:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps
|
||||
- start_postgres:
|
||||
db_name: litellm_test
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
- run:
|
||||
name: Generate Prisma client
|
||||
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
- when:
|
||||
condition: << parameters.seed >>
|
||||
steps:
|
||||
- run:
|
||||
name: Seed database schema
|
||||
command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
- run:
|
||||
name: Run << parameters.test_path >>
|
||||
command: |
|
||||
mkdir -p test-results
|
||||
coverage_args=()
|
||||
if [ -n "<< parameters.coverage_flag >>" ]; then
|
||||
coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml)
|
||||
fi
|
||||
timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \
|
||||
<< parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \
|
||||
--junitxml=test-results/junit.xml "${coverage_args[@]}"
|
||||
- when:
|
||||
condition: << parameters.coverage_flag >>
|
||||
steps:
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
||||
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
||||
-F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \
|
||||
-n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
mcp_integration:
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
environment:
|
||||
COVERAGE_CORE: sysmon
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Generate Prisma client
|
||||
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
- run:
|
||||
name: Run unit tests
|
||||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
mkdir -p test-results/unit
|
||||
shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
|
||||
if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
|
||||
mapfile -t files < <(printf '%s\n' "${shard}")
|
||||
set +e
|
||||
LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
|
||||
exit "$status"
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- run:
|
||||
name: Run MCP integration tests
|
||||
command: |
|
||||
mkdir -p test-results
|
||||
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
|
||||
timeout --signal=TERM 20m uv run --no-sync pytest \
|
||||
tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
|
||||
tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
|
||||
--reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
|
||||
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
|
||||
--cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload MCP integration coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
||||
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
||||
-F mcp-integration -C "$CIRCLE_SHA1" \
|
||||
-n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
redis_compat:
|
||||
parameters:
|
||||
redis_py:
|
||||
type: string
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_if_unrelated_changes:
|
||||
category: redis-compat
|
||||
- setup_litellm_test_deps
|
||||
- run:
|
||||
name: Pin redis-py version
|
||||
command: |
|
||||
uv pip install "redis==<< parameters.redis_py >>"
|
||||
uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
|
||||
- run:
|
||||
name: Install redis-server 7.2.16
|
||||
command: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
|
||||
docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
|
||||
docker rm "$cid"
|
||||
redis-server --version | grep -q 'v=7.2.16'
|
||||
- run:
|
||||
name: Run Redis compatibility tests
|
||||
command: |
|
||||
mkdir -p test-results
|
||||
env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
|
||||
timeout --signal=TERM 15m uv run --no-sync pytest \
|
||||
tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
|
||||
tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
|
||||
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
|
||||
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
|
||||
--tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
|
||||
--cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
|
||||
- when:
|
||||
condition:
|
||||
equal: ["5.3.1", << parameters.redis_py >>]
|
||||
steps:
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload Redis compatibility coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
|
||||
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
|
||||
-F redis-compat -C "$CIRCLE_SHA1" \
|
||||
-n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
|
|
@ -3376,6 +3523,39 @@ workflows:
|
|||
parameters:
|
||||
suite: [management, database]
|
||||
mode: [replica]
|
||||
- postgres_suite:
|
||||
name: proxy-behavior
|
||||
test_path: tests/proxy_behavior
|
||||
seed: true
|
||||
coverage_flag: lens-postgres
|
||||
timeout_minutes: 25
|
||||
- postgres_suite:
|
||||
name: proxy-security
|
||||
test_path: tests/proxy_security_tests
|
||||
seed: true
|
||||
timeout_minutes: 15
|
||||
- postgres_suite:
|
||||
name: schema-migration
|
||||
test_path: tests/proxy_migration_tests
|
||||
seed: false
|
||||
timeout_minutes: 20
|
||||
- postgres_suite:
|
||||
name: roi-database
|
||||
test_path: tests/integration/database/test_roi_observed.py
|
||||
seed: false
|
||||
coverage_flag: roi-postgres
|
||||
timeout_minutes: 10
|
||||
- mcp_integration:
|
||||
name: mcp-integration
|
||||
- redis_compat:
|
||||
name: redis-compat-<< matrix.redis_py >>
|
||||
matrix:
|
||||
parameters:
|
||||
redis_py:
|
||||
- "5.3.1"
|
||||
- "6.4.0"
|
||||
- "7.4.1"
|
||||
- "8.0.1"
|
||||
build_and_test:
|
||||
unless:
|
||||
or:
|
||||
|
|
@ -3385,7 +3565,6 @@ workflows:
|
|||
jobs:
|
||||
- using_litellm_on_windows
|
||||
- windows_release_wheel
|
||||
- unit
|
||||
- provider_replay_harness
|
||||
- base_sdk_install
|
||||
- local_testing_part1
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
|
||||
|
||||
has_client=false
|
||||
has_backend=false
|
||||
|
|
@ -10,9 +10,14 @@ has_provider_harness=false
|
|||
has_cost_map=false
|
||||
has_mcp_dependencies=false
|
||||
has_windows_release=false
|
||||
has_redis_compat=false
|
||||
outside_cost_map_set=false
|
||||
while IFS= read -r file || [ -n "$file" ]; do
|
||||
[ -n "$file" ] || continue
|
||||
case "$file" in
|
||||
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
|
||||
has_redis_compat=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
*.md | *.mdx) : ;;
|
||||
pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
|
||||
|
|
@ -54,6 +59,9 @@ case "$category" in
|
|||
windows-release)
|
||||
[ "$has_windows_release" = true ] && echo run || echo skip
|
||||
;;
|
||||
redis-compat)
|
||||
[ "$has_redis_compat" = true ] && echo run || echo skip
|
||||
;;
|
||||
backend)
|
||||
[ "$has_backend" = true ] && echo run || echo skip
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: path_filter.sh <backend|client|provider-harness>}"
|
||||
category="${1:?usage: path_filter.sh <backend|client|provider-harness|redis-compat>}"
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
run_full() {
|
||||
|
|
|
|||
|
|
@ -169,11 +169,11 @@ start_proxy() {
|
|||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
|
||||
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
|
||||
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
|
||||
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
|
||||
--host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
|
||||
--use_prisma_db_push --enforce_prisma_migration_check \
|
||||
--use_prisma_db_push \
|
||||
> "$results/$log_name" 2>&1 &
|
||||
launched_pid=$!
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,184 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
flag="${1:?usage: unit_selection.sh <codecov flag>}"
|
||||
|
||||
legacy_flags=(
|
||||
caching-local
|
||||
core-utils
|
||||
enterprise-package
|
||||
enterprise-routing
|
||||
integrations
|
||||
llm-other-providers
|
||||
llm-vertex-ai
|
||||
mcp-integration
|
||||
misc
|
||||
proxy-db-auth-checks
|
||||
proxy-db-budgets
|
||||
proxy-db-custom-logging
|
||||
proxy-db-db-and-spend
|
||||
proxy-db-endpoints-and-responses
|
||||
proxy-db-guardrails-hooks
|
||||
proxy-db-jwt-and-keys
|
||||
proxy-db-key-generation
|
||||
proxy-db-logging-misc
|
||||
proxy-db-proxy-runtime
|
||||
proxy-db-proxy-server-core
|
||||
proxy-db-proxy-utils
|
||||
proxy-extras
|
||||
proxy-infra
|
||||
responses-caching-types
|
||||
)
|
||||
|
||||
legacy_paths() {
|
||||
case "$1" in
|
||||
caching-local) echo tests/unit/caching ;;
|
||||
core-utils) echo tests/unit/litellm_core_utils ;;
|
||||
enterprise-package)
|
||||
echo tests/unit/enterprise/integrations
|
||||
echo tests/unit/enterprise/proxy/auth
|
||||
echo tests/unit/enterprise/proxy/guardrails
|
||||
echo tests/unit/enterprise/proxy/hooks
|
||||
echo tests/unit/enterprise/proxy/management_endpoints
|
||||
echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
|
||||
echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
|
||||
enterprise-routing)
|
||||
echo tests/unit/google_genai
|
||||
echo tests/unit/router_strategy
|
||||
echo tests/unit/router_utils
|
||||
echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
|
||||
echo tests/unit/enterprise/enterprise_callbacks/send_emails
|
||||
echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
|
||||
echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
|
||||
echo tests/unit/enterprise/proxy/test_enterprise_routes.py
|
||||
echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
|
||||
echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
|
||||
echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
|
||||
integrations) echo tests/unit/integrations ;;
|
||||
llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
|
||||
llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
|
||||
mcp-integration)
|
||||
echo tests/unit/experimental_mcp_client
|
||||
echo tests/unit/proxy/_experimental/mcp_server
|
||||
echo tests/unit/responses/mcp
|
||||
echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
|
||||
misc)
|
||||
find tests/unit -maxdepth 1 -name 'test_*.py'
|
||||
echo tests/unit/test_router
|
||||
echo tests/unit/a2a_protocol
|
||||
echo tests/unit/batches
|
||||
echo tests/unit/chat_completions
|
||||
echo tests/unit/completion_extras
|
||||
echo tests/unit/containers
|
||||
echo tests/unit/embeddings
|
||||
echo tests/unit/endpoints
|
||||
echo tests/unit/files
|
||||
echo tests/unit/images
|
||||
echo tests/unit/interactions
|
||||
echo tests/unit/messages
|
||||
echo tests/unit/rag
|
||||
echo tests/unit/rerank_api
|
||||
echo tests/unit/rust_bridge
|
||||
echo tests/unit/secret_managers
|
||||
echo tests/unit/vector_stores
|
||||
echo tests/unit/videos ;;
|
||||
proxy-db-auth-checks)
|
||||
echo tests/unit/proxy/auth/test_auth_checks.py
|
||||
echo tests/unit/proxy/auth/test_user_api_key_auth.py
|
||||
echo tests/unit/proxy/test_credential_slot_registry.py
|
||||
echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
|
||||
proxy-db-budgets)
|
||||
echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
|
||||
echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
|
||||
echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
|
||||
proxy-db-custom-logging)
|
||||
echo tests/unit/proxy/test_custom_callback_input.py
|
||||
echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
|
||||
proxy-db-db-and-spend)
|
||||
echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
|
||||
echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
|
||||
echo tests/unit/proxy/db/test_update_daily_tag_spend.py
|
||||
echo tests/unit/proxy/test_db_schema_changes.py
|
||||
echo tests/unit/proxy/test_prisma_client_backoff_retry.py
|
||||
echo tests/unit/proxy/test_update_spend.py
|
||||
echo tests/unit/skills/test_skills_db.py ;;
|
||||
proxy-db-endpoints-and-responses)
|
||||
echo tests/unit/proxy/lens
|
||||
echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
|
||||
echo tests/unit/proxy/common_utils/test_check_batch_cost.py
|
||||
echo tests/unit/proxy/common_utils/test_check_responses_cost.py
|
||||
echo tests/unit/proxy/common_utils/test_realtime_cache.py
|
||||
echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
|
||||
echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
|
||||
echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
|
||||
echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
|
||||
echo tests/unit/proxy/response_polling
|
||||
echo tests/unit/proxy/test_custom_tokenizer_bug.py
|
||||
echo tests/unit/proxy/test_get_favicon.py
|
||||
echo tests/unit/proxy/test_get_image.py
|
||||
echo tests/unit/proxy/test_prompt_test_endpoint.py
|
||||
echo tests/unit/proxy/test_reducto_ocr_route.py
|
||||
echo tests/unit/proxy/test_response_polling_pre_call_checks.py
|
||||
echo tests/unit/proxy/test_ui_path_detection.py ;;
|
||||
proxy-db-guardrails-hooks)
|
||||
echo tests/unit/proxy/hooks/test_banned_keyword_list.py
|
||||
echo tests/unit/proxy/test_proxy_setting_guardrails.py
|
||||
echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
|
||||
proxy-db-jwt-and-keys)
|
||||
echo tests/unit/proxy/auth/test_jwt.py
|
||||
echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
|
||||
echo tests/unit/proxy/test_proxy_custom_auth.py ;;
|
||||
proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
|
||||
proxy-db-logging-misc)
|
||||
echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
|
||||
echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
|
||||
echo tests/unit/proxy/test_proxy_reject_logging.py ;;
|
||||
proxy-db-proxy-runtime)
|
||||
echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
|
||||
echo tests/unit/proxy/auth/test_proxy_routes.py
|
||||
echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
|
||||
echo tests/unit/proxy/test_proxy_config_unit_test.py
|
||||
echo tests/unit/proxy/test_proxy_token_counter.py
|
||||
echo tests/unit/proxy/test_server_root_path.py ;;
|
||||
proxy-db-proxy-server-core)
|
||||
echo tests/unit/proxy/test_aproxy_startup.py
|
||||
echo tests/unit/proxy/test_proxy_server.py ;;
|
||||
proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
|
||||
proxy-extras) echo tests/unit/litellm_proxy_extras ;;
|
||||
proxy-infra)
|
||||
echo tests/unit/gateway
|
||||
echo tests/unit/proxy/management
|
||||
echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
|
||||
echo tests/unit/proxy/roi_calculator ;;
|
||||
responses-caching-types)
|
||||
find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
|
||||
echo tests/unit/types ;;
|
||||
*) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
expand() {
|
||||
while read -r path; do
|
||||
if [ -d "$path" ]; then
|
||||
find "$path" -name 'test_*.py'
|
||||
elif [ -f "$path" ]; then
|
||||
echo "$path"
|
||||
else
|
||||
echo "unit_selection.sh: $path does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$flag" = unit ]; then
|
||||
comm -23 \
|
||||
<(find tests/unit -name 'test_*.py' | sort) \
|
||||
<(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
|
||||
exit 0
|
||||
fi
|
||||
|
||||
legacy_paths "$flag" | expand | sort
|
||||
|
|
@ -1,416 +0,0 @@
|
|||
version: 2.1
|
||||
|
||||
commands:
|
||||
wait_for_service:
|
||||
parameters:
|
||||
url:
|
||||
type: string
|
||||
timeout:
|
||||
type: string
|
||||
default: "60"
|
||||
steps:
|
||||
- run:
|
||||
name: "Wait for << parameters.url >>"
|
||||
command: |
|
||||
TIMEOUT=<< parameters.timeout >>
|
||||
URL="<< parameters.url >>"
|
||||
ELAPSED=0
|
||||
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
|
||||
if echo "$URL" | grep -q '^tcp://'; then
|
||||
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
|
||||
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
|
||||
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
|
||||
sleep 2; ELAPSED=$((ELAPSED+2))
|
||||
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
||||
done
|
||||
else
|
||||
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
|
||||
sleep 2; ELAPSED=$((ELAPSED+2))
|
||||
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
|
||||
done
|
||||
fi
|
||||
echo "Service ready after ${ELAPSED}s"
|
||||
install_uv:
|
||||
steps:
|
||||
- run:
|
||||
name: Install uv (pinned 0.10.9)
|
||||
command: |
|
||||
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
|
||||
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
|
||||
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
|
||||
rm -f /tmp/uv-install.sh
|
||||
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
install_rust:
|
||||
steps:
|
||||
- run:
|
||||
name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
|
||||
command: |
|
||||
case "$(uname -m)" in
|
||||
x86_64)
|
||||
RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
|
||||
RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
|
||||
;;
|
||||
aarch64)
|
||||
RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
|
||||
RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
|
||||
;;
|
||||
*)
|
||||
echo "install_rust: unsupported architecture $(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
curl -sSLf -o /tmp/rustup-init \
|
||||
"https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
|
||||
echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c -
|
||||
chmod +x /tmp/rustup-init
|
||||
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
|
||||
rm -f /tmp/rustup-init
|
||||
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
rustc --version
|
||||
cargo --version
|
||||
install_codecov_cli:
|
||||
steps:
|
||||
- run:
|
||||
name: Install Codecov CLI (pinned v11.3.1)
|
||||
when: always
|
||||
command: |
|
||||
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
|
||||
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
|
||||
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
|
||||
(cd /tmp && sha256sum -c codecov.SHA256SUM)
|
||||
chmod +x /tmp/codecov
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
mv /tmp/codecov "$HOME/.local/bin/codecov"
|
||||
setup_litellm_enterprise_pip:
|
||||
steps:
|
||||
- run:
|
||||
name: "Install local version of litellm-enterprise"
|
||||
command: |
|
||||
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
|
||||
setup_test_deps:
|
||||
steps:
|
||||
- install_uv
|
||||
- install_rust
|
||||
- restore_cache:
|
||||
keys:
|
||||
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
- run:
|
||||
name: Install Dependencies
|
||||
command: |
|
||||
uv sync --frozen --all-groups --all-extras --python 3.12
|
||||
- setup_litellm_enterprise_pip
|
||||
- save_cache:
|
||||
paths:
|
||||
- ~/.cache/uv
|
||||
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
|
||||
- run:
|
||||
name: Generate Prisma client
|
||||
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
skip_unless_relevant:
|
||||
parameters:
|
||||
category:
|
||||
type: string
|
||||
default: backend
|
||||
base_ref:
|
||||
type: string
|
||||
default: ""
|
||||
pull_request_url:
|
||||
type: string
|
||||
default: ""
|
||||
steps:
|
||||
- run:
|
||||
name: "Skip job when no << parameters.category >>-relevant files changed"
|
||||
command: |
|
||||
export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}"
|
||||
export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>"
|
||||
[ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH
|
||||
bash .circleci/scripts/path_filter.sh << parameters.category >>
|
||||
start_postgres:
|
||||
parameters:
|
||||
db_name:
|
||||
type: string
|
||||
default: circle_test
|
||||
image:
|
||||
type: string
|
||||
default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
||||
steps:
|
||||
- run:
|
||||
name: Start PostgreSQL
|
||||
command: |
|
||||
docker run -d \
|
||||
--name postgres-db \
|
||||
-e POSTGRES_USER=postgres \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=<< parameters.db_name >> \
|
||||
-p 5432:5432 \
|
||||
<< parameters.image >>
|
||||
- wait_for_service:
|
||||
url: tcp://localhost:5432
|
||||
timeout: "60"
|
||||
start_redis:
|
||||
steps:
|
||||
- run:
|
||||
name: Start Redis
|
||||
command: |
|
||||
docker run -d \
|
||||
--name redis-cache \
|
||||
-p 6379:6379 \
|
||||
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
|
||||
- wait_for_service:
|
||||
url: tcp://localhost:6379
|
||||
timeout: "60"
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
parameters:
|
||||
flag:
|
||||
type: string
|
||||
default: unit
|
||||
shards:
|
||||
type: integer
|
||||
default: 6
|
||||
workers:
|
||||
type: integer
|
||||
default: 4
|
||||
dist:
|
||||
type: string
|
||||
default: loadscope
|
||||
base_ref:
|
||||
type: string
|
||||
default: ""
|
||||
pull_request_url:
|
||||
type: string
|
||||
default: ""
|
||||
legacy_mcp_peer:
|
||||
type: boolean
|
||||
default: false
|
||||
reruns:
|
||||
type: integer
|
||||
default: 0
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
parallelism: << parameters.shards >>
|
||||
environment:
|
||||
COVERAGE_CORE: sysmon
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
- checkout
|
||||
- skip_unless_relevant:
|
||||
base_ref: << parameters.base_ref >>
|
||||
pull_request_url: << parameters.pull_request_url >>
|
||||
- setup_test_deps
|
||||
- when:
|
||||
condition: << parameters.legacy_mcp_peer >>
|
||||
steps:
|
||||
- run:
|
||||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- run:
|
||||
name: "Run << parameters.flag >> shard"
|
||||
no_output_timeout: 20m
|
||||
command: |
|
||||
mkdir -p test-results/<< parameters.flag >>
|
||||
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
|
||||
[ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
|
||||
shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
|
||||
[ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
|
||||
mapfile -t files < <(printf '%s\n' "${shard}")
|
||||
xdist_args=()
|
||||
if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
|
||||
rerun_args=(-p no:rerunfailures)
|
||||
if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
|
||||
test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
|
||||
set +e
|
||||
env -i "${test_env[@]}" \
|
||||
uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
|
||||
exit "$status"
|
||||
- install_codecov_cli
|
||||
- run:
|
||||
name: Upload coverage
|
||||
when: always
|
||||
command: |
|
||||
[ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; }
|
||||
codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: coverage.xml
|
||||
documentation:
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- setup_test_deps
|
||||
- run:
|
||||
name: Checkout litellm-docs
|
||||
command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website
|
||||
- run:
|
||||
name: Run documentation validation
|
||||
command: |
|
||||
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
|
||||
integration:
|
||||
parameters:
|
||||
suite:
|
||||
type: string
|
||||
base_ref:
|
||||
type: string
|
||||
default: ""
|
||||
pull_request_url:
|
||||
type: string
|
||||
default: ""
|
||||
machine:
|
||||
image: ubuntu-2204:2024.04.1
|
||||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- skip_unless_relevant:
|
||||
base_ref: << parameters.base_ref >>
|
||||
pull_request_url: << parameters.pull_request_url >>
|
||||
- setup_test_deps
|
||||
- start_postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
- start_redis
|
||||
- run:
|
||||
name: Run owned integration contracts
|
||||
command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
|
||||
no_output_timeout: 15m
|
||||
- run:
|
||||
name: Stop owned database and Redis
|
||||
when: always
|
||||
command: |
|
||||
mkdir -p test-results/integration-<< parameters.suite >>
|
||||
docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
|
||||
docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
|
||||
docker rm -f postgres-db redis-cache
|
||||
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
|
||||
- store_test_results:
|
||||
path: test-results
|
||||
- store_artifacts:
|
||||
path: test-results
|
||||
|
||||
workflows:
|
||||
tests:
|
||||
when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_"))
|
||||
jobs:
|
||||
- unit:
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
workers: 2
|
||||
reruns: 2
|
||||
matrix:
|
||||
parameters:
|
||||
flag: [caching-local, proxy-extras, enterprise-routing]
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-mcp-integration
|
||||
flag: mcp-integration
|
||||
shards: 1
|
||||
workers: 2
|
||||
legacy_mcp_peer: true
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-<< matrix.flag >>
|
||||
shards: 1
|
||||
reruns: 2
|
||||
matrix:
|
||||
parameters:
|
||||
flag:
|
||||
- enterprise-package
|
||||
- proxy-infra
|
||||
- responses-caching-types
|
||||
- proxy-db-auth-checks
|
||||
- proxy-db-jwt-and-keys
|
||||
- proxy-db-proxy-server-core
|
||||
- proxy-db-proxy-runtime
|
||||
- proxy-db-custom-logging
|
||||
- proxy-db-logging-misc
|
||||
- proxy-db-db-and-spend
|
||||
- proxy-db-guardrails-hooks
|
||||
- proxy-db-budgets
|
||||
- proxy-db-endpoints-and-responses
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-llm-vertex-ai
|
||||
flag: llm-vertex-ai
|
||||
shards: 2
|
||||
workers: 1
|
||||
reruns: 2
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-llm-other-providers
|
||||
flag: llm-other-providers
|
||||
shards: 3
|
||||
reruns: 2
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-core-utils
|
||||
flag: core-utils
|
||||
shards: 2
|
||||
reruns: 1
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-integrations
|
||||
flag: integrations
|
||||
shards: 2
|
||||
reruns: 3
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-misc
|
||||
flag: misc
|
||||
shards: 2
|
||||
reruns: 2
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-proxy-db-proxy-utils
|
||||
flag: proxy-db-proxy-utils
|
||||
shards: 1
|
||||
reruns: 2
|
||||
dist: worksteal
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- unit:
|
||||
name: unit-proxy-db-key-generation
|
||||
flag: proxy-db-key-generation
|
||||
shards: 1
|
||||
workers: 0
|
||||
reruns: 2
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
- documentation
|
||||
- integration:
|
||||
name: integration-<< matrix.suite >>
|
||||
matrix:
|
||||
parameters:
|
||||
suite: [sdk]
|
||||
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
|
||||
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
|
||||
BIN
.github/assets/roi-calculator-integrations/after-github.jpg
vendored
Normal file
|
After Width: | Height: | Size: 77 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab-detail-top.jpg
vendored
Normal file
|
After Width: | Height: | Size: 79 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab-detail.jpg
vendored
Normal file
|
After Width: | Height: | Size: 61 KiB |
BIN
.github/assets/roi-calculator-integrations/after-gitlab.jpg
vendored
Normal file
|
After Width: | Height: | Size: 86 KiB |
BIN
.github/assets/roi-calculator-integrations/before-github.jpg
vendored
Normal file
|
After Width: | Height: | Size: 75 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-exit-loading.jpg
vendored
Normal file
|
After Width: | Height: | Size: 40 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-fallback-live.jpg
vendored
Normal file
|
After Width: | Height: | Size: 88 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-overview.jpg
vendored
Normal file
|
After Width: | Height: | Size: 95 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-people.jpg
vendored
Normal file
|
After Width: | Height: | Size: 96 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-pr-costs.jpg
vendored
Normal file
|
After Width: | Height: | Size: 94 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-pr-detail.jpg
vendored
Normal file
|
After Width: | Height: | Size: 70 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-preview-link.jpg
vendored
Normal file
|
After Width: | Height: | Size: 82 KiB |
BIN
.github/assets/roi-calculator-integrations/demo-with-live-errors.jpg
vendored
Normal file
|
After Width: | Height: | Size: 82 KiB |
BIN
.github/assets/roi-calculator-integrations/source-race-after.jpg
vendored
Normal file
|
After Width: | Height: | Size: 64 KiB |
BIN
.github/assets/roi-calculator-integrations/source-race-before.jpg
vendored
Normal file
|
After Width: | Height: | Size: 90 KiB |
10
.github/ci-coverage-allowlist.yml
vendored
|
|
@ -4,6 +4,14 @@ description: >-
|
|||
by a job nor listed here, so every entry below is a decision on the record.
|
||||
|
||||
test_paths:
|
||||
- reason: >-
|
||||
litellm.agent() end-to-end suite. It drives the real claude, codex and opencode CLIs and
|
||||
deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus
|
||||
LITELLM_PROXY_API_BASE / LITELLM_PROXY_API_KEY, and skips without them. Run manually
|
||||
before changing litellm/harness; the mocked coverage runs in tests/unit/harness and
|
||||
tests/unit/llms/*/harness
|
||||
paths:
|
||||
- tests/harness_e2e
|
||||
- reason: >-
|
||||
The Rust/Python parity harness is run manually through its local CLI. Recorded replay,
|
||||
fixture generation, and harness checks are intentionally outside pull request CI
|
||||
|
|
@ -13,7 +21,7 @@ test_paths:
|
|||
Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
|
||||
glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
|
||||
not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
|
||||
Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
|
||||
Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in
|
||||
test_caching.py by node ID. It does not run that file's other tests.
|
||||
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
|
||||
run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
|
||||
|
|
|
|||
17
.github/e2e-stack/down.sh
vendored
|
|
@ -1,17 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
|
||||
|
||||
for pid_file in "${STACK_DIR}"/pids/*.pid; do
|
||||
[[ -f "${pid_file}" ]] || continue
|
||||
pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
|
||||
kill -TERM "$(cat "${pid_file}")" 2>/dev/null
|
||||
rm -f "${pid_file}"
|
||||
done
|
||||
|
||||
for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do
|
||||
docker rm -f "${container}" >/dev/null 2>&1
|
||||
done
|
||||
|
||||
exit 0
|
||||
83
.github/e2e-stack/redact_output.py
vendored
|
|
@ -1,83 +0,0 @@
|
|||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from functools import reduce
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from secrets_to_env import MIN_MASKED_LENGTH
|
||||
|
||||
REDACTED: Final = "***"
|
||||
json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
def string_leaves(node: JsonValue) -> tuple[str, ...]:
|
||||
match node:
|
||||
case str():
|
||||
return (node,)
|
||||
case list():
|
||||
return tuple(leaf for child in node for leaf in string_leaves(child))
|
||||
case dict():
|
||||
return tuple(leaf for child in node.values() for leaf in string_leaves(child))
|
||||
return ()
|
||||
|
||||
|
||||
def field_lines(value: str) -> tuple[str, ...]:
|
||||
try:
|
||||
return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
|
||||
values: Final = frozenset(
|
||||
line.split("=", 1)[1].strip().strip("'")
|
||||
for path in values_files
|
||||
for line in path.read_text().splitlines()
|
||||
if "=" in line
|
||||
)
|
||||
texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
|
||||
renderings: Final = frozenset(
|
||||
rendering
|
||||
for text in texts
|
||||
if len(text) >= MIN_MASKED_LENGTH
|
||||
for rendering in (text, escape(text), escape(text, {'"': """}))
|
||||
)
|
||||
return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
|
||||
|
||||
|
||||
def redact(text: str, values: tuple[str, ...]) -> str:
|
||||
return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
|
||||
|
||||
|
||||
def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
|
||||
target: Final = out_dir / source.name
|
||||
with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
|
||||
_ = handle.write(redact(source.read_text(errors="replace"), values))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser: Final = argparse.ArgumentParser()
|
||||
_ = parser.add_argument("--values", action="append", type=Path, required=True)
|
||||
_ = parser.add_argument("--out", type=Path, required=True)
|
||||
_ = parser.add_argument("files", nargs="*", type=Path)
|
||||
args: Final = parser.parse_args()
|
||||
values_files: Final = tuple(args.values)
|
||||
out_dir: Final[Path] = args.out
|
||||
sources: Final = tuple(args.files)
|
||||
try:
|
||||
values: Final = masked_values(values_files)
|
||||
out_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
for source in sources:
|
||||
write_redacted(source, out_dir, values)
|
||||
except OSError as error:
|
||||
_ = sys.stderr.write(f"could not redact {error.filename}\n")
|
||||
return 1
|
||||
_ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
55
.github/e2e-stack/secrets_to_env.py
vendored
|
|
@ -1,55 +0,0 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
|
||||
ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
MIN_MASKED_LENGTH: Final = 8
|
||||
ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
env_path: Final = Path(sys.argv[1])
|
||||
try:
|
||||
secrets: Final = {
|
||||
key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
|
||||
}
|
||||
except (ValidationError, UnicodeError):
|
||||
_ = sys.stderr.write("expected a JSON object containing string environment values\n")
|
||||
return 1
|
||||
unusable: Final = tuple(
|
||||
key
|
||||
for key, value in secrets.items()
|
||||
if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
|
||||
)
|
||||
if unusable:
|
||||
_ = sys.stderr.write(
|
||||
f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
|
||||
)
|
||||
return 1
|
||||
if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
|
||||
_ = sys.stdout.write(
|
||||
"".join(
|
||||
f"::add-mask::{value.replace('%', '%25')}\n"
|
||||
for value in secrets.values()
|
||||
if len(value) >= MIN_MASKED_LENGTH
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
|
||||
try:
|
||||
with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
|
||||
os.fchmod(handle.fileno(), 0o600)
|
||||
_ = handle.write("\n".join(lines) + "\n")
|
||||
except OSError:
|
||||
_ = sys.stderr.write("could not write the environment file\n")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
52
.github/e2e-stack/select_tests.py
vendored
|
|
@ -1,52 +0,0 @@
|
|||
import re
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
|
||||
UNSUPPORTED: Final = re.compile(
|
||||
r"^tests/e2e/(ui|claude_code|load|migrations)/"
|
||||
r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$"
|
||||
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
|
||||
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
|
||||
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_s3_log_e2e\.py$"
|
||||
r"|^tests/e2e/secret_manager/"
|
||||
)
|
||||
HARNESS: Final = re.compile(
|
||||
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
||||
r"|^tests/e2e/idp_realm\.json$"
|
||||
r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
|
||||
r"|^tests/e2e/coverage_registry/management_cases\.py$"
|
||||
r"|^tests/e2e/gateway/"
|
||||
r"|^\.github/e2e-stack/"
|
||||
r"|^\.github/workflows/test-e2e-changed\.yml$"
|
||||
)
|
||||
UNEXPANDED: Final = re.compile(r"[*?\[]")
|
||||
|
||||
|
||||
def is_selectable(path: str) -> bool:
|
||||
return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
|
||||
|
||||
|
||||
def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
|
||||
direct: Final = frozenset(path for path in changed if is_selectable(path))
|
||||
harness_changed: Final = any(HARNESS.match(path) for path in changed)
|
||||
canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
|
||||
return tuple(sorted(direct | canary_tests))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
canary: Final = tuple(sys.argv[1:])
|
||||
unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
|
||||
if unexpanded:
|
||||
_ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
|
||||
return 1
|
||||
changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
|
||||
_ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
231
.github/e2e-stack/up.sh
vendored
|
|
@ -1,231 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
|
||||
CERTS_DIR="${STACK_DIR}/certs"
|
||||
LOGS_DIR="${STACK_DIR}/logs"
|
||||
PIDS_DIR="${STACK_DIR}/pids"
|
||||
|
||||
POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
|
||||
VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
|
||||
JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
|
||||
NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
|
||||
|
||||
LB_PORT="${E2E_LB_PORT:-4000}"
|
||||
GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
|
||||
GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
|
||||
BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
|
||||
REDIS_PORT="${E2E_REDIS_PORT:-6379}"
|
||||
DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
|
||||
DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
|
||||
DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
|
||||
DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
|
||||
DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
|
||||
JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
|
||||
JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
|
||||
JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
|
||||
KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
|
||||
|
||||
MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
|
||||
|
||||
mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
|
||||
chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
|
||||
chmod 755 "${CERTS_DIR}"
|
||||
|
||||
log() { printf 'e2e-stack: %s\n' "$*"; }
|
||||
|
||||
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
|
||||
|
||||
wait_for() {
|
||||
local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
|
||||
until eval "${check}"; do
|
||||
if ((SECONDS >= deadline)); then
|
||||
log "timed out waiting for ${label}"
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
log "${label} is up"
|
||||
}
|
||||
|
||||
if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
|
||||
set -a
|
||||
source "${REPO_ROOT}/tests/e2e/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [[ -z "${DD_API_KEY:-}" ]]; then
|
||||
log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
|
||||
exit 1
|
||||
fi
|
||||
export DD_SITE="${DD_SITE:-datadoghq.com}"
|
||||
|
||||
if ! port_open "${DATABASE_PORT}"; then
|
||||
docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
|
||||
-e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
|
||||
"${POSTGRES_IMAGE}" >/dev/null
|
||||
fi
|
||||
wait_for "postgres" "port_open ${DATABASE_PORT}"
|
||||
|
||||
if ! port_open "${JAEGER_QUERY_PORT}"; then
|
||||
docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
|
||||
"${JAEGER_IMAGE}" >/dev/null
|
||||
fi
|
||||
wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
|
||||
|
||||
openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
|
||||
openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
|
||||
-subj "/CN=litellm-e2e-ca" \
|
||||
-addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
|
||||
-out "${CERTS_DIR}/ca.crt" 2>/dev/null
|
||||
openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
|
||||
openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
|
||||
openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
|
||||
-CAcreateserial -days 7 -sha256 \
|
||||
-extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
|
||||
-out "${CERTS_DIR}/server.crt" 2>/dev/null
|
||||
chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
|
||||
|
||||
CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
|
||||
cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
|
||||
|
||||
docker rm -f e2e-valkey >/dev/null 2>&1 || true
|
||||
docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
|
||||
"${VALKEY_IMAGE}" valkey-server \
|
||||
--cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
|
||||
--tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
|
||||
--tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
|
||||
VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
|
||||
wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
|
||||
${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
|
||||
wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
|
||||
|
||||
CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
|
||||
CONFIG_PATH="${CONFIG_SOURCE}"
|
||||
if [[ "${REDIS_PORT}" != "6379" ]]; then
|
||||
CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
|
||||
sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
|
||||
fi
|
||||
|
||||
SERVER_ENV=(
|
||||
"LITELLM_MASTER_KEY=${MASTER_KEY}"
|
||||
"DATABASE_HOST=${DATABASE_HOST}"
|
||||
"DATABASE_PORT=${DATABASE_PORT}"
|
||||
"DATABASE_USER=${DATABASE_USER}"
|
||||
"DATABASE_PASSWORD=${DATABASE_PASSWORD}"
|
||||
"DATABASE_NAME=${DATABASE_NAME}"
|
||||
"DISABLE_SCHEMA_UPDATE=true"
|
||||
"REDIS_HOST=127.0.0.1"
|
||||
"REDIS_PORT=${REDIS_PORT}"
|
||||
"REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
|
||||
"CONFIG_FILE_PATH=${CONFIG_PATH}"
|
||||
"STORE_MODEL_IN_DB=True"
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
|
||||
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
|
||||
"PYTHONPATH=${REPO_ROOT}"
|
||||
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
|
||||
"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
|
||||
"JWT_AUDIENCE=litellm-e2e"
|
||||
)
|
||||
if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
|
||||
printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
|
||||
SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh
|
||||
|
||||
log "running migrations"
|
||||
env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
|
||||
|
||||
start_server() {
|
||||
local name="$1"; shift
|
||||
env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
|
||||
echo $! > "${PIDS_DIR}/${name}.pid"
|
||||
}
|
||||
|
||||
if [[ "$(uname)" == "Linux" ]]; then
|
||||
NGINX_UPSTREAM_HOST=127.0.0.1
|
||||
NGINX_DOCKER_ARGS=(--network host)
|
||||
else
|
||||
NGINX_UPSTREAM_HOST=host.docker.internal
|
||||
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
|
||||
fi
|
||||
|
||||
cat > "${STACK_DIR}/nginx.conf" <<EOF
|
||||
events {}
|
||||
http {
|
||||
map \$http_upgrade \$connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
upstream litellm_gateways {
|
||||
server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
|
||||
server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
|
||||
}
|
||||
server {
|
||||
listen ${LB_PORT};
|
||||
client_max_body_size 100m;
|
||||
location / {
|
||||
proxy_pass http://litellm_gateways;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host \$host;
|
||||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto \$scheme;
|
||||
proxy_set_header Upgrade \$http_upgrade;
|
||||
proxy_set_header Connection \$connection_upgrade;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 600s;
|
||||
proxy_send_timeout 600s;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen ${JAEGER_OTLP_TLS_PORT} ssl;
|
||||
ssl_certificate /certs/server.crt;
|
||||
ssl_certificate_key /certs/server.key;
|
||||
client_max_body_size 100m;
|
||||
location / {
|
||||
proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
docker rm -f e2e-nginx >/dev/null 2>&1 || true
|
||||
docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
|
||||
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
|
||||
-v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
|
||||
|
||||
wait_for "Jaeger OTLP TLS listener" \
|
||||
"curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
|
||||
|
||||
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
|
||||
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
|
||||
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
|
||||
|
||||
wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
|
||||
wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
|
||||
wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
|
||||
wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
|
||||
|
||||
cat > "${STACK_DIR}/stack.env" <<EOF
|
||||
LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
|
||||
LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
|
||||
LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
|
||||
LITELLM_MASTER_KEY=${MASTER_KEY}
|
||||
REDIS_HOST=127.0.0.1
|
||||
REDIS_PORT=${REDIS_PORT}
|
||||
E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
|
||||
E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
|
||||
E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
|
||||
E2E_KEYCLOAK_ADMIN_USER=admin
|
||||
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
|
||||
SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
|
||||
DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
|
||||
EOF
|
||||
|
||||
log "stack is up; pytest env written to ${STACK_DIR}/stack.env"
|
||||
140
.github/scripts/assert_ci_coverage.py
vendored
|
|
@ -9,7 +9,6 @@ import sys
|
|||
import warnings
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import yaml
|
||||
|
|
@ -22,11 +21,12 @@ TESTS_ROOT = REPO_ROOT / "tests"
|
|||
|
||||
ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
|
||||
PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
|
||||
TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
|
||||
TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"})
|
||||
DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
|
||||
TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
|
||||
IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
|
||||
TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
|
||||
IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
|
||||
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
|
||||
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
|
||||
GLOB_CHARS = frozenset("*?")
|
||||
|
|
@ -72,6 +72,17 @@ class Scalar:
|
|||
value: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Selection:
|
||||
included: frozenset[str]
|
||||
ignored: frozenset[str]
|
||||
|
||||
def covers(self, relative_path: str) -> bool:
|
||||
return any(_token_covers(token, relative_path) for token in self.included) and not any(
|
||||
_token_covers(token, relative_path) for token in self.ignored
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Finding:
|
||||
subject: str
|
||||
|
|
@ -111,57 +122,32 @@ def _uncommented(value: str) -> str:
|
|||
return COMMENT_RE.sub("", value)
|
||||
|
||||
|
||||
def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
match.group(0).rstrip("/")
|
||||
def _selection_for_scalar(scalar: Scalar) -> Selection:
|
||||
text: Final = _uncommented(scalar.value)
|
||||
ignored: Final = frozenset().union(
|
||||
*(
|
||||
frozenset(token.rstrip("/") for token in TEST_TOKEN_RE.findall(ignored_argument))
|
||||
for ignored_argument in IGNORE_ARG_RE.findall(text)
|
||||
)
|
||||
)
|
||||
included: Final = frozenset(
|
||||
match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(IGNORE_ARG_RE.sub("", text))
|
||||
)
|
||||
return Selection(included=included, ignored=ignored)
|
||||
|
||||
|
||||
def _invoked_selections(scalars: Iterable[Scalar]) -> tuple[Selection, ...]:
|
||||
selected_scalars: Final = tuple(
|
||||
scalar
|
||||
for scalar in scalars
|
||||
if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
|
||||
for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
|
||||
)
|
||||
selections: Final = tuple(_selection_for_scalar(scalar) for scalar in selected_scalars)
|
||||
return tuple(selection for selection in selections if selection.included)
|
||||
|
||||
|
||||
SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;")
|
||||
|
||||
|
||||
def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]:
|
||||
script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
|
||||
if not script.is_file():
|
||||
return MappingProxyType({})
|
||||
text: Final = _uncommented(script.read_text())
|
||||
return MappingProxyType(
|
||||
{
|
||||
label: frozenset(
|
||||
match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)
|
||||
)
|
||||
for label, body in SELECTION_ARM_RE.findall(text)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
|
||||
return frozenset(
|
||||
token for tokens in _unit_selection_arms(repo_root).values() for token in tokens
|
||||
)
|
||||
|
||||
|
||||
def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
scalar.value
|
||||
for scalar in scalars
|
||||
if scalar.key == "unit-flag" and "${{" not in scalar.value
|
||||
)
|
||||
|
||||
|
||||
def _shard_tokens(
|
||||
scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]
|
||||
) -> frozenset[str]:
|
||||
wired: Final = _wired_unit_flags(scalars)
|
||||
return _invoked_test_tokens(scalars) | frozenset(
|
||||
token
|
||||
for label, tokens in arms.items()
|
||||
if label in wired
|
||||
for token in tokens
|
||||
)
|
||||
def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
|
||||
return frozenset().union(*(selection.included for selection in _invoked_selections(scalars)))
|
||||
|
||||
|
||||
def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
|
||||
|
|
@ -224,11 +210,12 @@ def _dockerfiles() -> tuple[str, ...]:
|
|||
)
|
||||
|
||||
|
||||
def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
|
||||
def _uncovered_tests(allowlist: Allowlist, selections: tuple[Selection, ...]) -> tuple[Finding, ...]:
|
||||
uncovered = tuple(
|
||||
relative_path
|
||||
for relative_path in _test_files()
|
||||
if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
|
||||
if not any(selection.covers(relative_path) for selection in selections)
|
||||
and not allowlist.covers_test(relative_path)
|
||||
)
|
||||
directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
|
||||
return tuple(
|
||||
|
|
@ -518,7 +505,7 @@ def _check_slices() -> int:
|
|||
|
||||
|
||||
def _check_shards() -> int:
|
||||
findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms()))
|
||||
findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
|
||||
if findings:
|
||||
_report(
|
||||
"test directories and files that no shard claims",
|
||||
|
|
@ -544,23 +531,46 @@ def _integration_groups(runner: pathlib.Path) -> dict[str, tuple[str, ...]]:
|
|||
return {group: tuple(folders) for group, folders in ast.literal_eval(mapping).items()}
|
||||
|
||||
|
||||
def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
|
||||
module: Final = ast.parse(runner.read_text())
|
||||
literal: Final = next(
|
||||
(
|
||||
node.value
|
||||
for node in module.body
|
||||
if isinstance(node, ast.AnnAssign)
|
||||
and isinstance(node.target, ast.Name)
|
||||
and node.target.id == "GITHUB_FILES"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if literal is None:
|
||||
return frozenset()
|
||||
values: Final = literal.args[0] if isinstance(literal, ast.Call) else literal
|
||||
return frozenset(ast.literal_eval(values))
|
||||
|
||||
|
||||
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
|
||||
runner: Final = repo_root / "tests/integration/run.py"
|
||||
if not runner.exists():
|
||||
return frozenset(), ()
|
||||
groups: Final = _integration_groups(runner)
|
||||
github_files: Final = _integration_github_files(runner)
|
||||
integration_root: Final = repo_root / "tests/integration"
|
||||
paths: Final = frozenset(
|
||||
str(path.relative_to(repo_root))
|
||||
for folders in groups.values()
|
||||
for folder in folders
|
||||
for path in (integration_root / folder).rglob("test_*.py")
|
||||
if str(path.relative_to(repo_root)) not in github_files
|
||||
)
|
||||
browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
|
||||
browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()
|
||||
browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes)
|
||||
circle_path: Final = repo_root / ".circleci/config.yml"
|
||||
circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
|
||||
circle_test_path_tokens: Final = _invoked_test_tokens(
|
||||
scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path"
|
||||
)
|
||||
steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
|
||||
invoked: Final = any(
|
||||
".circleci/scripts/run_integration.sh" in scalar.value
|
||||
|
|
@ -595,10 +605,22 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
for path in (repo_root / ".github/workflows").glob("*.y*ml")
|
||||
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
|
||||
)
|
||||
findings: Final = tuple(
|
||||
Finding(path, "integration contract is also selected by GitHub Actions")
|
||||
for path in paths
|
||||
if any(_token_covers(token, path) for token in gha_tokens)
|
||||
findings: Final = (
|
||||
tuple(
|
||||
Finding(path, "integration contract is also selected by GitHub Actions")
|
||||
for path in paths
|
||||
if any(_token_covers(token, path) for token in gha_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration contract has no invoking job")
|
||||
for path in sorted(github_files)
|
||||
if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration file is missing")
|
||||
for path in sorted(github_files)
|
||||
if not (repo_root / path).is_file()
|
||||
)
|
||||
)
|
||||
browser_commands: Final = tuple(
|
||||
scalar.value
|
||||
|
|
@ -642,7 +664,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
return frozenset(), findings + (
|
||||
Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
|
||||
)
|
||||
return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings
|
||||
return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
|
@ -656,7 +678,11 @@ def main() -> int:
|
|||
|
||||
integration_paths, ownership_findings = _integration_ownership()
|
||||
test_findings = (
|
||||
_uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
|
||||
_uncovered_tests(
|
||||
allowlist,
|
||||
_invoked_selections(scalars)
|
||||
+ (Selection(included=integration_paths, ignored=frozenset()),),
|
||||
)
|
||||
+ ownership_findings
|
||||
)
|
||||
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
|
||||
|
|
|
|||
2
.github/scripts/verify_linux_native_wheel.py
vendored
|
|
@ -214,7 +214,7 @@ def main(
|
|||
native_module: Final = load_native_module(native_path)
|
||||
native_module_loads: Final = native_module is not None
|
||||
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
|
||||
native_size_limit: Final = 45_000_000
|
||||
native_size_limit: Final = 48_000_000
|
||||
native_size_within_limit: Final = native_member.file_size <= native_size_limit
|
||||
validations: Final = (
|
||||
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
|
||||
|
|
|
|||
29
.github/workflows/_test-unit-base.yml
vendored
|
|
@ -13,16 +13,6 @@ on:
|
|||
have its path existence-checked like any other token.
|
||||
required: true
|
||||
type: string
|
||||
unit-flag:
|
||||
description: >-
|
||||
Codecov flag of the `.circleci/tests.yml` job that now owns part of
|
||||
this shard. The shard also runs the files
|
||||
`.circleci/scripts/unit_selection.sh` lists for the flag, on every
|
||||
event, because the CircleCI pipeline is manual-only while the tests
|
||||
migrate.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
workers:
|
||||
description: "Number of pytest-xdist workers"
|
||||
required: false
|
||||
|
|
@ -79,12 +69,6 @@ on:
|
|||
description: "Unique name for the coverage artifact (must be unique per run)"
|
||||
required: true
|
||||
type: string
|
||||
legacy-mcp-peer:
|
||||
description: "Install the isolated SDK1 peer for MCP compatibility tests"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
|
@ -142,17 +126,10 @@ jobs:
|
|||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
LEGACY_MCP_PEER: ${{ inputs.legacy-mcp-peer }}
|
||||
run: |
|
||||
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
|
||||
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
|
||||
if [ "$LEGACY_MCP_PEER" = "true" ]; then
|
||||
uv venv --python "${UV_PYTHON}" .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "MCP_TEST_PEER_PYTHON=$GITHUB_WORKSPACE/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -171,7 +148,6 @@ jobs:
|
|||
timeout-minutes: ${{ inputs.timeout-minutes }}
|
||||
env:
|
||||
TEST_PATH: ${{ inputs.test-path }}
|
||||
UNIT_FLAG: ${{ inputs.unit-flag }}
|
||||
MAX_FAILURES: ${{ inputs.max-failures }}
|
||||
WORKERS: ${{ inputs.workers }}
|
||||
RERUNS: ${{ inputs.reruns }}
|
||||
|
|
@ -181,9 +157,6 @@ jobs:
|
|||
run: |
|
||||
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
|
||||
selection="${TEST_PATH}"
|
||||
if [ -n "${UNIT_FLAG}" ]; then
|
||||
selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')"
|
||||
fi
|
||||
if [ -z "${selection// /}" ]; then
|
||||
echo "shard selection is empty; nothing to run"
|
||||
exit 0
|
||||
|
|
|
|||
84
.github/workflows/image-scan.yml
vendored
|
|
@ -15,6 +15,9 @@ on:
|
|||
- gateway/main.py
|
||||
- backend/Dockerfile
|
||||
- backend/main.py
|
||||
- deploy/lens/**
|
||||
- litellm/proxy/lens/**
|
||||
- tests/e2e/migrations/lens_compose_smoke.sh
|
||||
- docker/component_entrypoint.sh
|
||||
- docker/entrypoint.sh
|
||||
- litellm/proxy/prisma_migration.py
|
||||
|
|
@ -37,6 +40,80 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lens-worker-image:
|
||||
name: lens-worker-image (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
if: >-
|
||||
github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: ubuntu-latest
|
||||
grype_sha256: edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343
|
||||
- arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
grype_sha256: 553e4c36d9d61349830ba6034d43b8700a7f10576d3e2f4981c0fd2b96086465
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Build the release worker
|
||||
env:
|
||||
RELEASE_TAG: sha-${{ github.sha }}
|
||||
run: docker build --build-arg LITELLM_RELEASE_TAG="${RELEASE_TAG}" -f deploy/lens/Dockerfile -t lens-worker-scan .
|
||||
- name: Verify the standalone worker on a read-only filesystem
|
||||
env:
|
||||
RELEASE_TAG: sha-${{ github.sha }}
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
|
||||
-e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c '
|
||||
import os
|
||||
import lens.worker
|
||||
from lens.release import release_tag
|
||||
from lens.trace_store import trace_store
|
||||
assert os.getuid() == 65532
|
||||
assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"]
|
||||
with trace_store() as store:
|
||||
assert store.count() == 0
|
||||
'
|
||||
- name: Reject a dependency whose hash has changed
|
||||
run: |
|
||||
docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps .
|
||||
sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \
|
||||
deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock"
|
||||
if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \
|
||||
--entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \
|
||||
--require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \
|
||||
> "$RUNNER_TEMP/hash-check.log" 2>&1; then
|
||||
echo "::error::Dependency hash mismatch was accepted"
|
||||
exit 1
|
||||
fi
|
||||
cat "$RUNNER_TEMP/hash-check.log"
|
||||
grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log"
|
||||
- name: Download Grype v0.114.0
|
||||
env:
|
||||
ARCH: ${{ matrix.arch }}
|
||||
GRYPE_SHA256: ${{ matrix.grype_sha256 }}
|
||||
run: |
|
||||
curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
|
||||
"https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_${ARCH}.tar.gz"
|
||||
echo "${GRYPE_SHA256} $RUNNER_TEMP/grype.tar.gz" | sha256sum -c -
|
||||
tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
|
||||
chmod +x "$RUNNER_TEMP/grype"
|
||||
- name: Scan the worker for fixable HIGH/CRITICAL CVEs
|
||||
env:
|
||||
GRYPE_MATCH_PYTHON_USING_CPES: "true"
|
||||
run: |
|
||||
"$RUNNER_TEMP/grype" lens-worker-scan \
|
||||
--config .grype.yaml --only-fixed --fail-on high --output table
|
||||
|
||||
image-scan:
|
||||
name: image-scan
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -113,7 +190,7 @@ jobs:
|
|||
persist-credentials: false
|
||||
|
||||
- name: Build runtime image
|
||||
run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
|
||||
run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
|
|
@ -127,6 +204,11 @@ jobs:
|
|||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
|
||||
|
||||
- name: Verify the bundled Lens Compose installation and restart
|
||||
env:
|
||||
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
|
||||
run: bash tests/e2e/migrations/lens_compose_smoke.sh
|
||||
|
||||
migrations-image:
|
||||
name: migrations-image
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
13
.github/workflows/lens-worker.yml
vendored
|
|
@ -34,7 +34,14 @@ jobs:
|
|||
with:
|
||||
persist-credentials: false
|
||||
- name: Build Lens worker
|
||||
run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
|
||||
run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
|
||||
- name: Reject custom builds without a matching release tag
|
||||
run: |
|
||||
if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
|
||||
echo "::error::An unversioned worker build unexpectedly succeeded"
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
|
||||
- name: Verify standalone imports with a read-only filesystem
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
|
||||
|
|
@ -54,11 +61,11 @@ jobs:
|
|||
-v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
|
||||
--entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
|
||||
- name: Publish versioned Lens worker
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }}
|
||||
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}
|
||||
run: |
|
||||
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
|
||||
docker tag lens-worker:${{ github.sha }} "$IMAGE"
|
||||
|
|
|
|||
264
.github/workflows/test-e2e-changed.yml
vendored
|
|
@ -1,264 +0,0 @@
|
|||
name: e2e-changed-tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: e2e-changed-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
detect:
|
||||
name: Detect changed e2e tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
tests: ${{ steps.changed.outputs.tests }}
|
||||
any: ${{ steps.changed.outputs.any }}
|
||||
steps:
|
||||
- name: Checkout the selector and the canary suite
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
sparse-checkout: |
|
||||
.github/e2e-stack
|
||||
tests/e2e/access_control
|
||||
tests/e2e/management/test_jwt_management_e2e.py
|
||||
tests/e2e/other/test_jwt_auth_e2e.py
|
||||
persist-credentials: false
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
- name: List the e2e test files this PR added or modified
|
||||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
|
||||
--jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
|
||||
| grep -Fxq "${HEAD_SHA}"
|
||||
files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
|
||||
--jq '.[] | select(.status != "removed") | .filename')"
|
||||
gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
|
||||
tests="$(printf '%s\n' "${files}" \
|
||||
| python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \
|
||||
tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)"
|
||||
echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
|
||||
if [ -n "${tests}" ]; then
|
||||
echo "any=true" >> "${GITHUB_OUTPUT}"
|
||||
echo "selected e2e tests: ${tests}"
|
||||
else
|
||||
echo "any=false" >> "${GITHUB_OUTPUT}"
|
||||
echo "no changed e2e test files supported by this stack; nothing to run"
|
||||
fi
|
||||
|
||||
run:
|
||||
name: Run changed e2e tests against the stage-mirror stack
|
||||
needs: detect
|
||||
if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
environment: e2e-changed
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6
|
||||
env:
|
||||
POSTGRES_USER: litellm
|
||||
POSTGRES_PASSWORD: dbpassword9090
|
||||
POSTGRES_DB: litellm
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U litellm"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
jaeger:
|
||||
image: jaegertracing/jaeger:2.10.0
|
||||
ports:
|
||||
- 4318:4318
|
||||
- 16686:16686
|
||||
steps:
|
||||
- name: Validate configuration
|
||||
env:
|
||||
ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
|
||||
run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen \
|
||||
--extra proxy --extra proxy-runtime --extra extra_proxy \
|
||||
--extra semantic-router --extra bedrock-realtime \
|
||||
--group ci --group proxy-dev --group e2e-dev
|
||||
uv pip install "pipecat-ai[openai]==1.4.0"
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Install Playwright chromium
|
||||
run: uv run --no-sync playwright install --with-deps chromium
|
||||
|
||||
- name: Configure AWS credentials
|
||||
id: aws
|
||||
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
|
||||
with:
|
||||
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
|
||||
aws-region: us-east-1
|
||||
role-session-name: litellm-e2e-changed-${{ github.run_id }}
|
||||
role-duration-seconds: 900
|
||||
output-env-credentials: false
|
||||
output-credentials: true
|
||||
|
||||
- name: Fetch provider credentials from AWS Secrets Manager
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
|
||||
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
run: |
|
||||
umask 077
|
||||
aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
|
||||
--query SecretString --output text \
|
||||
| uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
|
||||
aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
|
||||
--query SecretString --output text \
|
||||
| jq -R -s '{"LITELLM_LICENSE": .}' \
|
||||
| uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
|
||||
|
||||
- name: Boot the stage-mirror stack
|
||||
id: boot
|
||||
run: |
|
||||
umask 077
|
||||
if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
|
||||
echo "::error::stage-mirror stack failed to boot; raw logs are not published"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Export stack environment
|
||||
run: |
|
||||
master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
|
||||
echo "::add-mask::${master_key}"
|
||||
cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Run the selected tests three times
|
||||
env:
|
||||
TESTS: ${{ needs.detect.outputs.tests }}
|
||||
E2E_FIXTURE_MODE: live
|
||||
E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
|
||||
COLUMNS: '400'
|
||||
run: |
|
||||
umask 077
|
||||
read -r -a test_files <<< "${TESTS}"
|
||||
for pass in 1 2 3; do
|
||||
report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
|
||||
log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
|
||||
echo "::group::pass ${pass} of 3"
|
||||
set +e
|
||||
uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
|
||||
-o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
|
||||
status=$?
|
||||
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
|
||||
verified=$?
|
||||
set -e
|
||||
grep -E '^(FAILED|ERROR) ' "${log}" || true
|
||||
grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
|
||||
echo "::endgroup::"
|
||||
if [ "${status}" = "5" ]; then
|
||||
echo "::error::the selected files collected no runnable tests, so nothing was verified"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${status}" != "0" ]; then
|
||||
echo "::error::pass ${pass} of 3 failed with exit code ${status}"
|
||||
exit "${status}"
|
||||
fi
|
||||
if [ "${verified}" != "0" ]; then
|
||||
echo "::error::pass ${pass} of 3 did not verify every selected file"
|
||||
exit 1
|
||||
fi
|
||||
echo "pass ${pass} of 3 passed"
|
||||
done
|
||||
|
||||
- name: Redact the pytest output
|
||||
if: always() && steps.boot.outcome == 'success'
|
||||
run: |
|
||||
umask 077
|
||||
shopt -s nullglob
|
||||
uv run --no-sync python .github/e2e-stack/redact_output.py \
|
||||
--values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
|
||||
--out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
|
||||
|
||||
- name: Keep the redacted pytest output
|
||||
if: always() && steps.boot.outcome == 'success'
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: e2e-changed-pytest-output-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/e2e-redacted
|
||||
retention-days: 14
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Stop the stack
|
||||
if: always() && steps.boot.outcome != 'skipped'
|
||||
run: bash .github/e2e-stack/down.sh
|
||||
|
||||
- name: Remove credentials and raw output
|
||||
if: always()
|
||||
run: |
|
||||
rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
|
||||
rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
|
||||
|
||||
gate:
|
||||
name: e2e-changed-tests
|
||||
needs: [detect, run]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Require three successful passes when tests changed
|
||||
env:
|
||||
DETECT_RESULT: ${{ needs.detect.result }}
|
||||
ANY_TESTS: ${{ needs.detect.outputs.any }}
|
||||
RUN_RESULT: ${{ needs.run.result }}
|
||||
run: |
|
||||
if [ "${DETECT_RESULT}" != "success" ]; then
|
||||
echo "::error::changed-test detection did not succeed"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${ANY_TESTS}" = "false" ]; then
|
||||
echo "no changed e2e test files supported by this stack; nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
|
||||
echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
|
||||
exit 1
|
||||
fi
|
||||
4
.github/workflows/test-litellm-ui-unit.yml
vendored
|
|
@ -49,6 +49,10 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: npm ci
|
||||
|
||||
- name: Check UI production source types
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Run UI type tests (Vitest)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,27 @@ on:
|
|||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- "**/pyproject.toml"
|
||||
- "uv.lock"
|
||||
- "uv.toml"
|
||||
- ".python-version"
|
||||
- "rust-toolchain.toml"
|
||||
- "litellm-rust/**"
|
||||
- "litellm/__init__.py"
|
||||
- "litellm/proxy/proxy_server.py"
|
||||
- "litellm/**/*mcp*"
|
||||
- "litellm/**/*mcp*/**"
|
||||
- "litellm/integrations/arize/**"
|
||||
- "scripts/check_mcp_sdk_install.py"
|
||||
- "tests/base_sdk_tests/**"
|
||||
- ".github/workflows/test-mcp-dependency-resolution.yml"
|
||||
- ".github/actions/detect-changes/**"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/detect_changes.sh"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
- ".circleci/scripts/classify_changes.sh"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -17,7 +38,7 @@ concurrency:
|
|||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
|
|||
167
.github/workflows/test-postgres.yml
vendored
|
|
@ -1,167 +0,0 @@
|
|||
name: "Postgres Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
postgres:
|
||||
name: ${{ matrix.shard }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: ${{ matrix.job-timeout-minutes }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: litellm_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- shard: proxy-behavior
|
||||
test-path: "tests/proxy_behavior"
|
||||
seed: db-push
|
||||
workers: 0
|
||||
timeout-minutes: 25
|
||||
job-timeout-minutes: 50
|
||||
|
||||
- shard: proxy-security
|
||||
test-path: "tests/proxy_security_tests"
|
||||
seed: db-push
|
||||
workers: 0
|
||||
timeout-minutes: 15
|
||||
job-timeout-minutes: 40
|
||||
|
||||
- shard: schema-migration
|
||||
test-path: "tests/proxy_migration_tests"
|
||||
seed: none
|
||||
workers: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 45
|
||||
|
||||
env:
|
||||
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
timeout-minutes: 3
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
timeout-minutes: 2
|
||||
uses: ./.github/actions/detect-changes
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
|
||||
timeout-minutes: 5
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-postgres-
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
|
||||
timeout-minutes: 5
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-postgres-
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 12
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
run: |
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Seed database schema
|
||||
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
|
||||
- name: Run tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: ${{ matrix.timeout-minutes }}
|
||||
env:
|
||||
TEST_PATH: ${{ matrix.test-path }}
|
||||
WORKERS: ${{ matrix.workers }}
|
||||
PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }}
|
||||
run: |
|
||||
if [ "${WORKERS}" = "0" ]; then
|
||||
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
|
||||
else
|
||||
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
|
||||
fi
|
||||
|
||||
- name: Upload Lens database coverage
|
||||
if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
use_oidc: true
|
||||
version: v11.3.1
|
||||
root_dir: ${{ github.workspace }}
|
||||
files: coverage-lens-postgres.xml
|
||||
flags: lens-postgres
|
||||
fail_ci_if_error: true
|
||||
106
.github/workflows/test-redis-compat.yml
vendored
|
|
@ -1,106 +0,0 @@
|
|||
name: "Unit Tests: Redis Client Version Compatibility"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- "litellm/_redis.py"
|
||||
- "litellm/_redis_credential_provider.py"
|
||||
- "litellm/caching/redis_cache.py"
|
||||
- "litellm/caching/evicted_client_closer.py"
|
||||
- "tests/unit/test_redis.py"
|
||||
- "tests/local_testing/test_caching.py"
|
||||
- "tests/unit/caching/test_redis_connection_pool.py"
|
||||
- "tests/unit/caching/test_redis_cluster_cache.py"
|
||||
- "tests/unit/caching/test_evicted_client_closer.py"
|
||||
- ".github/workflows/test-redis-compat.yml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
redis-compat:
|
||||
name: "redis-py ${{ matrix.redis-version }}"
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
|
||||
# newer legs prove the inspect.signature introspection in litellm/_redis.py
|
||||
# keeps extracting kwargs on the redis-py releases people actually run now.
|
||||
# Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
|
||||
# specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
|
||||
# for the 6.x line.
|
||||
redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router
|
||||
|
||||
- name: Pin redis-py to the matrix version
|
||||
env:
|
||||
REDIS_VERSION: ${{ matrix.redis-version }}
|
||||
run: |
|
||||
uv pip install "redis==${REDIS_VERSION:?}"
|
||||
uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
|
||||
|
||||
- name: Build Redis for cluster authentication tests
|
||||
run: |
|
||||
curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz"
|
||||
echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP"
|
||||
make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server
|
||||
echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Run redis unit tests
|
||||
run: |
|
||||
redis-server --version
|
||||
uv run --no-sync pytest \
|
||||
tests/unit/test_redis.py \
|
||||
tests/unit/caching/test_redis_connection_pool.py \
|
||||
tests/unit/caching/test_redis_cluster_cache.py \
|
||||
tests/unit/caching/test_evicted_client_closer.py \
|
||||
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
|
||||
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
|
||||
--tb=short -vv \
|
||||
--reruns 2 \
|
||||
--reruns-delay 1 \
|
||||
--durations=20 \
|
||||
--cov=./litellm --cov-report=xml:coverage-redis.xml
|
||||
|
||||
- name: Upload Redis coverage
|
||||
if: matrix.redis-version == '5.3.1'
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
files: coverage-redis.xml
|
||||
flags: redis-compat
|
||||
fail_ci_if_error: false
|
||||
14
.github/workflows/test-rust.yml
vendored
|
|
@ -5,6 +5,8 @@ on:
|
|||
paths:
|
||||
- "litellm-rust/**"
|
||||
- "litellm/rust_bridge/**"
|
||||
- "scripts/generate_trace_types.py"
|
||||
- "scripts/trace_codegen/**"
|
||||
- "tests/test_litellm_rust/**"
|
||||
- "litellm/integrations/custom_logger.py"
|
||||
- "litellm/litellm_core_utils/litellm_logging.py"
|
||||
|
|
@ -32,6 +34,8 @@ on:
|
|||
paths:
|
||||
- "litellm-rust/**"
|
||||
- "litellm/rust_bridge/**"
|
||||
- "scripts/generate_trace_types.py"
|
||||
- "scripts/trace_codegen/**"
|
||||
- "tests/test_litellm_rust/**"
|
||||
- "litellm/integrations/custom_logger.py"
|
||||
- "litellm/litellm_core_utils/litellm_logging.py"
|
||||
|
|
@ -85,11 +89,11 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
- run: cargo clippy --workspace --all-targets --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema -- -D warnings
|
||||
|
||||
rust-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 30
|
||||
defaults:
|
||||
run:
|
||||
working-directory: litellm-rust
|
||||
|
|
@ -124,7 +128,11 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- run: cargo nextest run --workspace --locked
|
||||
- name: Check generated trace contracts
|
||||
working-directory: .
|
||||
run: uv run scripts/generate_trace_types.py --check
|
||||
|
||||
- run: cargo nextest run --workspace --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema
|
||||
|
||||
- run: cargo test --workspace --doc --locked
|
||||
|
||||
|
|
|
|||
100
.github/workflows/test-unit-proxy-db.yml
vendored
|
|
@ -20,13 +20,6 @@ concurrency:
|
|||
# rather than alphabetical letter ranges. Adding a new test file means adding it
|
||||
# to whichever group it belongs to, not reshuffling slices.
|
||||
#
|
||||
# `.circleci/tests.yml` runs each group's files on same-repo events under the
|
||||
# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
|
||||
# the file lists. That pipeline is manual-only while the tests migrate, so
|
||||
# `unit-flag` makes the shard run that list on every event. `test-path` keeps
|
||||
# the files that still reach real providers and never left
|
||||
# tests/proxy_unit_tests.
|
||||
#
|
||||
# Design targets:
|
||||
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
|
||||
# Most of a shard's time is pytest plugin load + xdist worker imports +
|
||||
|
|
@ -78,93 +71,136 @@ jobs:
|
|||
include:
|
||||
# Must run serially — event-loop conflict with the logging worker.
|
||||
- test-group: key-generation
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-key-generation
|
||||
test-path: >-
|
||||
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
|
||||
workers: 0
|
||||
dist: loadscope
|
||||
timeout: 20
|
||||
|
||||
# ---- auth: split into 2 shards ----
|
||||
- test-group: auth-checks
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-auth-checks
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_auth_checks.py
|
||||
tests/unit/proxy/auth/test_user_api_key_auth.py
|
||||
tests/unit/proxy/test_credential_slot_registry.py
|
||||
tests/unit/proxy/test_deprecated_key_grace_period.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: jwt-and-keys
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-jwt-and-keys
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_jwt.py
|
||||
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
|
||||
tests/unit/proxy/test_proxy_custom_auth.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
|
||||
- test-group: proxy-utils
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-proxy-utils
|
||||
test-path: >-
|
||||
tests/unit/proxy/test_proxy_utils.py
|
||||
workers: 4
|
||||
dist: worksteal
|
||||
timeout: 15
|
||||
|
||||
# ---- proxy server: split into 2 shards ----
|
||||
- test-group: proxy-server-core
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
|
||||
unit-flag: proxy-db-proxy-server-core
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
|
||||
tests/unit/proxy/test__lazy_features.py
|
||||
tests/unit/proxy/test_aproxy_startup.py
|
||||
tests/unit/proxy/test_proxy_server.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: proxy-runtime
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-proxy-runtime
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_multipart_bypass_repro.py
|
||||
tests/unit/proxy/auth/test_proxy_routes.py
|
||||
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
|
||||
tests/unit/proxy/test_proxy_config_unit_test.py
|
||||
tests/unit/proxy/test_proxy_token_counter.py
|
||||
tests/unit/proxy/test_server_root_path.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- logging: split into 2 shards ----
|
||||
- test-group: custom-logging
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
|
||||
unit-flag: proxy-db-custom-logging
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_custom_logger.py
|
||||
tests/unit/proxy/test_custom_callback_input.py
|
||||
tests/unit/proxy/test_custom_logger_s3_gcs.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: logging-misc
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-logging-misc
|
||||
test-path: >-
|
||||
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
|
||||
tests/unit/proxy/spend_tracking/test_search_api_logging.py
|
||||
tests/unit/proxy/test_proxy_reject_logging.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: db-and-spend
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-db-and-spend
|
||||
test-path: >-
|
||||
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
|
||||
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
|
||||
tests/unit/proxy/db/test_update_daily_tag_spend.py
|
||||
tests/unit/proxy/test_db_schema_changes.py
|
||||
tests/unit/proxy/test_prisma_client_backoff_retry.py
|
||||
tests/unit/proxy/test_update_spend.py
|
||||
tests/unit/skills/test_skills_db.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- guardrails + budget + hooks: split into 2 ----
|
||||
- test-group: guardrails-hooks
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-guardrails-hooks
|
||||
test-path: >-
|
||||
tests/unit/proxy/hooks/test_banned_keyword_list.py
|
||||
tests/unit/proxy/test_proxy_setting_guardrails.py
|
||||
tests/unit/proxy/test_unit_test_proxy_hooks.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: budgets
|
||||
test-path: ""
|
||||
unit-flag: proxy-db-budgets
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
|
||||
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
|
||||
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: endpoints-and-responses
|
||||
test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
|
||||
unit-flag: proxy-db-endpoints-and-responses
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_exception_mapping.py
|
||||
tests/unit/proxy/lens
|
||||
tests/unit/proxy/auth/test_models_fallback_endpoint.py
|
||||
tests/unit/proxy/common_utils/test_check_batch_cost.py
|
||||
tests/unit/proxy/common_utils/test_check_responses_cost.py
|
||||
tests/unit/proxy/common_utils/test_realtime_cache.py
|
||||
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
|
||||
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
|
||||
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
|
||||
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
|
||||
tests/unit/proxy/response_polling
|
||||
tests/unit/proxy/test_custom_tokenizer_bug.py
|
||||
tests/unit/proxy/test_get_favicon.py
|
||||
tests/unit/proxy/test_get_image.py
|
||||
tests/unit/proxy/test_prompt_test_endpoint.py
|
||||
tests/unit/proxy/test_reducto_ocr_route.py
|
||||
tests/unit/proxy/test_response_polling_pre_call_checks.py
|
||||
tests/unit/proxy/test_ui_path_detection.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: ${{ matrix.test-path }}
|
||||
unit-flag: ${{ matrix.unit-flag }}
|
||||
workers: ${{ matrix.workers }}
|
||||
reruns: 2
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
|
|
|
|||
126
.github/workflows/test-unit.yml
vendored
|
|
@ -35,10 +35,6 @@ concurrency:
|
|||
# a matrix and carries a shard-coverage guard that reads that file by name.
|
||||
# Folding it in here is a follow-up, together with generalising that guard into
|
||||
# assert_ci_coverage.py.
|
||||
#
|
||||
# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the
|
||||
# shard under the same Codecov flag. That pipeline is manual-only while the
|
||||
# tests migrate, so the shard also runs those files on every event.
|
||||
jobs:
|
||||
unit:
|
||||
name: ${{ matrix.shard }}
|
||||
|
|
@ -50,19 +46,11 @@ jobs:
|
|||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- shard: mcp-integration
|
||||
artifact-name: mcp-integration
|
||||
test-path: "tests/mcp_tests"
|
||||
unit-flag: mcp-integration
|
||||
workers: 2
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
|
||||
- shard: core-utils
|
||||
artifact-name: core-utils
|
||||
test-path: ""
|
||||
unit-flag: core-utils
|
||||
test-path: >-
|
||||
tests/unit/decisions
|
||||
tests/unit/litellm_core_utils
|
||||
workers: 2
|
||||
reruns: 1
|
||||
timeout-minutes: 20
|
||||
|
|
@ -70,8 +58,22 @@ jobs:
|
|||
|
||||
- shard: enterprise-routing
|
||||
artifact-name: enterprise-routing
|
||||
test-path: ""
|
||||
unit-flag: enterprise-routing
|
||||
test-path: >-
|
||||
tests/unit/google_genai
|
||||
tests/unit/router_strategy
|
||||
tests/unit/router_utils
|
||||
tests/unit/proxy/common_utils/test_cache_aware_routing.py
|
||||
tests/unit/enterprise/enterprise_callbacks/send_emails
|
||||
tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
|
||||
tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
|
||||
tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
|
||||
tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
|
||||
tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
|
||||
tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
|
||||
tests/unit/enterprise/proxy/test_enterprise_routes.py
|
||||
tests/unit/enterprise/proxy/test_file_deletion_blocking.py
|
||||
tests/unit/enterprise/proxy/test_managed_files_access_check.py
|
||||
tests/unit/enterprise/proxy/test_managed_files_hook.py
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -82,7 +84,7 @@ jobs:
|
|||
test-path: >-
|
||||
tests/test_litellm/integrations
|
||||
tests/test_litellm/tracing
|
||||
unit-flag: integrations
|
||||
tests/unit/integrations
|
||||
workers: 2
|
||||
reruns: 3
|
||||
timeout-minutes: 20
|
||||
|
|
@ -90,8 +92,8 @@ jobs:
|
|||
|
||||
- shard: Vertex AI
|
||||
artifact-name: llm-vertex-ai
|
||||
test-path: ""
|
||||
unit-flag: llm-vertex-ai
|
||||
test-path: >-
|
||||
tests/unit/llms/vertex_ai
|
||||
workers: 1
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -99,8 +101,10 @@ jobs:
|
|||
|
||||
- shard: All Other Providers
|
||||
artifact-name: llm-other-providers
|
||||
test-path: ""
|
||||
unit-flag: llm-other-providers
|
||||
test-path: >-
|
||||
tests/unit/llms
|
||||
--ignore=tests/unit/llms/vertex_ai
|
||||
--ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -110,7 +114,27 @@ jobs:
|
|||
artifact-name: misc
|
||||
test-path: >-
|
||||
tests/test_litellm/test_*.py
|
||||
unit-flag: misc
|
||||
tests/unit/test_*.py
|
||||
tests/unit/test_router
|
||||
tests/unit/a2a_protocol
|
||||
tests/unit/batches
|
||||
tests/unit/chat_completions
|
||||
tests/unit/completion_extras
|
||||
tests/unit/containers
|
||||
tests/unit/embeddings
|
||||
tests/unit/endpoints
|
||||
tests/unit/files
|
||||
tests/unit/harness
|
||||
tests/unit/images
|
||||
tests/unit/interactions
|
||||
tests/unit/messages
|
||||
tests/unit/rag
|
||||
tests/unit/rerank_api
|
||||
tests/unit/rust_bridge
|
||||
tests/unit/secret_managers
|
||||
tests/unit/vector_stores
|
||||
tests/unit/videos
|
||||
--ignore=tests/unit/rust_bridge/native_route_wheel_test.py
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -141,6 +165,7 @@ jobs:
|
|||
artifact-name: proxy-endpoints
|
||||
test-path: >-
|
||||
tests/unit/proxy/analytics_endpoints
|
||||
tests/unit/proxy/decisions_endpoints
|
||||
tests/unit/proxy/management_endpoints
|
||||
tests/unit/proxy/list_api
|
||||
tests/unit/proxy/memory
|
||||
|
|
@ -165,6 +190,7 @@ jobs:
|
|||
tests/unit/proxy/response_api_endpoints
|
||||
tests/unit/proxy/image_endpoints
|
||||
tests/unit/proxy/ocr_endpoints
|
||||
tests/unit/proxy/search_endpoints
|
||||
tests/unit/proxy/vector_store_endpoints
|
||||
tests/unit/proxy/agent_endpoints
|
||||
tests/unit/proxy/a2a
|
||||
|
|
@ -216,7 +242,10 @@ jobs:
|
|||
tests/unit/proxy/enterprise_billing
|
||||
tests/unit/proxy/types_utils
|
||||
tests/unit/proxy/logging_endpoints
|
||||
unit-flag: proxy-infra
|
||||
tests/unit/gateway
|
||||
tests/unit/proxy/management
|
||||
tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
|
||||
tests/unit/proxy/roi_calculator
|
||||
workers: 4
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -258,8 +287,8 @@ jobs:
|
|||
|
||||
- shard: caching-local
|
||||
artifact-name: caching-local
|
||||
test-path: ""
|
||||
unit-flag: caching-local
|
||||
test-path: >-
|
||||
tests/unit/caching
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -267,8 +296,8 @@ jobs:
|
|||
|
||||
- shard: proxy-extras
|
||||
artifact-name: proxy-extras
|
||||
test-path: ""
|
||||
unit-flag: proxy-extras
|
||||
test-path: >-
|
||||
tests/unit/litellm_proxy_extras
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -276,8 +305,15 @@ jobs:
|
|||
|
||||
- shard: enterprise-package
|
||||
artifact-name: enterprise-package
|
||||
test-path: ""
|
||||
unit-flag: enterprise-package
|
||||
test-path: >-
|
||||
tests/unit/enterprise/integrations
|
||||
tests/unit/enterprise/proxy/auth
|
||||
tests/unit/enterprise/proxy/guardrails
|
||||
tests/unit/enterprise/proxy/hooks
|
||||
tests/unit/enterprise/proxy/management_endpoints
|
||||
tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
|
||||
tests/unit/enterprise/proxy/test_liteadmin.py
|
||||
tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
|
||||
workers: 4
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
|
|
@ -285,19 +321,41 @@ jobs:
|
|||
|
||||
- shard: responses-caching-types
|
||||
artifact-name: responses-caching-types
|
||||
test-path: ""
|
||||
unit-flag: responses-caching-types
|
||||
test-path: >-
|
||||
tests/unit/responses
|
||||
tests/unit/types
|
||||
--ignore=tests/unit/responses/mcp
|
||||
workers: 2
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
|
||||
- shard: unit
|
||||
artifact-name: unit
|
||||
test-path: >-
|
||||
tests/unit/anthropic_interface
|
||||
tests/unit/compression
|
||||
tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py
|
||||
tests/unit/enterprise/enterprise_callbacks/test_llm_guard.py
|
||||
tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py
|
||||
tests/unit/integration_support
|
||||
tests/unit/models
|
||||
tests/unit/ocr
|
||||
tests/unit/passthrough
|
||||
tests/unit/realtime_api
|
||||
tests/unit/repositories
|
||||
tests/unit/sandbox
|
||||
tests/unit/skills/test_skills_main.py
|
||||
tests/unit/tracing
|
||||
workers: 2
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: ${{ matrix.test-path }}
|
||||
unit-flag: ${{ matrix.unit-flag || '' }}
|
||||
workers: ${{ matrix.workers }}
|
||||
reruns: ${{ matrix.reruns }}
|
||||
timeout-minutes: ${{ matrix.timeout-minutes }}
|
||||
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
|
||||
artifact-name: ${{ matrix.artifact-name }}
|
||||
legacy-mcp-peer: ${{ matrix.shard == 'mcp-integration' }}
|
||||
|
|
|
|||
4
.gitignore
vendored
|
|
@ -58,6 +58,7 @@ litellm/proxy/tests/package-lock.json
|
|||
ui/litellm-dashboard/.next
|
||||
ui/litellm-dashboard/node_modules
|
||||
ui/litellm-dashboard/next-env.d.ts
|
||||
*.tsbuildinfo
|
||||
ui/litellm-dashboard/package.json
|
||||
ui/litellm-dashboard/package-lock.json
|
||||
helm/litellm-helm/*.tgz
|
||||
|
|
@ -150,3 +151,6 @@ litellm.log
|
|||
|
||||
.coverage-rust
|
||||
coverage-rust.xml
|
||||
|
||||
# make lens-dev worker token, generated config and logs
|
||||
.lens-dev/
|
||||
|
|
|
|||
13
Dockerfile
|
|
@ -114,8 +114,20 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
|
|||
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
||||
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --no-deps /tmp/liteadmin
|
||||
|
||||
# Runtime stage
|
||||
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
||||
ARG LITELLM_RELEASE_TAG=""
|
||||
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
||||
|
||||
USER root
|
||||
|
||||
|
|
@ -141,6 +153,7 @@ ENV PATH="/app/.venv/bin:${PATH}" \
|
|||
# ship (manifest-scanning tools attribute everything in it to this image).
|
||||
# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY --from=liteadmin-builder /opt/liteadmin /opt/liteadmin
|
||||
COPY --from=builder /app/docker /app/docker
|
||||
COPY --from=builder /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
|
||||
|
|
|
|||
6
Makefile
|
|
@ -4,7 +4,7 @@
|
|||
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
|
||||
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
|
||||
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
|
||||
test-rust-extension rust-sqlx-prepare \
|
||||
test-rust-extension rust-sqlx-prepare lens-dev \
|
||||
info lint lint-inner lint-dev lint-checks format \
|
||||
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
|
||||
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
|
||||
|
|
@ -58,6 +58,7 @@ help:
|
|||
@echo " make test-unit-helm - Run helm unit tests"
|
||||
@echo " make test-rust-extension - Build the Rust extension and run its public Python tests"
|
||||
@echo " make rust-sqlx-prepare - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
|
||||
@echo " make lens-dev - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large --seed-logs\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)"
|
||||
@echo ""
|
||||
@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
|
||||
@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
|
||||
|
|
@ -311,6 +312,9 @@ test-rust-extension:
|
|||
rust-sqlx-prepare:
|
||||
cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
|
||||
|
||||
lens-dev:
|
||||
./scripts/lens_dev.sh $(ARGS)
|
||||
|
||||
test: install-test-deps
|
||||
$(UV_RUN) pytest tests/
|
||||
|
||||
|
|
|
|||
27
README.md
|
|
@ -268,6 +268,31 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
|
|||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Agents</b> - Run Claude Code, Codex, OpenCode or Deep Agents on any model (Python SDK)</summary>
|
||||
|
||||
### Python SDK - Agents
|
||||
|
||||
```python
|
||||
import litellm
|
||||
from litellm import Harness, sandbox
|
||||
|
||||
result = litellm.agent(
|
||||
Harness.CLAUDE_CODE, # or Harness.CODEX, Harness.OPENCODE, Harness.DEEPAGENTS
|
||||
"Find why tests/test_router.py is flaky and fix it.",
|
||||
sandbox=sandbox.local("./repo"),
|
||||
model="litellm_proxy/claude-sonnet-4-5", # a model group on your AI Gateway
|
||||
)
|
||||
|
||||
print(result.text, result.cost, [f.path for f in result.files])
|
||||
```
|
||||
|
||||
Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call the agent makes goes through your AI Gateway, tagged `harness,claude_code`. Drop the `litellm_proxy/` prefix to call a provider directly. Install `starlette uvicorn` plus the agent's CLI (`claude`, `codex` or `opencode`), or `deepagents langchain-litellm` for Deep Agents.
|
||||
|
||||
[**Docs: Agent Harnesses**](https://docs.litellm.ai/docs/harness)
|
||||
|
||||
</details>
|
||||
|
||||
### Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
|
||||
|
||||
| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
|
||||
|
|
@ -365,11 +390,13 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
|
|||
| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
|
||||
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |
|
||||
| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
|
||||
| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -71,6 +71,8 @@ RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component
|
|||
|
||||
# ---------- Runtime ----------
|
||||
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
||||
ARG LITELLM_RELEASE_TAG=""
|
||||
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
||||
|
||||
USER root
|
||||
|
||||
|
|
|
|||
|
|
@ -8,9 +8,13 @@ Run with:
|
|||
uvicorn backend.main:app --host 0.0.0.0 --port 4001
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Final
|
||||
|
||||
from fastapi.routing import Mount
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Mount
|
||||
from starlette.types import Lifespan
|
||||
|
||||
# See gateway/main.py for why we assemble DATABASE_URL(s) here before
|
||||
# importing proxy_server.
|
||||
|
|
@ -43,14 +47,16 @@ def _is_backend_route(route) -> bool:
|
|||
|
||||
# See gateway/main.py for why the trim runs inside the lifespan instead of at
|
||||
# module scope.
|
||||
_proxy_lifespan = app.router.lifespan_context
|
||||
_proxy_lifespan: Final = app.router.lifespan_context
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _backend_lifespan(app_):
|
||||
async with _proxy_lifespan(app_):
|
||||
async def _backend_lifespan(
|
||||
app_: Starlette, lifespan: Lifespan[Starlette] = _proxy_lifespan
|
||||
) -> AsyncGenerator[Mapping[str, object], None]:
|
||||
async with lifespan(app_) as state:
|
||||
app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
|
||||
yield
|
||||
yield state if state is not None else {}
|
||||
|
||||
|
||||
app.router.lifespan_context = _backend_lifespan
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/customer/",
|
||||
"/end_user/",
|
||||
"/sso/",
|
||||
"/liteadmin/slack/connect/",
|
||||
"/login",
|
||||
"/v2/login",
|
||||
"/v3/login",
|
||||
|
|
@ -60,6 +61,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
|
|||
# Tools / agents (registry & policy admin)
|
||||
"/v1/tool/",
|
||||
"/v1/agents",
|
||||
"/agent/daily/activity/",
|
||||
# Guardrails admin
|
||||
"/v2/guardrails/",
|
||||
# MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
|
||||
|
|
|
|||
|
|
@ -5710,6 +5710,17 @@
|
|||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(litellm_anthropic_wif_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "anthropic_wif",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
|
|
@ -5719,7 +5730,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_auth_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "auth",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5730,7 +5741,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_batch_write_to_db_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "batch_write_to_db",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5741,7 +5752,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_postgres_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "postgres",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5752,7 +5763,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_proxy_pre_call_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "proxy_pre_call",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5763,7 +5774,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis",
|
||||
"range": true,
|
||||
"refId": "E"
|
||||
"refId": "F"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5774,7 +5785,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_org_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis_daily_org_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "F"
|
||||
"refId": "G"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5785,7 +5796,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_tag_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis_daily_tag_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "G"
|
||||
"refId": "H"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5796,7 +5807,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_daily_team_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis_daily_team_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "H"
|
||||
"refId": "I"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5807,7 +5818,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_redis_window_spend_update_queue_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "redis_window_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "I"
|
||||
"refId": "J"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5818,7 +5829,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_reset_budget_job_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "reset_budget_job",
|
||||
"range": true,
|
||||
"refId": "J"
|
||||
"refId": "K"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5829,7 +5840,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_router_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "router",
|
||||
"range": true,
|
||||
"refId": "K"
|
||||
"refId": "L"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5840,7 +5851,7 @@
|
|||
"expr": "histogram_quantile(0.95, sum(rate(litellm_self_latency_bucket[$__rate_interval])) by (le))",
|
||||
"legendFormat": "self",
|
||||
"range": true,
|
||||
"refId": "L"
|
||||
"refId": "M"
|
||||
}
|
||||
],
|
||||
"title": "Service latency p95 (litellm_<service>_latency)",
|
||||
|
|
@ -5888,6 +5899,28 @@
|
|||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_anthropic_wif_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "anthropic_wif",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_anthropic_wif_cache_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "anthropic_wif_cache",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
|
|
@ -5897,7 +5930,7 @@
|
|||
"expr": "sum(rate(litellm_auth_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "auth",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5908,7 +5941,7 @@
|
|||
"expr": "sum(rate(litellm_batch_write_to_db_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "batch_write_to_db",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5919,7 +5952,7 @@
|
|||
"expr": "sum(rate(litellm_postgres_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "postgres",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5930,7 +5963,7 @@
|
|||
"expr": "sum(rate(litellm_proxy_pre_call_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "proxy_pre_call",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
"refId": "F"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5941,7 +5974,7 @@
|
|||
"expr": "sum(rate(litellm_redis_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis",
|
||||
"range": true,
|
||||
"refId": "E"
|
||||
"refId": "G"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5952,7 +5985,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_org_spend_update_queue_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis_daily_org_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "F"
|
||||
"refId": "H"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5963,7 +5996,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_tag_spend_update_queue_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis_daily_tag_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "G"
|
||||
"refId": "I"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5974,7 +6007,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_team_spend_update_queue_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis_daily_team_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "H"
|
||||
"refId": "J"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5985,7 +6018,7 @@
|
|||
"expr": "sum(rate(litellm_redis_window_spend_update_queue_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "redis_window_spend_update_queue",
|
||||
"range": true,
|
||||
"refId": "I"
|
||||
"refId": "K"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -5996,7 +6029,7 @@
|
|||
"expr": "sum(rate(litellm_reset_budget_job_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "reset_budget_job",
|
||||
"range": true,
|
||||
"refId": "J"
|
||||
"refId": "L"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6007,7 +6040,7 @@
|
|||
"expr": "sum(rate(litellm_router_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "router",
|
||||
"range": true,
|
||||
"refId": "K"
|
||||
"refId": "M"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6018,7 +6051,7 @@
|
|||
"expr": "sum(rate(litellm_self_total_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "self",
|
||||
"range": true,
|
||||
"refId": "L"
|
||||
"refId": "N"
|
||||
}
|
||||
],
|
||||
"title": "Service request rate (litellm_<service>_total_requests)",
|
||||
|
|
@ -6066,6 +6099,28 @@
|
|||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_anthropic_wif_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "anthropic_wif / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_anthropic_wif_cache_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "anthropic_wif_cache / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
|
|
@ -6075,7 +6130,7 @@
|
|||
"expr": "sum(rate(litellm_auth_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "auth / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6086,7 +6141,7 @@
|
|||
"expr": "sum(rate(litellm_batch_write_to_db_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "batch_write_to_db / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
"refId": "D"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6097,7 +6152,7 @@
|
|||
"expr": "sum(rate(litellm_postgres_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "postgres / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "C"
|
||||
"refId": "E"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6108,7 +6163,7 @@
|
|||
"expr": "sum(rate(litellm_proxy_pre_call_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "proxy_pre_call / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "D"
|
||||
"refId": "F"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6119,7 +6174,7 @@
|
|||
"expr": "sum(rate(litellm_redis_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "E"
|
||||
"refId": "G"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6130,7 +6185,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_org_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis_daily_org_spend_update_queue / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "F"
|
||||
"refId": "H"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6141,7 +6196,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_tag_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis_daily_tag_spend_update_queue / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "G"
|
||||
"refId": "I"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6152,7 +6207,7 @@
|
|||
"expr": "sum(rate(litellm_redis_daily_team_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis_daily_team_spend_update_queue / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "H"
|
||||
"refId": "J"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6163,7 +6218,7 @@
|
|||
"expr": "sum(rate(litellm_redis_window_spend_update_queue_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "redis_window_spend_update_queue / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "I"
|
||||
"refId": "K"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6174,7 +6229,7 @@
|
|||
"expr": "sum(rate(litellm_reset_budget_job_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "reset_budget_job / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "J"
|
||||
"refId": "L"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6185,7 +6240,7 @@
|
|||
"expr": "sum(rate(litellm_router_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "router / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "K"
|
||||
"refId": "M"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
|
|
@ -6196,7 +6251,7 @@
|
|||
"expr": "sum(rate(litellm_self_failed_requests_total[$__rate_interval])) by (error_class)",
|
||||
"legendFormat": "self / {{error_class}}",
|
||||
"range": true,
|
||||
"refId": "L"
|
||||
"refId": "N"
|
||||
}
|
||||
],
|
||||
"title": "Service failure rate (litellm_<service>_failed_requests)",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# LiteLLM All Prometheus Metrics dashboard
|
||||
|
||||
Every `litellm_*` metric family the proxy can expose on `/metrics` (136 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
|
||||
Every `litellm_*` metric family the proxy can expose on `/metrics` (141 families across 97 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
|
||||
|
||||
Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/unit/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,28 @@
|
|||
FROM python:3.12-slim
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
||||
FROM $LITELLM_BUILD_IMAGE AS builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy
|
||||
WORKDIR /app
|
||||
RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
|
||||
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
|
||||
COPY deploy/lens/requirements.lock /tmp/requirements.lock
|
||||
RUN uv venv --python python3.13 /app/.venv && \
|
||||
uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock
|
||||
|
||||
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
||||
ARG LITELLM_RELEASE_TAG=""
|
||||
RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
|
||||
RUN apk add --no-cache python-3.13
|
||||
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
|
||||
COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
|
||||
USER 65532:65532
|
||||
CMD ["python", "-m", "lens.worker"]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,15 @@
|
|||
**
|
||||
!deploy/
|
||||
!deploy/lens/
|
||||
!deploy/lens/requirements.lock
|
||||
!litellm/
|
||||
!litellm/proxy/
|
||||
!litellm/proxy/lens/
|
||||
!litellm/proxy/lens/__init__.py
|
||||
!litellm/proxy/lens/models.py
|
||||
!litellm/proxy/lens/trace_store.py
|
||||
!litellm/proxy/lens/analysis.py
|
||||
!litellm/proxy/lens/worker.py
|
||||
!litellm/proxy/lens/release.py
|
||||
!litellm/proxy/lens/prompts/
|
||||
!litellm/proxy/lens/prompts/**
|
||||
|
|
|
|||
|
|
@ -2,23 +2,103 @@
|
|||
|
||||
Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
|
||||
|
||||
## Start a worker
|
||||
## Install
|
||||
|
||||
Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
|
||||
Build LiteLLM and its worker from the same source commit with the same release identity. The worker runs separately and connects to your gateway using a limited worker token
|
||||
|
||||
In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in
|
||||
### New local installation
|
||||
|
||||
The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
|
||||
Install Docker with Compose and Git. This builds LiteLLM and its worker from the same checkout and starts the existing local tracing stack:
|
||||
|
||||
The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds
|
||||
```bash
|
||||
git clone https://github.com/BerriAI/litellm.git
|
||||
cd litellm
|
||||
export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)"
|
||||
export LENS_WORKER_IMAGE="litellm-lens-worker:${LITELLM_RELEASE_TAG}"
|
||||
export OPENAI_API_KEY='sk-...'
|
||||
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
|
||||
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
|
||||
docker compose -f docker/docker-compose.tracing.yml up -d --build
|
||||
```
|
||||
|
||||
For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
|
||||
Open `http://localhost:4002/ui/` and sign in as `admin` with password `sk-1234`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run:
|
||||
|
||||
```bash
|
||||
export LITELLM_URL=http://litellm:4000
|
||||
export LENS_WORKER_TOKEN='<paste-your-worker-token>'
|
||||
docker compose -f docker/docker-compose.tracing.yml -f deploy/lens/compose.yaml up -d
|
||||
```
|
||||
|
||||
The worker joins the gateway's Docker network, and the dashboard shows **Worker connected**. Save the token privately for restarts and upgrades
|
||||
|
||||
This stack is for local evaluation: it binds to localhost and uses development database credentials. For a hosted deployment, keep your normal database, keys, networking, and deployment process. Build both images from one source revision with the same `LITELLM_RELEASE_TAG`, publish the worker to your registry, and set `LENS_WORKER_IMAGE` on LiteLLM to that image
|
||||
|
||||
### Existing LiteLLM installation
|
||||
|
||||
Keep your deployment and PostgreSQL database. A working gateway/worker pair can stay as it is until you upgrade both. For a gateway built from source, use its exact commit and `LITELLM_RELEASE_TAG`; a release version or the latest commit on `main` is not a substitute for that source identity
|
||||
|
||||
The public development package is `ghcr.io/berriai/litellm-lens-worker-dev:sha-<full-commit>`. It publishes amd64 images on Lens-related changes, so an arbitrary source commit may have no image. Check the exact image exists before using it. If it is unavailable, your gateway uses a different release identity, or you need native arm64, build the worker from the gateway's checkout:
|
||||
|
||||
```bash
|
||||
export LITELLM_RELEASE_TAG='<gateway-release-identity>'
|
||||
export LENS_WORKER_IMAGE='<your-registry>/litellm-lens-worker:<your-image-tag>'
|
||||
docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \
|
||||
-f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" .
|
||||
```
|
||||
|
||||
For a remote worker host, publish that image to a registry the host can pull from. Set the gateway's `LENS_WORKER_IMAGE` to the resulting image reference, restart the gateway using its normal deployment process, then copy its install command. Prefer the published image digest for hosted installations. Do not change the gateway's release identity just to accept another worker
|
||||
|
||||
For Kubernetes or Render, run the standalone worker using `LITELLM_URL` and `LENS_WORKER_TOKEN` from setup. Keep existing databases and secrets. The worker needs no inbound port.
|
||||
|
||||
## Helm
|
||||
|
||||
The componentized source chart at `helm/litellm` includes an optional Lens worker. Use the chart from the same checkout as your gateway and keep your component image overrides in your values. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values:
|
||||
|
||||
```yaml
|
||||
lensWorker:
|
||||
enabled: true
|
||||
image:
|
||||
repository: <your-worker-image-repository>
|
||||
digest: sha256:<matching-worker-image-digest>
|
||||
tokenSecret:
|
||||
name: litellm-lens-worker
|
||||
key: token
|
||||
```
|
||||
|
||||
Set the worker repository and digest explicitly to an image built from the gateway's source commit and release identity. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart and update the gateway and worker image overrides together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too
|
||||
|
||||
## Standalone worker
|
||||
|
||||
Start with a source deployment that includes Lens, PostgreSQL, and agent tracing, and prepare its matching worker as described above. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
tracing:
|
||||
store:
|
||||
type: clickhouse
|
||||
url: os.environ/CLICKHOUSE_URL
|
||||
retention_days: 14
|
||||
```
|
||||
|
||||
The URL, database, and retention settings can also come from `CLICKHOUSE_URL`, `CLICKHOUSE_DATABASE`, and `AGENT_TRACING_RETENTION_DAYS` when omitted from YAML. A YAML value wins when both are set. The database defaults to `litellm`. `retention_days` defaults to 14 and applies to both traces and spend logs
|
||||
|
||||
Retention changes require a proxy restart. ClickHouse removes expired rows during background merges, not immediately at startup. Enable request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
|
||||
|
||||
In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in
|
||||
|
||||
The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. Once the matching image is available on the worker host, no second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
|
||||
|
||||
The dashboard uses the gateway's `LENS_WORKER_IMAGE` override when set. Public `:sha-<commit>` development images must match both the gateway commit and release identity. Build from source for the worker host's native architecture
|
||||
|
||||
After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying
|
||||
|
||||
For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and an explicit `LENS_WORKER_IMAGE` in a private environment file:
|
||||
|
||||
```bash
|
||||
docker compose --env-file /path/to/lens.env -f compose.yaml up -d
|
||||
```
|
||||
|
||||
Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
|
||||
To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
|
||||
|
||||
The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
|
||||
|
||||
|
|
@ -92,6 +172,37 @@ curl "$LITELLM_URL/lens/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITE
|
|||
|
||||
Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/lens/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /lens/{id}/findings/{finding_id}` with `status` and `reason`
|
||||
|
||||
## Local development
|
||||
|
||||
`make lens-dev ARGS=--seed` starts the full dev stack. The live dashboard is at `http://localhost:3000/ui/lens/`, with login at `http://localhost:3000/ui/login/`. Next.js forwards API requests to the proxy on port 4000, so login and navigation stay in the live UI and edits hot-reload
|
||||
|
||||
The default is Next.js dev with no production build (`LENS_DEV_BUILD_UI=0`). Set `LENS_DEV_BUILD_UI=1` when you also want a fresh static dashboard at `http://localhost:4000/ui/`. Build output goes to `.lens-dev/logs/ui-build.log`; a failed build stops startup. Both modes keep the live dashboard on port 3000. Startup checks the live login route before seeding and fails with the UI log path if Next.js exits. `LENS_DEV_STARTUP_TIMEOUT_SECONDS` controls startup readiness retries (default 300; `LENS_DEV_READINESS_REQUEST_TIMEOUT_SECONDS` caps each HTTP probe, default 5)
|
||||
|
||||
For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies spread over the last 24 hours, about 860,000 spans with linked request logs, plus three long sessions of roughly 1,150, 9,200 and 92,000 spans in a single trace for drawer paging and the oversized read path. Their trace IDs are printed at the end. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. Every profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks, and verifies linked spend totals through the proxy. Large seeds then copy that first copy inside ClickHouse and PostgreSQL with `INSERT ... SELECT`, rewriting trace, span and call IDs so each copy keeps its own spend, and verify the last copy through the proxy
|
||||
|
||||
Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key
|
||||
|
||||
Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse variables before starting the proxy and seeder so both processes use the same settings. Invalid, zero and negative values fail instead of silently falling back. Changing these limits does not require rebuilding Rust
|
||||
|
||||
| Environment variable | Default | Controls |
|
||||
| --- | --- | --- |
|
||||
| `LENS_DEV_SEED_COPIES` | 1 default, 2000 large | Total fixture copies |
|
||||
| `LENS_DEV_SEED_TIMEOUT_SECONDS` | 120 | Seeder HTTP timeout |
|
||||
| `OTLP_MAX_BODY_BYTES` | 16777216 | HTTP body and decompressed payload bytes |
|
||||
| `OTLP_MAX_CONCURRENT_INGESTS` | 2 | Concurrent proxy ingestion requests |
|
||||
| `OTLP_MAX_ATTRIBUTE_VALUE_BYTES` | 65536 | Stored attribute/content bytes |
|
||||
| `OTLP_MAX_DECODE_DEPTH` | 32 | Nested decode depth |
|
||||
| `OTLP_MAX_DECODE_NODES` | 65536 | JSON values or protobuf fields per export |
|
||||
| `OTLP_MAX_SPANS` | 4096 | Spans per export |
|
||||
| `OTLP_MAX_ATTRIBUTES` | 256 | Attributes per resource, scope, span, event or link |
|
||||
| `OTLP_MAX_EVENTS` | 256 | Events per span |
|
||||
| `OTLP_MAX_LINKS` | 256 | Links per span |
|
||||
| `OTLP_MAX_DECODED_SPAN_BYTES` | 16777216 | Decoded span allocation budget |
|
||||
| `CLICKHOUSE_TRACE_MAX_INSERT_BYTES` | 67108864 | Encoded trace or spend insert bytes |
|
||||
| `CLICKHOUSE_INSERT_TIMEOUT_SECONDS` | 30 | ClickHouse insert HTTP timeout |
|
||||
|
||||
The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits.
|
||||
|
||||
## Quality evaluation
|
||||
|
||||
Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
|
||||
|
|
@ -115,3 +226,19 @@ The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`,
|
|||
Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
|
||||
|
||||
Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push
|
||||
|
||||
|
||||
## Release compatibility
|
||||
|
||||
Gateway and worker builds carry the same `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release
|
||||
|
||||
The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Set an explicit `LENS_WORKER_IMAGE` for worker-only Compose. Verify that the image exists and matches the gateway before deploying it
|
||||
|
||||
For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version
|
||||
|
||||
The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes to `ghcr.io/berriai/litellm-lens-worker-dev` on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker
|
||||
|
||||
|
||||
## Worker dependencies
|
||||
|
||||
The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures
|
||||
|
|
|
|||
|
|
@ -3,4 +3,6 @@ services:
|
|||
build:
|
||||
context: ../..
|
||||
dockerfile: deploy/lens/Dockerfile
|
||||
args:
|
||||
LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?Set the release tag used by the gateway}
|
||||
image: litellm-lens-worker:local
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
services:
|
||||
lens-worker:
|
||||
image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:a8e8731d954916594eea462969946b9292fb771681ff515a9fd296b53f856c77}
|
||||
image: ${LENS_WORKER_IMAGE:-${LITELLM_VERSION:+ghcr.io/berriai/litellm-lens-worker:v}${LITELLM_VERSION:-}}
|
||||
environment:
|
||||
LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
|
||||
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
|
||||
|
|
|
|||
7
deploy/lens/config.yaml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
tracing:
|
||||
store:
|
||||
type: clickhouse
|
||||
url: os.environ/CLICKHOUSE_URL
|
||||
retention_days: 14
|
||||
2
deploy/lens/requirements.in
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
httpx==0.28.1
|
||||
pydantic==2.13.4
|
||||
172
deploy/lens/requirements.lock
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock
|
||||
annotated-types==0.8.0 \
|
||||
--hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \
|
||||
--hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0
|
||||
# via pydantic
|
||||
anyio==4.15.1 \
|
||||
--hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
|
||||
--hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
|
||||
# via httpx
|
||||
certifi==2026.7.22 \
|
||||
--hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \
|
||||
--hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
h11==0.16.0 \
|
||||
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
|
||||
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
|
||||
# via httpcore
|
||||
httpcore==1.0.9 \
|
||||
--hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \
|
||||
--hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8
|
||||
# via httpx
|
||||
httpx==0.28.1 \
|
||||
--hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
|
||||
--hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
|
||||
# via -r deploy/lens/requirements.in
|
||||
idna==3.20 \
|
||||
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
|
||||
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
|
||||
# via
|
||||
# anyio
|
||||
# httpx
|
||||
pydantic==2.13.4 \
|
||||
--hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \
|
||||
--hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6
|
||||
# via -r deploy/lens/requirements.in
|
||||
pydantic-core==2.46.4 \
|
||||
--hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \
|
||||
--hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \
|
||||
--hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \
|
||||
--hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \
|
||||
--hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \
|
||||
--hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \
|
||||
--hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \
|
||||
--hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \
|
||||
--hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \
|
||||
--hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \
|
||||
--hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \
|
||||
--hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \
|
||||
--hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \
|
||||
--hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \
|
||||
--hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \
|
||||
--hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \
|
||||
--hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \
|
||||
--hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \
|
||||
--hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \
|
||||
--hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \
|
||||
--hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \
|
||||
--hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \
|
||||
--hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \
|
||||
--hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \
|
||||
--hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \
|
||||
--hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \
|
||||
--hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \
|
||||
--hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \
|
||||
--hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \
|
||||
--hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \
|
||||
--hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \
|
||||
--hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \
|
||||
--hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \
|
||||
--hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \
|
||||
--hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \
|
||||
--hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \
|
||||
--hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \
|
||||
--hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \
|
||||
--hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \
|
||||
--hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \
|
||||
--hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \
|
||||
--hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \
|
||||
--hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \
|
||||
--hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \
|
||||
--hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \
|
||||
--hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \
|
||||
--hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \
|
||||
--hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \
|
||||
--hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \
|
||||
--hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \
|
||||
--hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \
|
||||
--hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \
|
||||
--hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \
|
||||
--hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \
|
||||
--hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \
|
||||
--hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \
|
||||
--hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \
|
||||
--hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \
|
||||
--hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \
|
||||
--hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \
|
||||
--hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \
|
||||
--hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \
|
||||
--hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \
|
||||
--hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \
|
||||
--hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \
|
||||
--hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \
|
||||
--hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \
|
||||
--hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \
|
||||
--hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \
|
||||
--hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \
|
||||
--hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \
|
||||
--hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \
|
||||
--hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \
|
||||
--hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \
|
||||
--hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \
|
||||
--hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \
|
||||
--hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \
|
||||
--hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \
|
||||
--hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \
|
||||
--hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \
|
||||
--hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \
|
||||
--hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \
|
||||
--hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \
|
||||
--hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \
|
||||
--hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \
|
||||
--hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \
|
||||
--hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \
|
||||
--hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \
|
||||
--hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \
|
||||
--hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \
|
||||
--hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \
|
||||
--hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \
|
||||
--hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \
|
||||
--hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \
|
||||
--hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \
|
||||
--hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \
|
||||
--hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \
|
||||
--hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \
|
||||
--hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \
|
||||
--hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \
|
||||
--hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \
|
||||
--hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \
|
||||
--hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \
|
||||
--hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \
|
||||
--hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \
|
||||
--hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \
|
||||
--hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \
|
||||
--hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \
|
||||
--hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \
|
||||
--hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \
|
||||
--hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \
|
||||
--hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \
|
||||
--hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \
|
||||
--hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \
|
||||
--hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \
|
||||
--hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \
|
||||
--hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \
|
||||
--hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \
|
||||
--hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \
|
||||
--hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae
|
||||
# via pydantic
|
||||
typing-extensions==4.16.0 \
|
||||
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
|
||||
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
|
||||
# via
|
||||
# anyio
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# typing-inspection
|
||||
typing-inspection==0.4.4 \
|
||||
--hash=sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47 \
|
||||
--hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147
|
||||
# via pydantic
|
||||
91
deploy/lens/stack.yaml
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: litellm-lens
|
||||
|
||||
services:
|
||||
litellm:
|
||||
image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix}
|
||||
entrypoint:
|
||||
- python3
|
||||
- -c
|
||||
- |
|
||||
import os, sys
|
||||
from urllib.parse import quote
|
||||
postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
|
||||
clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="")
|
||||
os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
|
||||
os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123"
|
||||
os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
|
||||
command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key}
|
||||
LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
|
||||
STORE_MODEL_IN_DB: "True"
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
|
||||
LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
|
||||
volumes:
|
||||
- ./config.yaml:/app/lens-config.yaml:ro
|
||||
ports:
|
||||
- "127.0.0.1:${LITELLM_PORT:-4000}:4000"
|
||||
networks: [proxy, storage]
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
lens-worker:
|
||||
profiles: [lens]
|
||||
image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
|
||||
environment:
|
||||
LITELLM_URL: http://litellm:4000
|
||||
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
|
||||
depends_on: [litellm]
|
||||
networks: [proxy]
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
|
||||
db:
|
||||
image: postgres:16
|
||||
environment:
|
||||
POSTGRES_DB: litellm
|
||||
POSTGRES_USER: litellm
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
networks: [storage]
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server:26.9.6.6
|
||||
environment:
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
|
||||
volumes:
|
||||
- clickhouse_data:/var/lib/clickhouse
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--user", "default", "--password", "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
|
||||
networks:
|
||||
proxy:
|
||||
storage:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
clickhouse_data:
|
||||
|
|
@ -6,8 +6,6 @@ services:
|
|||
context: .
|
||||
dockerfile: docker/Dockerfile.non_root
|
||||
target: runtime
|
||||
args:
|
||||
PROXY_EXTRAS_SOURCE: "local"
|
||||
depends_on:
|
||||
- squid
|
||||
user: "101:101"
|
||||
|
|
|
|||
44
docker-compose.liteadmin.yml
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
services:
|
||||
litellm:
|
||||
image: ${LITELLM_IMAGE:?Set the native-enabled gateway image}
|
||||
environment:
|
||||
LITELLM_ADMIN_AGENT_URL: http://liteadmin:10000
|
||||
ADMIN_AGENT_SERVICE_TOKEN: ${ADMIN_AGENT_SERVICE_TOKEN:?Set a shared worker token}
|
||||
PROXY_BASE_URL: ${LITELLM_PUBLIC_URL:?Set the existing HTTPS gateway URL}
|
||||
|
||||
liteadmin:
|
||||
image: ${LITELLM_IMAGE:?Set the same native-enabled image used by the gateway}
|
||||
command: ["--admin-agent"]
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
read_only: true
|
||||
cap_drop: [ALL]
|
||||
security_opt: [no-new-privileges:true]
|
||||
stop_grace_period: 75s
|
||||
environment:
|
||||
CONNECTION_AUTH_MODE: native
|
||||
LITELLM_BASE_URL: ${LITELLM_PUBLIC_URL:?Set the existing HTTPS gateway URL}
|
||||
LITELLM_MODEL: ${LITELLM_ADMIN_MODEL:?Set a gateway model with tool support}
|
||||
SLACK_BOT_TOKEN: ${SLACK_BOT_TOKEN:?Install the Slack app}
|
||||
SLACK_APP_TOKEN: ${SLACK_APP_TOKEN:?Enable Socket Mode}
|
||||
SLACK_WORKSPACE_ID: ${SLACK_WORKSPACE_ID:?Set the Slack workspace ID}
|
||||
ADMIN_AGENT_SERVICE_TOKEN: ${ADMIN_AGENT_SERVICE_TOKEN:?Set a shared worker token}
|
||||
CREDENTIAL_ENCRYPTION_KEY: ${CREDENTIAL_ENCRYPTION_KEY:?Set a persistent Fernet key}
|
||||
STATE_DB: /var/data/events.sqlite3
|
||||
ADMIN_READ_ONLY: ${ADMIN_READ_ONLY:-false}
|
||||
OPENAI_AGENTS_DISABLE_TRACING: "1"
|
||||
volumes:
|
||||
- liteadmin_state:/var/data
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m
|
||||
healthcheck:
|
||||
test: ["CMD", "/opt/liteadmin/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:10000/readyz', timeout=3)"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
start_period: 30s
|
||||
depends_on:
|
||||
litellm:
|
||||
condition: service_healthy
|
||||
|
||||
volumes:
|
||||
liteadmin_state:
|
||||
|
|
@ -113,6 +113,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
||||
ARG LITELLM_RELEASE_TAG=""
|
||||
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
||||
|
||||
USER root
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG PROXY_EXTRAS_SOURCE=published
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -44,7 +43,6 @@ COPY ui/litellm-dashboard/ ./
|
|||
RUN npm run build
|
||||
|
||||
FROM $LITELLM_BUILD_IMAGE AS builder
|
||||
ARG PROXY_EXTRAS_SOURCE
|
||||
WORKDIR /app
|
||||
USER root
|
||||
|
||||
|
|
@ -107,26 +105,14 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
|
|||
touch /var/lib/litellm/ui/.litellm_ui_ready
|
||||
|
||||
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
|
||||
if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
|
||||
uv sync --frozen --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--extra bedrock-realtime \
|
||||
--python python3.13 \
|
||||
--no-sources-package litellm-proxy-extras; \
|
||||
else \
|
||||
uv sync --frozen --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--extra bedrock-realtime \
|
||||
--python python3.13; \
|
||||
fi
|
||||
uv sync --frozen --no-default-groups --no-editable \
|
||||
--extra proxy \
|
||||
--extra proxy-runtime \
|
||||
--extra extra_proxy \
|
||||
--extra semantic-router \
|
||||
--extra saml \
|
||||
--extra bedrock-realtime \
|
||||
--python python3.13
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
|
|
@ -136,7 +122,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
|
||||
|
||||
FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
||||
ARG PROXY_EXTRAS_SOURCE
|
||||
ARG LITELLM_RELEASE_TAG=""
|
||||
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
||||
WORKDIR /app
|
||||
USER root
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,13 @@ services:
|
|||
litellm:
|
||||
image: docker.litellm.ai/berriai/litellm:main-stable
|
||||
ports:
|
||||
- "4000:4000"
|
||||
# LITELLM_BIND is empty by default, so this stays "4000:4000". The quickstart
|
||||
# script sets it to "127.0.0.1:" so new installs listen on this machine only.
|
||||
- "${LITELLM_BIND:-}${LITELLM_PORT:-4000}:4000"
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env - see the header of this file}
|
||||
LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?set it in .env - see the header of this file}
|
||||
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
|
||||
DATABASE_URL: postgresql://litellm:${POSTGRES_PASSWORD:-litellm}@db:5432/litellm
|
||||
STORE_MODEL_IN_DB: "True"
|
||||
depends_on:
|
||||
db:
|
||||
|
|
@ -27,7 +29,7 @@ services:
|
|||
image: postgres:16
|
||||
environment:
|
||||
POSTGRES_USER: litellm
|
||||
POSTGRES_PASSWORD: litellm
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-litellm}
|
||||
POSTGRES_DB: litellm
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U litellm"]
|
||||
|
|
|
|||
|
|
@ -5,16 +5,19 @@ services:
|
|||
build:
|
||||
context: ..
|
||||
target: runtime
|
||||
args:
|
||||
LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:-}
|
||||
command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: local-tracing-master-key
|
||||
LITELLM_MASTER_KEY: sk-1234
|
||||
LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
|
||||
LITELLM_SALT_KEY: sk-local-tracing-salt-key
|
||||
DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
|
||||
STORE_MODEL_IN_DB: "True"
|
||||
CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
|
||||
CLICKHOUSE_READER_URL: http://default:local-tracing@clickhouse:8123
|
||||
CLICKHOUSE_DATABASE: litellm
|
||||
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
||||
LENS_WORKER_IMAGE: ${LENS_WORKER_IMAGE:-}
|
||||
volumes:
|
||||
- ./tracing-config.yaml:/app/tracing-config.yaml:ro
|
||||
ports:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
#!/bin/sh
|
||||
|
||||
if [ "$1" = "--admin-agent" ]; then
|
||||
shift
|
||||
export CONNECTION_AUTH_MODE=native
|
||||
exec /opt/liteadmin/bin/litellm-admin-agent --web "$@"
|
||||
fi
|
||||
|
||||
case "$USE_DDTRACE" in
|
||||
[Tt][Rr][Uu][Ee])
|
||||
export DD_TRACE_OPENAI_ENABLED="False"
|
||||
|
|
|
|||
|
|
@ -7,4 +7,7 @@ model_list:
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
tracing:
|
||||
store: clickhouse
|
||||
store:
|
||||
type: clickhouse
|
||||
url: os.environ/CLICKHOUSE_URL
|
||||
retention_days: 14
|
||||
|
|
|
|||
|
|
@ -7,15 +7,19 @@
|
|||
## This accepts a list of user id's for whom calls will be rejected
|
||||
|
||||
|
||||
from typing import Optional, Literal
|
||||
import litellm
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from typing import Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class _ENTERPRISE_BlockedUserList(CustomLogger):
|
||||
enforces_request_content: bool = True
|
||||
|
|
@ -54,6 +58,7 @@ class _ENTERPRISE_BlockedUserList(CustomLogger):
|
|||
if litellm.set_verbose is True:
|
||||
print(print_statement) # noqa
|
||||
|
||||
@with_service_target(AUTH_OBJECTS_TARGET)
|
||||
async def async_pre_call_hook(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Base class for sending emails to user after creating keys or invite links
|
|||
import html
|
||||
import json
|
||||
import os
|
||||
from typing import List, Literal, Optional
|
||||
from typing import Final, List, Literal, Optional
|
||||
|
||||
from litellm_enterprise.types.enterprise_callbacks.send_emails import (
|
||||
EmailEvent,
|
||||
|
|
@ -15,6 +15,7 @@ from litellm_enterprise.types.enterprise_callbacks.send_emails import (
|
|||
SendKeyRotatedEmailEvent,
|
||||
)
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import (
|
||||
|
|
@ -45,9 +46,13 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
WebhookEvent,
|
||||
)
|
||||
from litellm.repositories.table_repositories import InvitationLinkRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.secret_managers.main import get_secret_bool
|
||||
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
|
||||
|
||||
_BUDGET_ALERT_CLAIMS_TARGET: Final = "budget_alert_claims"
|
||||
|
||||
|
||||
def _max_budget_alert_id(user_info: CallInfo) -> str:
|
||||
if user_info.event_group == Litellm_EntityType.TEAM_MEMBER:
|
||||
|
|
@ -437,6 +442,7 @@ class BaseEmailLogger(CustomLogger):
|
|||
html_body=email_html_content,
|
||||
)
|
||||
|
||||
@with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
|
||||
async def budget_alerts(
|
||||
self,
|
||||
type: Literal[
|
||||
|
|
@ -606,6 +612,7 @@ class BaseEmailLogger(CustomLogger):
|
|||
await self._release_budget_alert_claim(_cache, _cache_key)
|
||||
return
|
||||
|
||||
@with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
|
||||
async def _handle_multi_threshold_max_budget_alert(
|
||||
self,
|
||||
user_info: CallInfo,
|
||||
|
|
@ -691,6 +698,7 @@ class BaseEmailLogger(CustomLogger):
|
|||
)
|
||||
await self._release_budget_alert_claim(_cache, _cache_key)
|
||||
|
||||
@with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
|
||||
async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None:
|
||||
try:
|
||||
await cache.async_delete_cache(key=cache_key)
|
||||
|
|
@ -838,7 +846,7 @@ class BaseEmailLogger(CustomLogger):
|
|||
)
|
||||
return None
|
||||
|
||||
user_row = await prisma_client.db.litellm_usertable.find_unique(
|
||||
user_row = await UserRepository(prisma_client).table.find_unique(
|
||||
where={"user_id": user_id}
|
||||
)
|
||||
|
||||
|
|
@ -923,7 +931,7 @@ class BaseEmailLogger(CustomLogger):
|
|||
try:
|
||||
# Try to get existing invitation
|
||||
existing_invitations = (
|
||||
await prisma_client.db.litellm_invitationlink.find_many(
|
||||
await InvitationLinkRepository(prisma_client).table.find_many(
|
||||
where={"user_id": user_id},
|
||||
order={"created_at": "desc"},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm_enterprise.types.enterprise_callbacks.send_emails import (
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.db.db_span import db_span
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
|
@ -94,16 +95,17 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
|
|||
json_settings = json.dumps(general_settings, default=str)
|
||||
|
||||
# Save updated general settings
|
||||
await prisma_client.db.litellm_config.upsert(
|
||||
where={"param_name": "general_settings"},
|
||||
data={
|
||||
"create": {
|
||||
"param_name": "general_settings",
|
||||
"param_value": json_settings,
|
||||
async with db_span("save_email_settings", "LiteLLM_Config"):
|
||||
await prisma_client.db.litellm_config.upsert(
|
||||
where={"param_name": "general_settings"},
|
||||
data={
|
||||
"create": {
|
||||
"param_name": "general_settings",
|
||||
"param_value": json_settings,
|
||||
},
|
||||
"update": {"param_value": json_settings},
|
||||
},
|
||||
"update": {"param_value": json_settings},
|
||||
},
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ from litellm.constants import (
|
|||
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
|
||||
MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
)
|
||||
from litellm.repositories.table_repositories import ManagedObjectRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models
|
||||
|
|
@ -58,25 +62,19 @@ class _ManagedObjectRow(Protocol):
|
|||
|
||||
|
||||
def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
|
||||
table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
|
||||
return table
|
||||
return ManagedObjectRepository(prisma_client).table
|
||||
|
||||
|
||||
def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
|
||||
table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
|
||||
return table
|
||||
return UserRepository(prisma_client).table
|
||||
|
||||
|
||||
def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
|
||||
table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
|
||||
prisma_client.db.litellm_verificationtoken
|
||||
)
|
||||
return table
|
||||
return VerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
|
||||
table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
|
||||
return table
|
||||
return TeamRepository(prisma_client).table
|
||||
|
||||
|
||||
class CheckBatchCost:
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.constants import (
|
|||
MAX_OBJECTS_PER_POLL_CYCLE,
|
||||
STALE_OBJECT_CLEANUP_BATCH_SIZE,
|
||||
)
|
||||
from litellm.repositories.table_repositories import ManagedObjectRepository
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
|
||||
|
|
@ -43,8 +44,7 @@ class _ManagedObjectRow(Protocol):
|
|||
|
||||
|
||||
def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
|
||||
table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
|
||||
return table
|
||||
return ManagedObjectRepository(prisma_client).table
|
||||
|
||||
|
||||
class CheckResponsesCost:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
|
|||
|
||||
from . import ui_crud_endpoints # side-effect: registers extra UI settings
|
||||
from .audit_logging_endpoints import router as audit_logging_router
|
||||
from .liteadmin import router as liteadmin_router
|
||||
from .management_endpoints import management_endpoints_router
|
||||
from .utils import _should_block_robots
|
||||
|
||||
|
|
@ -14,6 +15,7 @@ __all__ = ["router", "ui_crud_endpoints"]
|
|||
router = APIRouter()
|
||||
router.include_router(email_events_router)
|
||||
router.include_router(audit_logging_router)
|
||||
router.include_router(liteadmin_router)
|
||||
router.include_router(management_endpoints_router)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -26,6 +26,7 @@ from pydantic import ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm import Router, verbose_logger
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._uuid import uuid
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.constants import MAX_FILE_LIST_LIMIT
|
||||
|
|
@ -144,6 +145,7 @@ def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) ->
|
|||
class _ManagedFileRow(Protocol):
|
||||
unified_file_id: str
|
||||
file_object: OpenAIFileObject
|
||||
flat_model_file_ids: Sequence[str]
|
||||
storage_backend: Optional[str]
|
||||
storage_url: Optional[str]
|
||||
created_by: Optional[str]
|
||||
|
|
@ -201,6 +203,16 @@ def _managed_file_table(prisma_client: PrismaClient) -> _ManagedFileTableActions
|
|||
return prisma_client.db.litellm_managedfiletable
|
||||
|
||||
|
||||
def _iter_provider_file_id_pairs(
|
||||
rows: Sequence[_ManagedFileRow],
|
||||
requested_provider_file_ids: frozenset[str],
|
||||
) -> Iterator[tuple[str, str]]:
|
||||
for row in rows:
|
||||
for provider_file_id in row.flat_model_file_ids:
|
||||
if provider_file_id in requested_provider_file_ids:
|
||||
yield provider_file_id, row.unified_file_id
|
||||
|
||||
|
||||
def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableActions:
|
||||
return prisma_client.db.litellm_managedobjecttable
|
||||
|
||||
|
|
@ -218,6 +230,9 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
|
|||
)
|
||||
|
||||
|
||||
_MANAGED_FILES_TARGET: Final = "managed_files"
|
||||
|
||||
|
||||
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
||||
# Class variables or attributes
|
||||
def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
|
||||
|
|
@ -231,6 +246,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
|
||||
return PrometheusLogger.get_instance()
|
||||
|
||||
@with_service_target(_MANAGED_FILES_TARGET)
|
||||
async def store_unified_file_id(
|
||||
self,
|
||||
file_id: str,
|
||||
|
|
@ -314,6 +330,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
verbose_logger.warning(f"could not resolve org for managed object attribution: {e}")
|
||||
return None
|
||||
|
||||
@with_service_target(_MANAGED_FILES_TARGET)
|
||||
async def store_unified_object_id(
|
||||
self,
|
||||
unified_object_id: str,
|
||||
|
|
@ -401,6 +418,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
},
|
||||
)
|
||||
|
||||
@with_service_target(_MANAGED_FILES_TARGET)
|
||||
async def get_unified_file_id(
|
||||
self, file_id: str, litellm_parent_otel_span: Optional[Span] = None
|
||||
) -> Optional[LiteLLM_ManagedFileTable]:
|
||||
|
|
@ -423,6 +441,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump())
|
||||
return None
|
||||
|
||||
@with_service_target(_MANAGED_FILES_TARGET)
|
||||
async def delete_unified_file_id(
|
||||
self, file_id: str, litellm_parent_otel_span: Optional[Span] = None
|
||||
) -> OpenAIFileObject:
|
||||
|
|
@ -710,6 +729,39 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return None
|
||||
return batch_obj
|
||||
|
||||
async def get_unified_file_ids_for_provider_file_ids(
|
||||
self,
|
||||
provider_file_ids: Sequence[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> Mapping[str, str]:
|
||||
if not provider_file_ids:
|
||||
return MappingProxyType({})
|
||||
|
||||
unique_provider_file_ids: Final = tuple(dict.fromkeys(provider_file_ids))
|
||||
owner_filter: Final = build_owner_filter(user_api_key_dict)
|
||||
if owner_filter is None:
|
||||
return MappingProxyType({})
|
||||
|
||||
provider_file_ids_list: Final = [ # mutable-ok: Prisma hasSome requires a list
|
||||
provider_file_id for provider_file_id in unique_provider_file_ids
|
||||
]
|
||||
rows: Final = await _managed_file_table(self.prisma_client).find_many(
|
||||
where={ # mutable-ok: Prisma requires a plain dictionary for where
|
||||
**owner_filter,
|
||||
"flat_model_file_ids": { # mutable-ok: Prisma requires a plain filter dictionary
|
||||
"hasSome": provider_file_ids_list,
|
||||
},
|
||||
}
|
||||
)
|
||||
return MappingProxyType(
|
||||
dict(
|
||||
_iter_provider_file_id_pairs(
|
||||
rows,
|
||||
frozenset(unique_provider_file_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
async def get_user_created_file_ids(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
|
||||
) -> List[OpenAIFileObject]:
|
||||
|
|
@ -1595,7 +1647,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
|
||||
owner_filter: Final = build_owner_filter(user_api_key_dict)
|
||||
if owner_filter is None:
|
||||
return FileListPage(**build_list_page([]))
|
||||
return FileListPage.model_validate(build_list_page([]))
|
||||
|
||||
if after:
|
||||
cursor_row = await _managed_file_table(self.prisma_client).find_first(
|
||||
|
|
@ -1634,7 +1686,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
cursor_id = chunk[-1].unified_file_id
|
||||
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
|
||||
|
||||
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
|
||||
return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
|
||||
|
||||
def _is_batch_polling_enabled(self) -> bool:
|
||||
"""
|
||||
|
|
|
|||
283
enterprise/litellm_enterprise/proxy/liteadmin.py
Normal file
|
|
@ -0,0 +1,283 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import html
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Annotated, Final
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse, Response
|
||||
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.proxy._experimental.mcp_server.oauth_utils import get_request_base_url
|
||||
from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
|
||||
|
||||
router: Final = APIRouter()
|
||||
_PREFIX: Final = "/liteadmin/slack/connect/"
|
||||
_COOKIE: Final = "__Host-litellm-slack-connect-"
|
||||
_HEADERS: Final = {
|
||||
"Cache-Control": "no-store",
|
||||
"Referrer-Policy": "same-origin",
|
||||
"X-Frame-Options": "DENY",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'",
|
||||
}
|
||||
|
||||
|
||||
class LinkDetails(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, strict=True, extra="forbid")
|
||||
workspace_id: str = Field(min_length=1, max_length=64)
|
||||
slack_user_id: str = Field(min_length=1, max_length=64)
|
||||
email: str = Field(min_length=1, max_length=320)
|
||||
|
||||
|
||||
class AdminSession(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
user_id: str
|
||||
credential: SecretStr
|
||||
expires_at: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeAdminContext:
|
||||
worker_url: str
|
||||
service_token: SecretStr
|
||||
client: httpx.AsyncClient
|
||||
session_user: Callable[[Request], Awaitable[str | None]]
|
||||
load_user: Callable[[str], Awaitable[LiteLLM_UserTable | None]]
|
||||
mint_session: Callable[[LiteLLM_UserTable], AdminSession]
|
||||
|
||||
async def worker_request(self, token: str, session: AdminSession | None = None) -> httpx.Response:
|
||||
if re.fullmatch(r"[A-Za-z0-9_-]{43}", token) is None:
|
||||
raise HTTPException(410, "Connection link expired. Send connect in Slack for a new link")
|
||||
try:
|
||||
response: Final = await self.client.request(
|
||||
"GET" if session is None else "POST",
|
||||
f"{self.worker_url}/internal/liteadmin/links/{token}",
|
||||
headers={"X-LiteLLM-Admin-Agent-Token": self.service_token.get_secret_value()},
|
||||
json=None
|
||||
if session is None
|
||||
else {
|
||||
"user_id": session.user_id,
|
||||
"credential": session.credential.get_secret_value(),
|
||||
"expires_at": session.expires_at,
|
||||
},
|
||||
timeout=15,
|
||||
follow_redirects=False,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
raise HTTPException(503, "LiteAdmin is temporarily unavailable") from None
|
||||
if response.status_code == 410:
|
||||
raise HTTPException(410, "Connection link expired. Send connect in Slack for a new link")
|
||||
if response.status_code == 403:
|
||||
raise HTTPException(403, "Connect your own active LiteLLM proxy-admin account with the same email as Slack")
|
||||
if response.status_code != 200:
|
||||
raise HTTPException(503, "LiteAdmin could not verify this connection")
|
||||
return response
|
||||
|
||||
async def details(self, token: str) -> LinkDetails:
|
||||
response: Final = await self.worker_request(token)
|
||||
try:
|
||||
return LinkDetails.model_validate_json(response.content)
|
||||
except ValidationError:
|
||||
raise HTTPException(503, "LiteAdmin could not verify this connection") from None
|
||||
|
||||
async def admin(self, user_id: str, details: LinkDetails) -> LiteLLM_UserTable:
|
||||
user: Final = await self.load_user(user_id)
|
||||
if (
|
||||
user is None
|
||||
or user.user_role != LitellmUserRoles.PROXY_ADMIN.value
|
||||
or not user.user_email
|
||||
or user.user_email.strip().casefold() != details.email.strip().casefold()
|
||||
):
|
||||
raise HTTPException(403, "Connect your own active LiteLLM proxy-admin account with the same email as Slack")
|
||||
return user
|
||||
|
||||
|
||||
def _page(title: str, body: str) -> HTMLResponse:
|
||||
return HTMLResponse(
|
||||
f'<!doctype html><html lang="en"><meta charset="utf-8">'
|
||||
f'<meta name="viewport" content="width=device-width,initial-scale=1"><title>{html.escape(title)}</title>'
|
||||
"<style>body{font:17px system-ui;color:#18252f;max-width:560px;margin:10vh auto;padding:24px}"
|
||||
"p{line-height:1.6}button{font:inherit;border:0;border-radius:8px;padding:14px 20px;background:#5b3fd1;"
|
||||
"color:white;cursor:pointer}small{color:#556}</style>"
|
||||
f"<main><h1>{html.escape(title)}</h1>{body}</main></html>",
|
||||
headers=_HEADERS,
|
||||
)
|
||||
|
||||
|
||||
def _cookie_name(token: str) -> str:
|
||||
return _COOKIE + hashlib.sha256(token.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
async def _session_user(request: Request) -> str | None:
|
||||
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
|
||||
get_authenticated_browser_user_id,
|
||||
)
|
||||
|
||||
return await get_authenticated_browser_user_id(request)
|
||||
|
||||
|
||||
async def _load_user(user_id: str) -> LiteLLM_UserTable | None:
|
||||
from litellm.proxy.auth.auth_checks import get_user_object
|
||||
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
|
||||
|
||||
if prisma_client is None:
|
||||
raise HTTPException(503, "LiteAdmin requires a database")
|
||||
try:
|
||||
return await get_user_object(
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
user_id_upsert=False,
|
||||
check_db_only=True,
|
||||
)
|
||||
except UserNotFoundError:
|
||||
return None
|
||||
except Exception:
|
||||
raise HTTPException(503, "LiteAdmin could not verify your current permissions") from None
|
||||
|
||||
|
||||
def mint_admin_session(user: LiteLLM_UserTable) -> AdminSession:
|
||||
from litellm.proxy.auth.auth_checks import LITELLM_SESSION_TOKEN_PREFIX
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_bearer_token
|
||||
|
||||
expires: Final = datetime.now(timezone.utc) + timedelta(hours=24)
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
token="liteadmin-" + secrets.token_urlsafe(24),
|
||||
key_name="LiteAdmin Slack",
|
||||
key_alias="LiteAdmin Slack",
|
||||
user_id=user.user_id,
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
models=TypeAdapter(list[str]).validate_python(user.model_dump().get("models", [])),
|
||||
expires=expires,
|
||||
is_session_token=True,
|
||||
)
|
||||
return AdminSession(
|
||||
user_id=user.user_id,
|
||||
credential=SecretStr(
|
||||
encrypt_bearer_token(auth.model_dump_json(exclude_none=True), LITELLM_SESSION_TOKEN_PREFIX)
|
||||
),
|
||||
expires_at=expires.timestamp(),
|
||||
)
|
||||
|
||||
|
||||
def validate_native_configuration(
|
||||
worker_url: str, service_token: str, enterprise: bool, database_available: bool
|
||||
) -> None:
|
||||
if not worker_url:
|
||||
raise HTTPException(404, "LiteAdmin Slack is not enabled")
|
||||
if not enterprise:
|
||||
raise HTTPException(403, "LiteAdmin Slack requires LiteLLM Enterprise")
|
||||
if not database_available:
|
||||
raise HTTPException(503, "LiteAdmin requires a database")
|
||||
try:
|
||||
parsed: Final = urlsplit(worker_url)
|
||||
port: Final = parsed.port
|
||||
except ValueError:
|
||||
raise HTTPException(503, "LiteAdmin worker configuration is invalid") from None
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or not parsed.hostname
|
||||
or port == 0
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.path
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or len(service_token) < 32
|
||||
or any(character.isspace() for character in service_token)
|
||||
):
|
||||
raise HTTPException(503, "LiteAdmin worker configuration is invalid")
|
||||
|
||||
|
||||
async def native_admin_context() -> NativeAdminContext:
|
||||
from litellm.proxy.proxy_server import premium_user, prisma_client
|
||||
|
||||
worker_url: Final = os.getenv("LITELLM_ADMIN_AGENT_URL", "").rstrip("/")
|
||||
service_token: Final = os.getenv("ADMIN_AGENT_SERVICE_TOKEN", "")
|
||||
validate_native_configuration(worker_url, service_token, premium_user is True, prisma_client is not None)
|
||||
client: Final = get_async_httpx_client(
|
||||
llm_provider="liteadmin_native", params={"timeout": 15.0, "follow_redirects": False}
|
||||
).client
|
||||
return NativeAdminContext(
|
||||
worker_url, SecretStr(service_token), client, _session_user, _load_user, mint_admin_session
|
||||
)
|
||||
|
||||
|
||||
@router.get(_PREFIX + "{token}", include_in_schema=False, response_class=HTMLResponse)
|
||||
async def connect_page(
|
||||
request: Request,
|
||||
token: str,
|
||||
context: Annotated[NativeAdminContext, Depends(native_admin_context)],
|
||||
) -> Response:
|
||||
details: Final = await context.details(token)
|
||||
base_url: Final = get_request_base_url(request)
|
||||
parsed_base: Final = urlsplit(base_url)
|
||||
if parsed_base.scheme != "https":
|
||||
raise HTTPException(400, "LiteAdmin account connections require HTTPS")
|
||||
user_id: Final = await context.session_user(request)
|
||||
if user_id is None:
|
||||
return RedirectResponse(
|
||||
base_url + "/sso/key/generate?" + urlencode({"return_to": parsed_base.path + _PREFIX + token}),
|
||||
status_code=303,
|
||||
headers=_HEADERS,
|
||||
)
|
||||
await context.admin(user_id, details)
|
||||
csrf: Final = secrets.token_urlsafe(32)
|
||||
page: Final = _page(
|
||||
"Connect LiteAdmin to Slack",
|
||||
f"<p>Connect <strong>{html.escape(details.email)}</strong> to LiteAdmin in your Slack workspace?</p>"
|
||||
"<p>Model requests and administrative actions will use your own LiteLLM account and current permissions</p>"
|
||||
f'<form method="post"><input type="hidden" name="csrf" value="{csrf}">'
|
||||
'<button type="submit">Connect account</button></form>'
|
||||
"<p><small>This connection lasts 24 hours. Send disconnect in Slack to remove the saved session</small></p>",
|
||||
)
|
||||
page.set_cookie(_cookie_name(token), csrf, max_age=600, secure=True, httponly=True, samesite="strict", path="/")
|
||||
return page
|
||||
|
||||
|
||||
@router.post(_PREFIX + "{token}", include_in_schema=False, response_class=HTMLResponse)
|
||||
async def connect_account(
|
||||
request: Request,
|
||||
token: str,
|
||||
context: Annotated[NativeAdminContext, Depends(native_admin_context)],
|
||||
) -> Response:
|
||||
base_url: Final = get_request_base_url(request)
|
||||
parsed_base: Final = urlsplit(base_url)
|
||||
origin: Final = f"{parsed_base.scheme}://{parsed_base.netloc}"
|
||||
if parsed_base.scheme != "https" or request.headers.get("Origin") != origin:
|
||||
raise HTTPException(403, "Reopen your private Slack connection link")
|
||||
if request.headers.get("Content-Type", "").split(";", 1)[0] != "application/x-www-form-urlencoded":
|
||||
raise HTTPException(400, "Expected a connection form")
|
||||
form: Final = await request.form(max_fields=1, max_files=0, max_part_size=1024)
|
||||
supplied: Final = form.get("csrf")
|
||||
expected: Final = request.cookies.get(_cookie_name(token), "")
|
||||
if (
|
||||
not isinstance(supplied, str)
|
||||
or len(expected) != 43
|
||||
or len(supplied) != 43
|
||||
or not hmac.compare_digest(supplied.encode(), expected.encode())
|
||||
):
|
||||
raise HTTPException(403, "Reopen your private Slack connection link")
|
||||
user_id: Final = await context.session_user(request)
|
||||
if user_id is None:
|
||||
raise HTTPException(401, "Your login expired. Reopen your private Slack connection link")
|
||||
details: Final = await context.details(token)
|
||||
user: Final = await context.admin(user_id, details)
|
||||
await context.worker_request(token, context.mint_session(user))
|
||||
page: Final = _page(
|
||||
"Account connected", "<p>Return to Slack and ask LiteAdmin to list your teams or check a budget</p>"
|
||||
)
|
||||
page.delete_cookie(_cookie_name(token), path="/", secure=True, httponly=True, samesite="strict")
|
||||
return page
|
||||
|
|
@ -22,7 +22,7 @@ from litellm._uuid import uuid
|
|||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.auth_checks import delete_cached_project_object
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.management.teams.access import is_team_admin
|
||||
from litellm.proxy.management.teams.authz import is_team_admin
|
||||
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
|
||||
from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
|
||||
from litellm.proxy.management_helpers.utils import (
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.72"
|
||||
version = "0.1.73"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.72"
|
||||
version = "0.1.73"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -9,9 +9,13 @@ Run with:
|
|||
uvicorn gateway.main:app --host 0.0.0.0 --port 4000
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Final
|
||||
|
||||
from fastapi.routing import Mount
|
||||
from starlette.applications import Starlette
|
||||
from starlette.routing import Mount
|
||||
from starlette.types import Lifespan
|
||||
|
||||
# Assemble DATABASE_URL (+ DATABASE_URL_READ_REPLICA) from the discrete
|
||||
# DATABASE_* env vars before proxy_server imports spin up Prisma. Handles
|
||||
|
|
@ -54,14 +58,16 @@ def _is_gateway_route(route) -> bool:
|
|||
# register routes. A module-load filter would miss routes added during
|
||||
# startup; running inside the lifespan, after the inner __aenter__, catches
|
||||
# them while still completing before uvicorn opens the listener.
|
||||
_proxy_lifespan = app.router.lifespan_context
|
||||
_proxy_lifespan: Final = app.router.lifespan_context
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _gateway_lifespan(app_):
|
||||
async with _proxy_lifespan(app_):
|
||||
async def _gateway_lifespan(
|
||||
app_: Starlette, lifespan: Lifespan[Starlette] = _proxy_lifespan
|
||||
) -> AsyncGenerator[Mapping[str, object], None]:
|
||||
async with lifespan(app_) as state:
|
||||
app_.router.routes = [r for r in app_.router.routes if _is_gateway_route(r)]
|
||||
yield
|
||||
yield state if state is not None else {}
|
||||
|
||||
|
||||
app.router.lifespan_context = _gateway_lifespan
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Path allowlist for the gateway component.
|
||||
|
||||
The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
|
||||
audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
|
||||
audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
|
||||
responses, vector stores, passthrough providers, realtime websockets, MCP
|
||||
tool-call endpoints, and operational endpoints (/health, /metrics, and the
|
||||
/debug/memory/summary read of the serving worker's RSS).
|
||||
|
|
@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/v1/rerank",
|
||||
"/v2/rerank",
|
||||
"/rerank",
|
||||
"/v1/decisions",
|
||||
"/decisions",
|
||||
"/v1/ocr",
|
||||
"/ocr",
|
||||
"/v1/rag/",
|
||||
|
|
|
|||
|
|
@ -57,6 +57,19 @@ spec:
|
|||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
env:
|
||||
{{- include "litellm.proxyEnv" . | nindent 12 }}
|
||||
{{- if .Values.liteadmin.enabled }}
|
||||
- name: LITELLM_ADMIN_AGENT_URL
|
||||
value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" . | trunc 53 | trimSuffix "-") | quote }}
|
||||
- name: ADMIN_AGENT_SERVICE_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ required "liteadmin.existingSecret is required" .Values.liteadmin.existingSecret }}
|
||||
key: ADMIN_AGENT_SERVICE_TOKEN
|
||||
{{- if not (hasKey (default dict .Values.envVars) "PROXY_BASE_URL") }}
|
||||
- name: PROXY_BASE_URL
|
||||
value: {{ required "liteadmin.gatewayUrl is required" .Values.liteadmin.gatewayUrl | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- include "litellm.proxyMetricsEnv" . | nindent 12 }}
|
||||
{{- if .Values.collector.enabled }}
|
||||
{{- include "litellm.collectorEnv" . | nindent 12 }}
|
||||
|
|
|
|||
112
helm/litellm-helm/templates/liteadmin.yaml
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
{{- if .Values.liteadmin.enabled }}
|
||||
{{- $name := printf "%s-liteadmin" (include "litellm.fullname" . | trunc 53 | trimSuffix "-") }}
|
||||
{{- $secret := required "liteadmin.existingSecret is required" .Values.liteadmin.existingSecret }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ $name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ $name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 75
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
fsGroup: 10001
|
||||
runAsNonRoot: true
|
||||
containers:
|
||||
- name: liteadmin
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
args: ["--admin-agent"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: {{ $secret }}
|
||||
env:
|
||||
- name: CONNECTION_AUTH_MODE
|
||||
value: native
|
||||
- name: LITELLM_BASE_URL
|
||||
value: {{ required "liteadmin.gatewayUrl is required" .Values.liteadmin.gatewayUrl | quote }}
|
||||
- name: LITELLM_MODEL
|
||||
value: {{ required "liteadmin.model is required" .Values.liteadmin.model | quote }}
|
||||
- name: STATE_DB
|
||||
value: /var/data/events.sqlite3
|
||||
- name: ADMIN_READ_ONLY
|
||||
value: {{ .Values.liteadmin.readOnly | quote }}
|
||||
- name: OPENAI_AGENTS_DISABLE_TRACING
|
||||
value: "1"
|
||||
ports:
|
||||
- name: health
|
||||
containerPort: 10000
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: health
|
||||
periodSeconds: 15
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: health
|
||||
periodSeconds: 30
|
||||
resources:
|
||||
{{- toYaml .Values.liteadmin.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: state
|
||||
mountPath: /var/data
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: state
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ $name }}
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 64Mi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ $name }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
ports:
|
||||
- port: 10000
|
||||
targetPort: health
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
{{- with .Values.liteadmin.storageClassName }}
|
||||
storageClassName: {{ . | quote }}
|
||||
{{- end }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.liteadmin.storageSize }}
|
||||
{{- end }}
|
||||
|
|
@ -112,6 +112,24 @@ tests:
|
|||
name: CUSTOM_VAR
|
||||
value: "custom_value"
|
||||
|
||||
- it: should override a user-supplied DISABLE_SCHEMA_UPDATE so the Job always migrates
|
||||
template: migrations-job.yaml
|
||||
set:
|
||||
envVars:
|
||||
DISABLE_SCHEMA_UPDATE: "true"
|
||||
migrationJob:
|
||||
enabled: true
|
||||
asserts:
|
||||
# The Job is what owns the schema, so it renders its own
|
||||
# DISABLE_SCHEMA_UPDATE=false after envVars and extraEnvVars. Kubernetes
|
||||
# takes the last value for a duplicated name, so the user's "true" cannot
|
||||
# leave the schema unmigrated. Skipping migrations is migrationJob.enabled.
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[-1]
|
||||
value:
|
||||
name: DISABLE_SCHEMA_UPDATE
|
||||
value: "false"
|
||||
|
||||
- it: should not include DATABASE_URL when deployStandalone is false
|
||||
template: migrations-job.yaml
|
||||
set:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,20 @@
|
|||
# Declare variables to be passed into your templates.
|
||||
|
||||
replicaCount: 1
|
||||
liteadmin:
|
||||
enabled: false
|
||||
existingSecret: ""
|
||||
gatewayUrl: ""
|
||||
model: ""
|
||||
readOnly: false
|
||||
storageSize: 1Gi
|
||||
storageClassName: ""
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
# numWorkers: 2
|
||||
|
||||
image:
|
||||
|
|
@ -545,7 +559,6 @@ redis:
|
|||
# Prisma migration job settings
|
||||
migrationJob:
|
||||
enabled: true # Enable or disable the schema migration Job
|
||||
retries: 3 # Number of retries for the Job in case of failure
|
||||
backoffLimit: 4 # Backoff limit for Job restarts
|
||||
# Wall-clock budget for the whole Job, shared across every `backoffLimit`
|
||||
# retry rather than granted per attempt. Without it a migration that blocks
|
||||
|
|
@ -554,7 +567,6 @@ migrationJob:
|
|||
# stop reconciling the whole chart until someone deletes the Job by hand.
|
||||
# Set to null to opt out and restore the unbounded behaviour.
|
||||
activeDeadlineSeconds: 1800
|
||||
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
|
||||
# Optional service account for the migration job.
|
||||
# Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
|
||||
# In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
|
||||
|
|
|
|||
|
|
@ -471,6 +471,24 @@ Directory of the collector's unix socket, shared by the gateway and
|
|||
collector containers through an emptyDir. Empty when the sidecar is off
|
||||
or gateway.collector.address is a tcp://127.0.0.1:<port> address.
|
||||
*/}}
|
||||
{{- define "litellm.lensWorker.image" -}}
|
||||
{{- if .Values.lensWorker.image.digest -}}
|
||||
{{- if not (regexMatch "^sha256:[0-9a-f]{64}$" .Values.lensWorker.image.digest) -}}
|
||||
{{- fail "lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters" -}}
|
||||
{{- end -}}
|
||||
{{- printf "%s@%s" .Values.lensWorker.image.repository .Values.lensWorker.image.digest -}}
|
||||
{{- else -}}
|
||||
{{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}}
|
||||
{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
|
||||
{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
|
||||
{{- $repository := .Values.lensWorker.image.repository -}}
|
||||
{{- if and (hasPrefix "sha-" $tag) (eq $repository "ghcr.io/berriai/litellm-lens-worker") -}}
|
||||
{{- $repository = "ghcr.io/berriai/litellm-lens-worker-dev" -}}
|
||||
{{- end -}}
|
||||
{{- printf "%s:%s" $repository $tag -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "litellm.gateway.collectorSocketDir" -}}
|
||||
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ spec:
|
|||
containerPort: 4001
|
||||
protocol: TCP
|
||||
env:
|
||||
- name: LENS_WORKER_IMAGE
|
||||
value: {{ include "litellm.lensWorker.image" . | quote }}
|
||||
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
|
||||
{{- if .Values.gateway.config.create }}
|
||||
- name: CONFIG_FILE_PATH
|
||||
|
|
|
|||
72
helm/litellm/templates/lens/deployment.yaml
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
{{- if .Values.lensWorker.enabled }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "litellm.fullname" . }}-lens-worker
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: lens-worker
|
||||
spec:
|
||||
replicas: {{ .Values.lensWorker.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: lens-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "litellm.commonLabels" . | nindent 8 }}
|
||||
app.kubernetes.io/component: lens-worker
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
fsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: lens-worker
|
||||
image: {{ include "litellm.lensWorker.image" . | quote }}
|
||||
imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
env:
|
||||
- name: LITELLM_URL
|
||||
value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
|
||||
- name: LENS_WORKER_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }}
|
||||
key: {{ .Values.lensWorker.tokenSecret.key | quote }}
|
||||
resources:
|
||||
{{- toYaml .Values.lensWorker.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
|
||||
{{- with .Values.lensWorker.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.lensWorker.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.lensWorker.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
174
helm/litellm/tests/lens_worker_tests.yaml
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
suite: Lens worker release and credentials
|
||||
templates:
|
||||
- lens/deployment.yaml
|
||||
- backend/deployment.yaml
|
||||
- gateway/configmap.yaml
|
||||
values:
|
||||
- ./values/required.yaml
|
||||
tests:
|
||||
- it: installs the development package for a source commit
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: sha-0123456789abcdef
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef
|
||||
- it: advertises the development package for standalone source workers
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: sha-0123456789abcdef
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_WORKER_IMAGE
|
||||
value: ghcr.io/berriai/litellm-lens-worker-dev:sha-0123456789abcdef
|
||||
- it: preserves an explicit private source image repository
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: sha-0123456789abcdef
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
lensWorker.image.repository: registry.example/lens-worker
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: registry.example/lens-worker:sha-0123456789abcdef
|
||||
- it: pins the worker to its approved digest even when its tag changes
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
lensWorker.image.tag: replaced-release
|
||||
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
- it: advertises the approved digest to standalone installers
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.image.tag: replaced-release
|
||||
lensWorker.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_WORKER_IMAGE
|
||||
value: ghcr.io/berriai/litellm-lens-worker@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
|
||||
- it: refuses a malformed digest instead of falling back to the tag
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.image.digest: sha256:invalid
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.image.digest must be sha256 followed by 64 lowercase hex characters
|
||||
- it: keeps the worker opt in
|
||||
template: lens/deployment.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: requires a limited worker credential when enabled
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token
|
||||
- it: uses the chart release and a secret without granting Kubernetes access
|
||||
template: lens/deployment.yaml
|
||||
chart:
|
||||
appVersion: v1.2.3
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker:v1.2.3
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
|
||||
value:
|
||||
name: lens-credential
|
||||
key: token
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
|
||||
value: true
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].emptyDir
|
||||
value:
|
||||
medium: Memory
|
||||
sizeLimit: 1Gi
|
||||
- it: advertises the same private dev image to standalone installers
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
lensWorker.image.repository: registry.example/lens-worker
|
||||
lensWorker.image.tag: branch-main-1234567
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_WORKER_IMAGE
|
||||
value: registry.example/lens-worker:branch-main-1234567
|
||||
- it: supports an external gateway and a registry override
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
lensWorker.url: https://gateway.example/proxy
|
||||
lensWorker.image.repository: registry.example/lens-worker
|
||||
lensWorker.image.tag: branch-main-1234567
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: registry.example/lens-worker:branch-main-1234567
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].env[0].value
|
||||
value: https://gateway.example/proxy
|
||||
- it: prefixes a numeric chart release with v
|
||||
template: lens/deployment.yaml
|
||||
chart:
|
||||
appVersion: 1.2.3-rc.4
|
||||
set:
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-rc.4
|
||||
- it: follows a backend image override when no worker tag is set
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: branch-main-1234567
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker:branch-main-1234567
|
||||
- it: recommends the overridden backend release for standalone installers
|
||||
template: backend/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: v1.2.3-dev.4
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: LENS_WORKER_IMAGE
|
||||
value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4
|
||||
- it: normalizes a numeric backend tag to the published worker tag
|
||||
template: lens/deployment.yaml
|
||||
set:
|
||||
backend.image.tag: 1.2.3-dev.4
|
||||
lensWorker.enabled: true
|
||||
lensWorker.tokenSecret.name: lens-credential
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4
|
||||
|
|
@ -629,3 +629,26 @@ ui:
|
|||
affinity: {}
|
||||
# Same shape as gateway.topologySpreadConstraints.
|
||||
topologySpreadConstraints: []
|
||||
|
||||
lensWorker:
|
||||
enabled: false
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: ghcr.io/berriai/litellm-lens-worker
|
||||
tag: ""
|
||||
digest: ""
|
||||
pullPolicy: IfNotPresent
|
||||
tokenSecret:
|
||||
name: ""
|
||||
key: token
|
||||
url: ""
|
||||
tmpSizeLimit: 1Gi
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 2Gi
|
||||
nodeSelector: {}
|
||||
tolerations: []
|
||||
affinity: {}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
-- CreateTable
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundInteractionSettlement" (
|
||||
"interaction_id" TEXT NOT NULL,
|
||||
"custom_llm_provider" TEXT NOT NULL,
|
||||
"create_context" JSONB NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"claimed_at" TIMESTAMP(3),
|
||||
"claimed_by" TEXT,
|
||||
"settled_at" TIMESTAMP(3),
|
||||
"outcome" TEXT,
|
||||
|
||||
CONSTRAINT "LiteLLM_BackgroundInteractionSettlement_pkey" PRIMARY KEY ("interaction_id")
|
||||
);
|
||||
|
||||
-- CreateIndex
|
||||
CREATE INDEX IF NOT EXISTS "idx_background_interaction_settlement_claimed_at" ON "LiteLLM_BackgroundInteractionSettlement"("claimed_at");
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterDailySpend" (
|
||||
"date" TEXT NOT NULL,
|
||||
"api_key" TEXT NOT NULL,
|
||||
"user_id" TEXT NOT NULL,
|
||||
"router_name" TEXT NOT NULL,
|
||||
"router_type" TEXT NOT NULL,
|
||||
"turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
CONSTRAINT "LiteLLM_AutoRouterDailySpend_pkey" PRIMARY KEY ("date", "api_key", "user_id", "router_name", "router_type")
|
||||
);
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
CREATE INDEX IF NOT EXISTS "LiteLLM_LensWorker_active_scope_idx"
|
||||
ON "LiteLLM_LensWorker" USING GIN ((data->'scope') jsonb_path_ops)
|
||||
WHERE data @> '{"revoked": false}'::jsonb;
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
-- CreateIndex (CONCURRENTLY)
|
||||
--
|
||||
-- Disclaimer:
|
||||
-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
|
||||
-- single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
|
||||
-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
|
||||
-- interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
|
||||
-- - Do not edit this file after it has been applied to any database: Prisma checksums
|
||||
-- migrations; add a new migration instead.
|
||||
-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
|
||||
-- without IF NOT EXISTS if you must support older versions).
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_flat_model_file_ids_idx" ON "LiteLLM_ManagedFileTable" USING GIN ("flat_model_file_ids");
|
||||
|
|
@ -56,8 +56,10 @@ REQUEST_LOG_INDEXES: Final = (
|
|||
)
|
||||
|
||||
_IDENTIFIER_MAX_BYTES: Final = 63
|
||||
_PARENT_LOCK_TIMEOUT: Final = "2s"
|
||||
_PARENT_LOCK_ATTEMPTS: Final = 30
|
||||
_DDL_LOCK_TIMEOUT: Final = "200ms"
|
||||
_DDL_LOCK_ATTEMPTS: Final = 10
|
||||
_DDL_RETRY_BASE_SECONDS: Final = 0.25
|
||||
_DDL_RETRY_MAX_SECONDS: Final = 8.0
|
||||
_LOCK_HANDOVER_SECONDS: Final = 2.0
|
||||
_DIGEST_LENGTH: Final = 8
|
||||
_CREATE_INDEX_STATEMENT: Final = re.compile(
|
||||
|
|
@ -181,6 +183,32 @@ def _under_migration_lock(connection: "psycopg.Connection[tuple[object, ...]]",
|
|||
return step()
|
||||
|
||||
|
||||
def _with_bounded_lock(
|
||||
connection: "psycopg.Connection[tuple[object, ...]]", step: Callable[[], bool], what: str
|
||||
) -> bool:
|
||||
"""Run `step` under the migration lock with a short lock_timeout, so a DDL statement that has to wait for open
|
||||
transactions holds new writes back for at most that long; retry with capped exponential backoff, holding the
|
||||
migration lock per attempt only and releasing it while sleeping. False when another process holds the migration
|
||||
lock or every attempt timed out."""
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
|
||||
for attempt in range(_DDL_LOCK_ATTEMPTS):
|
||||
if attempt:
|
||||
time.sleep(min(_DDL_RETRY_MAX_SECONDS, _DDL_RETRY_BASE_SECONDS * 2.0**attempt) * random.uniform(0.5, 1.0))
|
||||
connection.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(_DDL_LOCK_TIMEOUT)))
|
||||
try:
|
||||
return _under_migration_lock(connection, step)
|
||||
except psycopg.errors.LockNotAvailable:
|
||||
logger.info("Waiting for open transactions before %s", what)
|
||||
finally:
|
||||
connection.execute("SET lock_timeout = 0")
|
||||
logger.warning(
|
||||
"Could not get the lock for %s without holding writes back, leaving it for the next index build", what
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_index(connection: "psycopg.Connection[tuple[object, ...]]", schema: str, index: RequestLogIndex) -> bool:
|
||||
from psycopg.rows import class_row
|
||||
|
||||
|
|
@ -368,12 +396,13 @@ def build_index_on_partitioned_table(
|
|||
"Index %s already exists on %s rather than %s, leaving it alone", parent_index, existing.table, parent_table
|
||||
)
|
||||
return False
|
||||
if existing is None and not _under_migration_lock(
|
||||
if existing is None and not _with_bounded_lock(
|
||||
connection,
|
||||
lambda: (
|
||||
_adopt_equivalent_index(connection, schema, parent_table, parent_index, index)
|
||||
or _create_parent_index(connection, schema, parent_index, parent_table, index)
|
||||
),
|
||||
f"creating the parent index {parent_index}",
|
||||
):
|
||||
return False
|
||||
children: Final = _children_without_the_index(connection, schema, parent_table, parent_index)
|
||||
|
|
@ -393,29 +422,15 @@ def _create_parent_index(
|
|||
table: str,
|
||||
index: RequestLogIndex,
|
||||
) -> bool:
|
||||
"""Create the metadata-only parent index. Postgres takes a SHARE lock on the
|
||||
parent for that statement, so it waits for in-flight writes and queues new ones
|
||||
behind it; a short lock_timeout with retries keeps every such pause bounded."""
|
||||
import psycopg
|
||||
"""Create the metadata-only parent index. The caller bounds Postgres's SHARE lock wait on the parent."""
|
||||
from psycopg import sql
|
||||
|
||||
prefix: Final = sql.SQL("CREATE INDEX IF NOT EXISTS {} ON ONLY {} ").format(
|
||||
sql.Identifier(name), sql.Identifier(schema, table)
|
||||
)
|
||||
statement: Final = _create_index_statement(connection, prefix, index.definition)
|
||||
connection.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(_PARENT_LOCK_TIMEOUT)))
|
||||
try:
|
||||
for _ in range(_PARENT_LOCK_ATTEMPTS):
|
||||
try:
|
||||
connection.execute(statement)
|
||||
return True
|
||||
except psycopg.errors.LockNotAvailable:
|
||||
logger.info("Waiting for in-flight writes to %s before creating the parent index %s", table, name)
|
||||
time.sleep(random.uniform(0.1, 0.5))
|
||||
finally:
|
||||
connection.execute("SET lock_timeout = 0")
|
||||
logger.warning("Could not get the parent lock on %s to create %s, leaving it for the next index build", table, name)
|
||||
return False
|
||||
connection.execute(statement)
|
||||
return True
|
||||
|
||||
|
||||
def _attach_child_index(
|
||||
|
|
@ -445,4 +460,4 @@ def _attach_child_index(
|
|||
logger.info("Attached index %s on partition %s to %s", child_index, child.name, parent_index)
|
||||
return True
|
||||
|
||||
return _under_migration_lock(connection, attach)
|
||||
return _with_bounded_lock(connection, attach, f"attaching {child_index}")
|
||||
|
|
|
|||
|
|
@ -1144,6 +1144,7 @@ model LiteLLM_ManagedFileTable {
|
|||
updated_by String?
|
||||
|
||||
@@index([unified_file_id])
|
||||
@@index([flat_model_file_ids], type: Gin)
|
||||
@@index([team_id, created_at(sort: Desc)])
|
||||
}
|
||||
|
||||
|
|
@ -1744,6 +1745,27 @@ model LiteLLM_AutoRouterUserSession {
|
|||
@@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
|
||||
}
|
||||
|
||||
// Auto-routed requests per UTC request day and router: the selected-day money behind the
|
||||
// auto-router usage view. Written in the same statement as the session rollup, so a day row
|
||||
// and its session row never disagree; corrected in the same transaction as late baselines.
|
||||
model LiteLLM_AutoRouterDailySpend {
|
||||
date String
|
||||
api_key String
|
||||
user_id String
|
||||
router_name String
|
||||
router_type String
|
||||
turns Int @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
|
||||
@@id([date, api_key, user_id, router_name, router_type])
|
||||
}
|
||||
|
||||
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
|
||||
// either direction. forward duplicates the requests the keys did not route through the
|
||||
// router through it, answering whether they should adopt it; reverse duplicates the
|
||||
|
|
@ -1895,6 +1917,22 @@ model LiteLLM_WorkflowMessage {
|
|||
@@index([run_id])
|
||||
}
|
||||
|
||||
// Pending billing settlements for background interactions, keyed by the
|
||||
// interaction id so any replica can settle one that another replica created.
|
||||
// `claimed_at` is the exactly-once gate: the first conditional update wins.
|
||||
model LiteLLM_BackgroundInteractionSettlement {
|
||||
interaction_id String @id
|
||||
custom_llm_provider String
|
||||
create_context Json
|
||||
created_at DateTime @default(now())
|
||||
claimed_at DateTime?
|
||||
claimed_by String?
|
||||
settled_at DateTime?
|
||||
outcome String?
|
||||
|
||||
@@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
|
||||
}
|
||||
|
||||
model LiteLLM_Lens {
|
||||
id String @id
|
||||
version Int @default(0)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import functools
|
||||
import glob
|
||||
import os
|
||||
import random
|
||||
|
|
@ -10,7 +11,8 @@ import time
|
|||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final, Optional
|
||||
from typing import TYPE_CHECKING, Final, Optional, Union
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from litellm_proxy_extras import prisma_toolchain
|
||||
from litellm_proxy_extras._logging import logger
|
||||
|
|
@ -78,6 +80,23 @@ class _InvalidIndex:
|
|||
table_size: str
|
||||
|
||||
MAX_MIGRATE_DEPLOY_ATTEMPTS = 4
|
||||
LIBPQ_URL_PARAMS: Final = frozenset(
|
||||
{
|
||||
"sslmode",
|
||||
"sslcert",
|
||||
"sslkey",
|
||||
"sslrootcert",
|
||||
"sslpassword",
|
||||
"application_name",
|
||||
"connect_timeout",
|
||||
"client_encoding",
|
||||
"options",
|
||||
"service",
|
||||
"gssencmode",
|
||||
"krbsrvname",
|
||||
"target_session_attrs",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -185,6 +204,66 @@ def _max_migration_timestamp(names) -> int:
|
|||
return max(_migration_timestamp(n) for n in names)
|
||||
|
||||
|
||||
_REDACTED: Final = "REDACTED"
|
||||
_PASSWORD_QUERY_KEYS: Final = frozenset(("password", "sslpassword"))
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _secret_shape_redactor() -> Callable[[str], str]:
|
||||
try:
|
||||
from litellm._logging import redact_secrets
|
||||
except ImportError:
|
||||
return lambda text: text
|
||||
return redact_secrets
|
||||
|
||||
|
||||
def _url_passwords(url: str) -> frozenset[str]:
|
||||
try:
|
||||
parts: Final = urlsplit(url)
|
||||
except ValueError:
|
||||
return frozenset()
|
||||
query_pairs: Final = tuple(pair.partition("=") for pair in parts.query.split("&"))
|
||||
raw_query_passwords: Final = tuple(
|
||||
value for key, separator, value in query_pairs if separator and key.lower() in _PASSWORD_QUERY_KEYS
|
||||
)
|
||||
raw_passwords: Final = ((parts.password,) if parts.password else ()) + raw_query_passwords
|
||||
return frozenset(password for password in raw_passwords + tuple(map(unquote, raw_passwords)) if password)
|
||||
|
||||
|
||||
def _configured_database_passwords() -> frozenset[str]:
|
||||
database_url: Final = os.getenv("DATABASE_URL")
|
||||
direct_url: Final = os.getenv("DIRECT_URL")
|
||||
database_passwords: Final = _url_passwords(database_url) if database_url else frozenset()
|
||||
direct_passwords: Final = _url_passwords(direct_url) if direct_url else frozenset()
|
||||
return database_passwords | direct_passwords
|
||||
|
||||
|
||||
def _redact_credentials(text: str) -> str:
|
||||
"""Mask configured database passwords before passing the text to LiteLLM redaction."""
|
||||
passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True)
|
||||
alternation: Final = "|".join(re.escape(password) for password in passwords)
|
||||
password_pattern: Final = (
|
||||
re.compile(rf"(?P<lead>:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE)
|
||||
if passwords
|
||||
else None
|
||||
)
|
||||
result: Final = password_pattern.sub(rf"\g<lead>{_REDACTED}", text) if password_pattern is not None else text
|
||||
return _secret_shape_redactor()(result)
|
||||
|
||||
|
||||
def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
|
||||
if isinstance(command, tuple):
|
||||
return tuple(_redact_credentials(str(argument)) for argument in command)
|
||||
if isinstance(command, list):
|
||||
return [_redact_credentials(str(argument)) for argument in command]
|
||||
return _redact_credentials(str(command))
|
||||
|
||||
|
||||
def _redact_command_error(error: subprocess.CalledProcessError) -> str:
|
||||
redacted_command: Final = _redacted_command(error.cmd)
|
||||
return str(subprocess.CalledProcessError(error.returncode, redacted_command))
|
||||
|
||||
|
||||
def _get_prisma_command() -> str:
|
||||
"""Get the Prisma command to use, bypassing Python wrapper in offline mode."""
|
||||
if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
|
||||
|
|
@ -298,7 +377,8 @@ class ProxyExtrasDBManager:
|
|||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(
|
||||
f"Error creating baseline migration: {e}, {e.stderr}, {e.stdout}"
|
||||
f"Error creating baseline migration: {_redact_command_error(e)}, "
|
||||
f"{_redact_credentials(str(e.stderr))}, {_redact_credentials(str(e.stdout))}"
|
||||
)
|
||||
raise e
|
||||
|
||||
|
|
@ -336,9 +416,8 @@ class ProxyExtrasDBManager:
|
|||
pass
|
||||
|
||||
@staticmethod
|
||||
def _failed_migration_logs(migration_name: str) -> Optional[str]:
|
||||
"""Return failed migration logs, or None if the ledger is unavailable."""
|
||||
database_url = os.getenv("DATABASE_URL")
|
||||
def _read_migration_ledger(query: str, params: tuple[str, ...]) -> "tuple[object, ...] | None":
|
||||
database_url: Final = os.getenv("DATABASE_URL")
|
||||
if not database_url:
|
||||
return None
|
||||
|
||||
|
|
@ -347,28 +426,37 @@ class ProxyExtrasDBManager:
|
|||
except ImportError:
|
||||
return None
|
||||
|
||||
cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
|
||||
ledger_table = psycopg.sql.SQL("{}.{}").format(
|
||||
psycopg.sql.Identifier(
|
||||
ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
|
||||
),
|
||||
cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
|
||||
ledger_table: Final = psycopg.sql.SQL("{}.{}").format(
|
||||
psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"),
|
||||
psycopg.sql.Identifier("_prisma_migrations"),
|
||||
)
|
||||
try:
|
||||
with psycopg.connect(
|
||||
cleaned_url, connect_timeout=10, autocommit=True
|
||||
) as conn:
|
||||
row = conn.execute(
|
||||
psycopg.sql.SQL(
|
||||
"SELECT logs FROM {} "
|
||||
"WHERE migration_name = %s AND finished_at IS NULL "
|
||||
"AND rolled_back_at IS NULL"
|
||||
).format(ledger_table),
|
||||
(migration_name,),
|
||||
).fetchone()
|
||||
with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
|
||||
row: Final = conn.execute(psycopg.sql.SQL(query).format(ledger_table), params).fetchone()
|
||||
except (psycopg.OperationalError, psycopg.DatabaseError):
|
||||
return None
|
||||
return (row[0] or "") if row else ""
|
||||
return tuple(row) if row is not None else ()
|
||||
|
||||
@staticmethod
|
||||
def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
|
||||
row: Final = ProxyExtrasDBManager._read_migration_ledger(
|
||||
"SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
|
||||
"AND finished_at IS NULL AND rolled_back_at IS NULL",
|
||||
(migration_name, started_at),
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return row[0] if row and isinstance(row[0], str) else ""
|
||||
|
||||
@staticmethod
|
||||
def _failed_migration_recovered(migration_name: str, started_at: str) -> bool:
|
||||
row: Final = ProxyExtrasDBManager._read_migration_ledger(
|
||||
"SELECT 1 FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
|
||||
"AND (finished_at IS NOT NULL OR rolled_back_at IS NOT NULL)",
|
||||
(migration_name, started_at),
|
||||
)
|
||||
return bool(row)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_specific_migration(migration_name: str):
|
||||
|
|
@ -689,30 +777,43 @@ class ProxyExtrasDBManager:
|
|||
|
||||
@staticmethod
|
||||
def _strip_prisma_query_params(url: str) -> str:
|
||||
"""Remove Prisma-specific query params (connection_limit, pool_timeout,
|
||||
schema, etc.) from DATABASE_URL so psycopg can parse it."""
|
||||
"""Rewrite a Prisma-dialect URL for libpq: drop the Prisma-only params
|
||||
(connection_limit, pool_timeout, schema, pgbouncer, sslaccept, ...) and
|
||||
translate Prisma's TLS params back, since libpq reads ``sslcert`` as a
|
||||
client certificate where Prisma reads it as the CA."""
|
||||
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
parsed: Final = urlparse(url)
|
||||
if not parsed.query:
|
||||
return url
|
||||
libpq_params = {
|
||||
"sslmode",
|
||||
"sslcert",
|
||||
"sslkey",
|
||||
"sslrootcert",
|
||||
"sslpassword",
|
||||
"application_name",
|
||||
"connect_timeout",
|
||||
"client_encoding",
|
||||
"options",
|
||||
"service",
|
||||
"gssencmode",
|
||||
"krbsrvname",
|
||||
"target_session_attrs",
|
||||
}
|
||||
kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
|
||||
return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote)))
|
||||
pairs: Final = tuple(parse_qsl(parsed.query))
|
||||
kept: Final = tuple((k, v) for k, v in pairs if k in LIBPQ_URL_PARAMS)
|
||||
sslaccept: Final = next((v for k, v in pairs if k == "sslaccept"), None)
|
||||
libpq_pairs: Final = ProxyExtrasDBManager._libpq_tls_params(kept, sslaccept)
|
||||
return urlunparse(parsed._replace(query=urlencode(libpq_pairs, quote_via=quote)))
|
||||
|
||||
@staticmethod
|
||||
def _libpq_tls_params(
|
||||
pairs: "tuple[tuple[str, str], ...]", sslaccept: "str | None"
|
||||
) -> "tuple[tuple[str, str], ...]":
|
||||
"""Undo ``translate_libpq_ssl_params``. Prisma's ``sslcert`` is the CA and
|
||||
``sslaccept=strict`` checks chain and hostname, which libpq only does in
|
||||
``sslmode=verify-full``, so strict becomes ``sslrootcert`` plus
|
||||
``verify-full`` whatever ``sslmode`` said (``disable`` stays off). Prisma
|
||||
defaults an absent ``sslaccept`` to ``accept_invalid_certs`` and anything
|
||||
else to strict. Without strict it checks nothing, so the CA is dropped and
|
||||
``sslmode`` is kept as is: libpq only verifies when a root cert is present.
|
||||
A URL that also carries ``sslkey`` is libpq's own client-certificate form
|
||||
and is kept."""
|
||||
keys: Final = frozenset(k for k, _ in pairs)
|
||||
if "sslcert" not in keys or "sslkey" in keys:
|
||||
return pairs
|
||||
sslmode: Final = next((v for k, v in pairs if k == "sslmode"), None)
|
||||
rest: Final = tuple((k, v) for k, v in pairs if k not in ("sslcert", "sslmode"))
|
||||
if sslaccept in (None, "accept_invalid_certs") or sslmode == "disable":
|
||||
return rest if sslmode is None else rest + (("sslmode", sslmode),)
|
||||
root_cert: Final = tuple(("sslrootcert", v) for k, v in pairs if k == "sslcert" and "sslrootcert" not in keys)
|
||||
return rest + root_cert + (("sslmode", "verify-full"),)
|
||||
|
||||
@staticmethod
|
||||
def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
|
||||
|
|
@ -1072,6 +1173,11 @@ class ProxyExtrasDBManager:
|
|||
return match.group(1) if match else None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _v2_failed_migration_started_at(stderr: str, migration_name: str) -> "str | None":
|
||||
match: Final = re.search(rf"`{re.escape(migration_name)}` migration started at ([^\r\n]+?) failed", stderr)
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _v2_roll_back_migration_best_effort(migration_name: str) -> None:
|
||||
from litellm_proxy_extras.migration_lock import migration_environment
|
||||
|
|
@ -1100,8 +1206,11 @@ class ProxyExtrasDBManager:
|
|||
|
||||
if "P3009" in stderr:
|
||||
migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr)
|
||||
if migration_name:
|
||||
ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name)
|
||||
started_at: Final = (
|
||||
ProxyExtrasDBManager._v2_failed_migration_started_at(stderr, migration_name) if migration_name else None
|
||||
)
|
||||
if migration_name and started_at:
|
||||
ledger_logs: Final = ProxyExtrasDBManager._failed_migration_logs(migration_name, started_at)
|
||||
if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs:
|
||||
logger.info(
|
||||
"Migration %s failed in a concurrent migrate deploy "
|
||||
|
|
@ -1110,6 +1219,14 @@ class ProxyExtrasDBManager:
|
|||
)
|
||||
ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)
|
||||
return budget.spend()
|
||||
if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at):
|
||||
logger.info(
|
||||
"Migration %s started at %s was already rolled back or completed by a concurrent "
|
||||
"migrate deploy, retrying",
|
||||
migration_name,
|
||||
started_at,
|
||||
)
|
||||
return budget.spend()
|
||||
raise RuntimeError(
|
||||
"Migration completion could not be verified. LiteLLM startup has stopped.\n\n"
|
||||
f"Prisma migration history (migration name and start time):\n{stderr}\n\n"
|
||||
|
|
@ -1518,6 +1635,11 @@ class ProxyExtrasDBManager:
|
|||
f"Error: {stderr}"
|
||||
)
|
||||
raise
|
||||
else:
|
||||
logger.error(
|
||||
"prisma migrate deploy failed with an error the resolver does not handle: "
|
||||
f"{_redact_credentials(stderr)}"
|
||||
)
|
||||
else:
|
||||
if ProxyExtrasDBManager.spend_logs_is_partitioned():
|
||||
raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
|
||||
|
|
@ -1532,7 +1654,7 @@ class ProxyExtrasDBManager:
|
|||
)
|
||||
return True
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(
|
||||
logger.error(
|
||||
"Attempt %s timed out. Raise %s if this database needs longer to apply its schema.",
|
||||
attempt + 1,
|
||||
PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR,
|
||||
|
|
@ -1545,7 +1667,12 @@ class ProxyExtrasDBManager:
|
|||
if attempts_left > 0
|
||||
else ""
|
||||
)
|
||||
logger.info(f"The process failed to execute. Details: {e}.{retry_msg}")
|
||||
stderr_detail: Final = (
|
||||
f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else ""
|
||||
)
|
||||
logger.error(
|
||||
f"The process failed to execute. Details: {_redact_command_error(e)}.{stderr_detail}{retry_msg}"
|
||||
)
|
||||
time.sleep(random.randrange(5, 15))
|
||||
finally:
|
||||
os.chdir(original_dir)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.103"
|
||||
version = "0.4.105"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.103"
|
||||
version = "0.4.105"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
105
litellm-rust/Cargo.lock
generated
|
|
@ -97,6 +97,53 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d"
|
||||
|
||||
[[package]]
|
||||
name = "askama"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6024d73179f43f15ccd2b881bfea6fee7f3a46ec53f33b52210dea749ebebaa4"
|
||||
dependencies = [
|
||||
"askama_macros",
|
||||
"itoa",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "askama_derive"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071ee5ebf2138e3ad180e0aacf6940c2cab5e6d8333741d9925c7bee2b153f39"
|
||||
dependencies = [
|
||||
"askama_parser",
|
||||
"memchr",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc-hash",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "askama_macros"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "643e1c7cbb6aec1d920332fe51a7c0d8219e273dcb8602db03f5263e4d16487b"
|
||||
dependencies = [
|
||||
"askama_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "askama_parser"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c5ae75772275d268b03ab8bdccdd12117b6169ee23256942b34e46c9f476583"
|
||||
dependencies = [
|
||||
"rustc-hash",
|
||||
"unicode-ident",
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "asn1-rs"
|
||||
version = "0.7.2"
|
||||
|
|
@ -4089,6 +4136,8 @@ dependencies = [
|
|||
"litellm-storage-clickhouse",
|
||||
"litellm-token-counter",
|
||||
"litellm-traces",
|
||||
"litellm-traces-cache",
|
||||
"litellm-traces-clickhouse",
|
||||
"litellm-tracing",
|
||||
"prost",
|
||||
"pyo3",
|
||||
|
|
@ -4101,6 +4150,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"serde_with",
|
||||
"sha2 0.10.9",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
|
|
@ -4299,9 +4349,13 @@ dependencies = [
|
|||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sqlx",
|
||||
"testcontainers-modules",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"url",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4384,22 +4438,67 @@ dependencies = [
|
|||
name = "litellm-traces"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"askama",
|
||||
"base64 0.22.1",
|
||||
"criterion",
|
||||
"flate2",
|
||||
"litellm-http",
|
||||
"litellm-storage-clickhouse",
|
||||
"indexmap 2.14.0",
|
||||
"litellm-llms-types",
|
||||
"macro_rules_attribute",
|
||||
"opentelemetry-proto",
|
||||
"prost",
|
||||
"rstest",
|
||||
"schemars 1.2.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-traces-cache"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"litellm-traces",
|
||||
"moka",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-traces-clickhouse"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"askama",
|
||||
"flate2",
|
||||
"futures-util",
|
||||
"hmac 0.12.1",
|
||||
"jsonschema",
|
||||
"litellm-http",
|
||||
"litellm-storage-clickhouse",
|
||||
"litellm-traces",
|
||||
"litellm-traces-cache",
|
||||
"macro_rules_attribute",
|
||||
"moka",
|
||||
"rstest",
|
||||
"schemars 1.2.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"testcontainers-modules",
|
||||
"thiserror 2.0.19",
|
||||
"time",
|
||||
"tokio",
|
||||
"url",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ litellm-config = { path = "crates/config" }
|
|||
litellm-router = { path = "crates/router" }
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
litellm-traces = { path = "crates/traces" }
|
||||
litellm-traces-cache = { path = "crates/traces-cache" }
|
||||
litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
|
||||
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-gateway-mcp = { path = "crates/gateway-mcp" }
|
||||
|
|
@ -63,6 +65,7 @@ litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
|
|||
litellm-host-python = { path = "crates/host-python" }
|
||||
litellm-python-compat = { path = "crates/python-compat" }
|
||||
|
||||
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
|
||||
tracing = "0.1"
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }
|
||||
axum-login = "0.18.0"
|
||||
|
|
@ -93,7 +96,10 @@ serde = { version = "1.0", features = ["derive"] }
|
|||
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
||||
serde_with = { version = "=3.16.1", default-features = false, features = ["std", "macros"] }
|
||||
sha2 = "0.10"
|
||||
syn = { version = "2", default-features = false }
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
|
||||
proc-macro2 = "1"
|
||||
quote = "1"
|
||||
subtle = "2"
|
||||
thiserror = "2.0"
|
||||
tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@ use serde::Deserialize;
|
|||
pub use error::Error;
|
||||
pub use mcp::{McpAuth, McpServer, McpTransport};
|
||||
pub use model::{LiteLlmParams, Model};
|
||||
pub use settings::{GeneralSettings, LiteLlmSettings, RouterSettings};
|
||||
pub use settings::{
|
||||
ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings,
|
||||
TracingStoreSettings,
|
||||
};
|
||||
pub use value::{AdditionalFields, Flag, NumberOrString, Object, OneOrMany, Value};
|
||||
|
||||
#[derive(Clone, Default, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -5,6 +5,47 @@ use serde::Deserialize;
|
|||
|
||||
use crate::{AdditionalFields, Flag, NumberOrString, Object, OneOrMany, Value};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TracingStoreKind {
|
||||
Clickhouse,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ClickHouseStoreSettings {
|
||||
#[serde(rename = "type")]
|
||||
pub kind: TracingStoreKind,
|
||||
pub url: Option<SecretValue>,
|
||||
pub database: Option<String>,
|
||||
pub retention_days: Option<NumberOrString>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for ClickHouseStoreSettings {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("ClickHouseStoreSettings")
|
||||
.field("kind", &self.kind)
|
||||
.field("database", &self.database)
|
||||
.field("retention_days", &self.retention_days)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TracingStoreSettings {
|
||||
ClickHouse(ClickHouseStoreSettings),
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Debug, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct TracingSettings {
|
||||
pub store: Option<TracingStoreSettings>,
|
||||
#[serde(flatten)]
|
||||
pub additional_fields: AdditionalFields,
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct GeneralSettings {
|
||||
|
|
@ -14,6 +55,7 @@ pub struct GeneralSettings {
|
|||
pub admission_queue_timeout_seconds: f64,
|
||||
pub master_key: Option<SecretValue>,
|
||||
pub database_url: Option<SecretValue>,
|
||||
pub tracing: Option<TracingSettings>,
|
||||
pub database_connection_pool_limit: Option<u64>,
|
||||
pub database_connection_timeout: Option<f64>,
|
||||
pub database_connect_timeout: Option<f64>,
|
||||
|
|
@ -50,6 +92,7 @@ impl Default for GeneralSettings {
|
|||
admission_queue_timeout_seconds: 1.0,
|
||||
master_key: None,
|
||||
database_url: None,
|
||||
tracing: None,
|
||||
database_connection_pool_limit: Some(10),
|
||||
database_connection_timeout: Some(60.0),
|
||||
database_connect_timeout: None,
|
||||
|
|
@ -97,6 +140,7 @@ impl fmt::Debug for GeneralSettings {
|
|||
)
|
||||
.field("master_key", &self.master_key)
|
||||
.field("database_url", &self.database_url)
|
||||
.field("tracing", &self.tracing)
|
||||
.field("store_model_in_db", &self.store_model_in_db)
|
||||
.field("additional_fields", &self.additional_fields.keys())
|
||||
.finish_non_exhaustive()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use litellm_config::{Config, Error, Flag, NumberOrString};
|
||||
use litellm_config::{Config, Error, Flag, NumberOrString, TracingStoreSettings};
|
||||
use rstest::{fixture, rstest};
|
||||
use tempfile::TempDir;
|
||||
|
||||
|
|
@ -113,6 +113,54 @@ fn missing_general_settings_has_no_master_key() {
|
|||
assert!(config.general_settings.master_key.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_are_typed_and_redact_the_url() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: https://writer:password@example.com\n database: analytics\n retention_days: 7\n",
|
||||
)
|
||||
.unwrap();
|
||||
let tracing = config.general_settings.tracing.as_ref().unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) = tracing.store.as_ref() else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.url.as_ref().unwrap().expose(),
|
||||
"https://writer:password@example.com"
|
||||
);
|
||||
assert_eq!(store.database.as_deref(), Some("analytics"));
|
||||
assert_eq!(store.retention_days, Some(NumberOrString::Number(7.0)));
|
||||
assert!(!format!("{config:?}").contains("password"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_accept_environment_references() {
|
||||
let config = Config::from_yaml(
|
||||
"general_settings:\n tracing:\n store:\n type: clickhouse\n url: os.environ/CLICKHOUSE_URL\n retention_days: os.environ/RETENTION_DAYS\n",
|
||||
)
|
||||
.unwrap();
|
||||
let Some(TracingStoreSettings::ClickHouse(store)) =
|
||||
config.general_settings.tracing.unwrap().store
|
||||
else {
|
||||
panic!("expected ClickHouse tracing store")
|
||||
};
|
||||
assert_eq!(
|
||||
store.retention_days,
|
||||
Some(NumberOrString::String(
|
||||
"os.environ/RETENTION_DAYS".to_owned()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_string_store() {
|
||||
assert!(Config::from_yaml("general_settings:\n tracing:\n store: clickhouse\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracing_settings_reject_removed_reader_configuration() {
|
||||
assert!(Config::from_yaml("general_settings:\n tracing:\n store:\n type: clickhouse\n reader_url: http://localhost:8123\n").is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn empty_config_matches_python_defaults() {
|
||||
let config = Config::from_yaml("{}").unwrap();
|
||||
|
|
|
|||