scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

chore: merge main into the stale cost map boot branch

Browse source 
main now pins method and api_key on the recorded request itself, so the
branch keeps main's copy and no longer carries a change of its own.
This commit is contained in:
yuneng-berri 2026-10-06 08:56:08 +00:00
commit 26479d04ed
No known key found for this signature in database
1338 changed files with 85828 additions and 28896 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

209
.circleci/config.yml
View file

@ -38,7 +38,7 @@ commands:
parameters:
category:
type: enum
enum: ["backend", "client", "provider-harness"]
enum: ["backend", "client", "provider-harness", "redis-compat"]
default: "backend"
steps:
- run:
@ -229,6 +229,19 @@ commands:
- wait_for_service:
url: tcp://localhost:6379
timeout: "60"
install_codecov_cli:
steps:
- run:
name: Install Codecov CLI (pinned v11.3.1)
when: always
command: |
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
(cd /tmp && sha256sum -c codecov.SHA256SUM)
chmod +x /tmp/codecov
mkdir -p "$HOME/.local/bin"
mv /tmp/codecov "$HOME/.local/bin/codecov"
start_openai_record_replay_proxy:
description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream/<host>/ so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced."
steps:
@ -3279,30 +3292,164 @@ jobs:
- store_artifacts:
path: test-results
unit:
postgres_suite:
parameters:
test_path:
type: string
seed:
type: boolean
coverage_flag:
type: string
default: ""
timeout_minutes:
type: integer
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
parallelism: 4
environment:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- start_postgres:
db_name: litellm_test
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- when:
condition: << parameters.seed >>
steps:
- run:
name: Seed database schema
command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Run << parameters.test_path >>
command: |
mkdir -p test-results
coverage_args=()
if [ -n "<< parameters.coverage_flag >>" ]; then
coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml)
fi
timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \
<< parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \
--junitxml=test-results/junit.xml "${coverage_args[@]}"
- when:
condition: << parameters.coverage_flag >>
steps:
- install_codecov_cli
- run:
name: Upload coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \
-n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
mcp_integration:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
environment:
COVERAGE_CORE: sysmon
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- checkout
- skip_if_unrelated_changes
- setup_litellm_test_deps
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- run:
name: Run unit tests
name: Install MCP SDK1 peer
command: |
mkdir -p test-results/unit
shard="$(find tests/unit -name 'test_*.py' | sort | circleci tests split --split-by=timings --timings-type=filename)"
if [ -z "${shard}" ]; then echo "shard ${CIRCLE_NODE_INDEX} received no tests/unit files; nothing to run"; exit 0; fi
mapfile -t files < <(printf '%s\n' "${shard}")
set +e
LITELLM_LOCAL_MODEL_COST_MAP=True uv run --no-sync pytest "${files[@]}" -p no:rerunfailures -p no:pytest-retry --timeout=90 -n 4 --dist=loadscope --tb=short -o junit_family=xunit1 --junitxml=test-results/unit/junit.xml
status=$?
set -e
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from tests/unit; passing"; exit 0; fi
exit "$status"
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- run:
name: Run MCP integration tests
command: |
mkdir -p test-results
env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \
timeout --signal=TERM 20m uv run --no-sync pytest \
tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \
tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \
--reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \
--cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- install_codecov_cli
- run:
name: Upload MCP integration coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F mcp-integration -C "$CIRCLE_SHA1" \
-n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
redis_compat:
parameters:
redis_py:
type: string
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: redis-compat
- setup_litellm_test_deps
- run:
name: Pin redis-py version
command: |
uv pip install "redis==<< parameters.redis_py >>"
uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)"
- run:
name: Install redis-server 7.2.16
command: |
mkdir -p "$HOME/.local/bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server"
docker rm "$cid"
redis-server --version | grep -q 'v=7.2.16'
- run:
name: Run Redis compatibility tests
command: |
mkdir -p test-results
env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \
timeout --signal=TERM 15m uv run --no-sync pytest \
tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
--tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \
--cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml
- when:
condition:
equal: ["5.3.1", << parameters.redis_py >>]
steps:
- install_codecov_cli
- run:
name: Upload Redis compatibility coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; }
codecov upload-process --disable-search --fail-on-error -f coverage.xml \
-F redis-compat -C "$CIRCLE_SHA1" \
-n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
@ -3376,6 +3523,39 @@ workflows:
parameters:
suite: [management, database]
mode: [replica]
- postgres_suite:
name: proxy-behavior
test_path: tests/proxy_behavior
seed: true
coverage_flag: lens-postgres
timeout_minutes: 25
- postgres_suite:
name: proxy-security
test_path: tests/proxy_security_tests
seed: true
timeout_minutes: 15
- postgres_suite:
name: schema-migration
test_path: tests/proxy_migration_tests
seed: false
timeout_minutes: 20
- postgres_suite:
name: roi-database
test_path: tests/integration/database/test_roi_observed.py
seed: false
coverage_flag: roi-postgres
timeout_minutes: 10
- mcp_integration:
name: mcp-integration
- redis_compat:
name: redis-compat-<< matrix.redis_py >>
matrix:
parameters:
redis_py:
- "5.3.1"
- "6.4.0"
- "7.4.1"
- "8.0.1"
build_and_test:
unless:
or:
@ -3385,7 +3565,6 @@ workflows:
jobs:
- using_litellm_on_windows
- windows_release_wheel
- unit
- provider_replay_harness
- base_sdk_install
- local_testing_part1

10
.circleci/scripts/classify_changes.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release>}"
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
has_client=false
has_backend=false
@ -10,9 +10,14 @@ has_provider_harness=false
has_cost_map=false
has_mcp_dependencies=false
has_windows_release=false
has_redis_compat=false
outside_cost_map_set=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
has_redis_compat=true ;;
esac
case "$file" in
*.md | *.mdx) : ;;
pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
@ -54,6 +59,9 @@ case "$category" in
windows-release)
[ "$has_windows_release" = true ] && echo run || echo skip
;;
redis-compat)
[ "$has_redis_compat" = true ] && echo run || echo skip
;;
backend)
[ "$has_backend" = true ] && echo run || echo skip
;;

2
.circleci/scripts/path_filter.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: path_filter.sh <backend|client|provider-harness>}"
category="${1:?usage: path_filter.sh <backend|client|provider-harness|redis-compat>}"
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
run_full() {

187
.circleci/scripts/unit_selection.sh
View file

@ -1,187 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
flag="${1:?usage: unit_selection.sh <codecov flag>}"
legacy_flags=(
caching-local
core-utils
enterprise-package
enterprise-routing
integrations
llm-other-providers
llm-vertex-ai
mcp-integration
misc
proxy-db-auth-checks
proxy-db-budgets
proxy-db-custom-logging
proxy-db-db-and-spend
proxy-db-endpoints-and-responses
proxy-db-guardrails-hooks
proxy-db-jwt-and-keys
proxy-db-key-generation
proxy-db-logging-misc
proxy-db-proxy-runtime
proxy-db-proxy-server-core
proxy-db-proxy-utils
proxy-extras
proxy-infra
responses-caching-types
)
legacy_paths() {
case "$1" in
caching-local) echo tests/unit/caching ;;
core-utils) echo tests/unit/litellm_core_utils ;;
enterprise-package)
echo tests/unit/enterprise/integrations
echo tests/unit/enterprise/proxy/auth
echo tests/unit/enterprise/proxy/guardrails
echo tests/unit/enterprise/proxy/hooks
echo tests/unit/enterprise/proxy/management_endpoints
echo tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
echo tests/unit/enterprise/proxy/test_liteadmin.py
echo tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py ;;
enterprise-routing)
echo tests/unit/google_genai
echo tests/unit/router_strategy
echo tests/unit/router_utils
echo tests/unit/proxy/common_utils/test_cache_aware_routing.py
echo tests/unit/enterprise/enterprise_callbacks/send_emails
echo tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
echo tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
echo tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
echo tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
echo tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
echo tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
echo tests/unit/enterprise/proxy/test_enterprise_routes.py
echo tests/unit/enterprise/proxy/test_file_deletion_blocking.py
echo tests/unit/enterprise/proxy/test_managed_files_access_check.py
echo tests/unit/enterprise/proxy/test_managed_files_hook.py ;;
integrations) echo tests/unit/integrations ;;
llm-other-providers) find tests/unit/llms -name 'test_*.py' -not -path 'tests/unit/llms/vertex_ai/*' ;;
llm-vertex-ai) echo tests/unit/llms/vertex_ai ;;
mcp-integration)
echo tests/unit/experimental_mcp_client
echo tests/unit/proxy/_experimental/mcp_server
echo tests/unit/responses/mcp
echo tests/mcp_tests/test_proxy_mcp_e2e.py ;;
misc)
find tests/unit -maxdepth 1 -name 'test_*.py'
echo tests/unit/test_router
echo tests/unit/a2a_protocol
echo tests/unit/batches
echo tests/unit/chat_completions
echo tests/unit/completion_extras
echo tests/unit/containers
echo tests/unit/embeddings
echo tests/unit/endpoints
echo tests/unit/files
echo tests/unit/harness
echo tests/unit/images
echo tests/unit/interactions
echo tests/unit/messages
echo tests/unit/rag
echo tests/unit/rerank_api
echo tests/unit/rust_bridge
echo tests/unit/secret_managers
echo tests/unit/vector_stores
echo tests/unit/videos ;;
proxy-db-auth-checks)
echo tests/unit/proxy/auth/test_auth_checks.py
echo tests/unit/proxy/auth/test_user_api_key_auth.py
echo tests/unit/proxy/test_credential_slot_registry.py
echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
proxy-db-budgets)
echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
echo tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
echo tests/unit/proxy/test_zero_cost_model_budget_bypass.py ;;
proxy-db-custom-logging)
echo tests/unit/proxy/test_custom_callback_input.py
echo tests/unit/proxy/test_custom_logger_s3_gcs.py ;;
proxy-db-db-and-spend)
echo tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
echo tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
echo tests/unit/proxy/db/test_update_daily_tag_spend.py
echo tests/unit/proxy/test_db_schema_changes.py
echo tests/unit/proxy/test_prisma_client_backoff_retry.py
echo tests/unit/proxy/test_update_spend.py
echo tests/unit/skills/test_skills_db.py ;;
proxy-db-endpoints-and-responses)
echo tests/unit/proxy/lens
echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
echo tests/unit/proxy/common_utils/test_check_batch_cost.py
echo tests/unit/proxy/common_utils/test_check_responses_cost.py
echo tests/unit/proxy/common_utils/test_realtime_cache.py
echo tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
echo tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
echo tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
echo tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
echo tests/unit/proxy/response_polling
echo tests/unit/proxy/test_custom_tokenizer_bug.py
echo tests/unit/proxy/test_get_favicon.py
echo tests/unit/proxy/test_get_image.py
echo tests/unit/proxy/test_prompt_test_endpoint.py
echo tests/unit/proxy/test_reducto_ocr_route.py
echo tests/unit/proxy/test_response_polling_pre_call_checks.py
echo tests/unit/proxy/test_ui_path_detection.py ;;
proxy-db-guardrails-hooks)
echo tests/unit/proxy/hooks/test_banned_keyword_list.py
echo tests/unit/proxy/test_proxy_setting_guardrails.py
echo tests/unit/proxy/test_unit_test_proxy_hooks.py ;;
proxy-db-jwt-and-keys)
echo tests/unit/proxy/auth/test_jwt.py
echo tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
echo tests/unit/proxy/test_proxy_custom_auth.py ;;
proxy-db-key-generation) echo tests/unit/proxy/management_endpoints/test_key_generate_prisma.py ;;
proxy-db-logging-misc)
echo tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
echo tests/unit/proxy/spend_tracking/test_search_api_logging.py
echo tests/unit/proxy/test_proxy_reject_logging.py ;;
proxy-db-proxy-runtime)
echo tests/unit/proxy/auth/test_multipart_bypass_repro.py
echo tests/unit/proxy/auth/test_proxy_routes.py
echo tests/unit/proxy/middleware/test_request_size_limit_middleware.py
echo tests/unit/proxy/test_proxy_config_unit_test.py
echo tests/unit/proxy/test_proxy_token_counter.py
echo tests/unit/proxy/test_server_root_path.py ;;
proxy-db-proxy-server-core)
echo tests/unit/proxy/test__lazy_features.py
echo tests/unit/proxy/test_aproxy_startup.py
echo tests/unit/proxy/test_proxy_server.py ;;
proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
proxy-extras) echo tests/unit/litellm_proxy_extras ;;
proxy-infra)
echo tests/unit/gateway
echo tests/unit/proxy/management
echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
echo tests/unit/proxy/roi_calculator ;;
responses-caching-types)
find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
echo tests/unit/types ;;
*) echo "unit_selection.sh: unknown flag $1" >&2; exit 1 ;;
esac
}
expand() {
while read -r path; do
if [ -d "$path" ]; then
find "$path" -name 'test_*.py'
elif [ -f "$path" ]; then
echo "$path"
else
echo "unit_selection.sh: $path does not exist" >&2
exit 1
fi
done
}
if [ "$flag" = unit ]; then
comm -23 \
<(find tests/unit -name 'test_*.py' | sort) \
<(for legacy in "${legacy_flags[@]}"; do legacy_paths "$legacy"; done | expand | sort)
exit 0
fi
legacy_paths "$flag" | expand | sort

416
.circleci/tests.yml
View file

@ -1,416 +0,0 @@
version: 2.1
commands:
wait_for_service:
parameters:
url:
type: string
timeout:
type: string
default: "60"
steps:
- run:
name: "Wait for << parameters.url >>"
command: |
TIMEOUT=<< parameters.timeout >>
URL="<< parameters.url >>"
ELAPSED=0
echo "Waiting up to ${TIMEOUT}s for ${URL} ..."
if echo "$URL" | grep -q '^tcp://'; then
HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)
PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)
while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do
sleep 2; ELAPSED=$((ELAPSED+2))
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
done
else
while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do
sleep 2; ELAPSED=$((ELAPSED+2))
if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi
done
fi
echo "Service ready after ${ELAPSED}s"
install_uv:
steps:
- run:
name: Install uv (pinned 0.10.9)
command: |
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
install_rust:
steps:
- run:
name: Install Rust (rustup 1.28.2, toolchain 1.98.0)
command: |
case "$(uname -m)" in
x86_64)
RUSTUP_TRIPLE=x86_64-unknown-linux-gnu
RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c
;;
aarch64)
RUSTUP_TRIPLE=aarch64-unknown-linux-gnu
RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c
;;
*)
echo "install_rust: unsupported architecture $(uname -m)" >&2
exit 1
;;
esac
curl -sSLf -o /tmp/rustup-init \
"https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init"
echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c -
chmod +x /tmp/rustup-init
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0
rm -f /tmp/rustup-init
echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.cargo/bin:$PATH"
rustc --version
cargo --version
install_codecov_cli:
steps:
- run:
name: Install Codecov CLI (pinned v11.3.1)
when: always
command: |
curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov
curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM
[ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ]
(cd /tmp && sha256sum -c codecov.SHA256SUM)
chmod +x /tmp/codecov
mkdir -p "$HOME/.local/bin"
mv /tmp/codecov "$HOME/.local/bin/codecov"
setup_litellm_enterprise_pip:
steps:
- run:
name: "Install local version of litellm-enterprise"
command: |
uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"
setup_test_deps:
steps:
- install_uv
- install_rust
- restore_cache:
keys:
- v3-integration-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- save_cache:
paths:
- ~/.cache/uv
key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
- run:
name: Generate Prisma client
command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
skip_unless_relevant:
parameters:
category:
type: string
default: backend
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
steps:
- run:
name: "Skip job when no << parameters.category >>-relevant files changed"
command: |
export CIRCLE_PULL_REQUEST="${CIRCLE_PULL_REQUEST:-<< parameters.pull_request_url >>}"
export PATH_FILTER_BASE_BRANCH="<< parameters.base_ref >>"
[ -n "$PATH_FILTER_BASE_BRANCH" ] || unset PATH_FILTER_BASE_BRANCH
bash .circleci/scripts/path_filter.sh << parameters.category >>
start_postgres:
parameters:
db_name:
type: string
default: circle_test
image:
type: string
default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
steps:
- run:
name: Start PostgreSQL
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=<< parameters.db_name >> \
-p 5432:5432 \
<< parameters.image >>
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
start_redis:
steps:
- run:
name: Start Redis
command: |
docker run -d \
--name redis-cache \
-p 6379:6379 \
redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
- wait_for_service:
url: tcp://localhost:6379
timeout: "60"
jobs:
unit:
parameters:
flag:
type: string
default: unit
shards:
type: integer
default: 6
workers:
type: integer
default: 4
dist:
type: string
default: loadscope
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
legacy_mcp_peer:
type: boolean
default: false
reruns:
type: integer
default: 0
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
parallelism: << parameters.shards >>
environment:
COVERAGE_CORE: sysmon
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
- checkout
- skip_unless_relevant:
base_ref: << parameters.base_ref >>
pull_request_url: << parameters.pull_request_url >>
- setup_test_deps
- when:
condition: << parameters.legacy_mcp_peer >>
steps:
- run:
name: Install MCP SDK1 peer
command: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
- run:
name: "Run << parameters.flag >> shard"
no_output_timeout: 20m
command: |
mkdir -p test-results/<< parameters.flag >>
selection="$(bash .circleci/scripts/unit_selection.sh << parameters.flag >>)" || { echo "unit_selection.sh failed for << parameters.flag >>"; exit 1; }
[ -n "${selection}" ] || { echo "unit_selection.sh produced no files for << parameters.flag >>"; exit 1; }
shard="$(printf '%s\n' "${selection}" | circleci tests split --split-by=timings --timings-type=filename)" || { echo "circleci tests split failed for << parameters.flag >>"; exit 1; }
[ -n "${shard}" ] || { echo "shard ${CIRCLE_NODE_INDEX} received no << parameters.flag >> files; nothing to run"; exit 0; }
mapfile -t files < <(printf '%s\n' "${shard}")
xdist_args=()
if [ "<< parameters.workers >>" -gt 0 ]; then xdist_args=(-n << parameters.workers >> --dist=<< parameters.dist >>); fi
rerun_args=(-p no:rerunfailures)
if [ "<< parameters.reruns >>" -gt 0 ]; then rerun_args=(--reruns << parameters.reruns >> --reruns-delay 1 --rerun-except "from pytest-timeout"); fi
test_env=(PATH="$PATH" HOME="$HOME" CI=true COVERAGE_CORE="$COVERAGE_CORE" LITELLM_LOCAL_MODEL_COST_MAP="$LITELLM_LOCAL_MODEL_COST_MAP")
if [ -n "${MCP_TEST_PEER_PYTHON:-}" ]; then test_env+=(MCP_TEST_PEER_PYTHON="$MCP_TEST_PEER_PYTHON"); fi
set +e
env -i "${test_env[@]}" \
uv run --no-sync pytest "${files[@]}" "${rerun_args[@]}" -p no:pytest-retry --timeout=90 "${xdist_args[@]}" --tb=short --durations=20 -o junit_family=xunit1 --junitxml=test-results/<< parameters.flag >>/junit.xml --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml --cov-config=pyproject.toml
status=$?
set -e
if [ "$status" -eq 5 ]; then echo "pytest collected no tests from the shard; passing"; exit 0; fi
exit "$status"
- install_codecov_cli
- run:
name: Upload coverage
when: always
command: |
[ -f coverage.xml ] || { echo "no coverage.xml produced; skipping upload"; exit 0; }
codecov upload-process --disable-search -f coverage.xml -F << parameters.flag >> -C "$CIRCLE_SHA1" -n "<< parameters.flag >>-${CIRCLE_NODE_INDEX}-${CIRCLE_BUILD_NUM}" --git-service github
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
- store_artifacts:
path: coverage.xml
documentation:
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- setup_test_deps
- run:
name: Checkout litellm-docs
command: rm -rf docs/my-website && git clone --depth 1 https://github.com/BerriAI/litellm-docs.git docs/my-website
- run:
name: Run documentation validation
command: |
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
integration:
parameters:
suite:
type: string
base_ref:
type: string
default: ""
pull_request_url:
type: string
default: ""
machine:
image: ubuntu-2204:2024.04.1
resource_class: large
working_directory: ~/project
steps:
- checkout
- skip_unless_relevant:
base_ref: << parameters.base_ref >>
pull_request_url: << parameters.pull_request_url >>
- setup_test_deps
- start_postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
- start_redis
- run:
name: Run owned integration contracts
command: env -i PATH="$PATH" HOME="$HOME" CIRCLE_SHA1="$CIRCLE_SHA1" CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID" bash .circleci/scripts/run_integration.sh << parameters.suite >>
no_output_timeout: 15m
- run:
name: Stop owned database and Redis
when: always
command: |
mkdir -p test-results/integration-<< parameters.suite >>
docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
docker rm -f postgres-db redis-cache
test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
- store_test_results:
path: test-results
- store_artifacts:
path: test-results
workflows:
tests:
when: (pipeline.event.name == "push" and pipeline.git.branch == "main") or pipeline.event.name == "api" or (pipeline.event.name == "pull_request" and (pipeline.event.github.pull_request.base.ref == "main" or pipeline.event.github.pull_request.base.ref starts-with "litellm_"))
jobs:
- unit:
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-<< matrix.flag >>
shards: 1
workers: 2
reruns: 2
matrix:
parameters:
flag: [caching-local, proxy-extras, enterprise-routing]
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-mcp-integration
flag: mcp-integration
shards: 1
workers: 2
legacy_mcp_peer: true
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-<< matrix.flag >>
shards: 1
reruns: 2
matrix:
parameters:
flag:
- enterprise-package
- proxy-infra
- responses-caching-types
- proxy-db-auth-checks
- proxy-db-jwt-and-keys
- proxy-db-proxy-server-core
- proxy-db-proxy-runtime
- proxy-db-custom-logging
- proxy-db-logging-misc
- proxy-db-db-and-spend
- proxy-db-guardrails-hooks
- proxy-db-budgets
- proxy-db-endpoints-and-responses
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-llm-vertex-ai
flag: llm-vertex-ai
shards: 2
workers: 1
reruns: 2
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-llm-other-providers
flag: llm-other-providers
shards: 3
reruns: 2
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-core-utils
flag: core-utils
shards: 2
reruns: 1
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-integrations
flag: integrations
shards: 2
reruns: 3
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-misc
flag: misc
shards: 2
reruns: 2
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-proxy-db-proxy-utils
flag: proxy-db-proxy-utils
shards: 1
reruns: 2
dist: worksteal
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- unit:
name: unit-proxy-db-key-generation
flag: proxy-db-key-generation
shards: 1
workers: 0
reruns: 2
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>
- documentation
- integration:
name: integration-<< matrix.suite >>
matrix:
parameters:
suite: [sdk]
base_ref: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.base.ref or "" >>
pull_request_url: << pipeline.event.name == "pull_request" and pipeline.event.github.pull_request.url or "" >>

0
.github/pull_request_template.md → .github/PULL_REQUEST_TEMPLATE/general.md vendored
View file

1
.github/PULL_REQUEST_TEMPLATE/rust.md vendored Normal file
View file

@ -0,0 +1 @@
<!-- Write the final PR description as concise bullets only -->

19
.github/actions/rust-bridge/action.yml vendored Normal file
View file

@ -0,0 +1,19 @@
name: Set up the Rust bridge
description: Select the shared Rust bridge artifact or the Cargo cache
inputs:
artifact:
description: Rust bridge artifact name
required: false
default: ""
runs:
using: composite
steps:
- name: Restore the Cargo build cache
if: inputs.artifact == ''
uses: ./.github/actions/cache-cargo-build
- name: Download the Rust bridge artifact
if: inputs.artifact != ''
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e
with:
name: ${{ inputs.artifact }}
path: rust-bridge-dist

BIN
.github/assets/lens-result-retries/before-results.jpg vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 94 KiB

BIN
.github/assets/lens-result-retries/partial-results.jpg vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 94 KiB

2
.github/ci-coverage-allowlist.yml vendored
View file

@ -21,7 +21,7 @@ test_paths:
Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
Separately, test-redis-compat.yml selects two IAM cluster authentication tests in
Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in
test_caching.py by node ID. It does not run that file's other tests.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider

17
.github/e2e-stack/down.sh vendored
View file

@ -1,17 +0,0 @@
#!/usr/bin/env bash
set -uo pipefail
STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
for pid_file in "${STACK_DIR}"/pids/*.pid; do
[[ -f "${pid_file}" ]] || continue
pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
kill -TERM "$(cat "${pid_file}")" 2>/dev/null
rm -f "${pid_file}"
done
for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do
docker rm -f "${container}" >/dev/null 2>&1
done
exit 0

83
.github/e2e-stack/redact_output.py vendored
View file

@ -1,83 +0,0 @@
import argparse
import os
import sys
from functools import reduce
from pathlib import Path
from typing import Final
from xml.sax.saxutils import escape
from pydantic import JsonValue, TypeAdapter, ValidationError
from secrets_to_env import MIN_MASKED_LENGTH
REDACTED: Final = "***"
json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def string_leaves(node: JsonValue) -> tuple[str, ...]:
match node:
case str():
return (node,)
case list():
return tuple(leaf for child in node for leaf in string_leaves(child))
case dict():
return tuple(leaf for child in node.values() for leaf in string_leaves(child))
return ()
def field_lines(value: str) -> tuple[str, ...]:
try:
return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines())
except ValidationError:
return ()
def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]:
values: Final = frozenset(
line.split("=", 1)[1].strip().strip("'")
for path in values_files
for line in path.read_text().splitlines()
if "=" in line
)
texts: Final = frozenset(text for value in values for text in (value, *field_lines(value)))
renderings: Final = frozenset(
rendering
for text in texts
if len(text) >= MIN_MASKED_LENGTH
for rendering in (text, escape(text), escape(text, {'"': "&quot;"}))
)
return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering)))
def redact(text: str, values: tuple[str, ...]) -> str:
return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text)
def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None:
target: Final = out_dir / source.name
with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle:
_ = handle.write(redact(source.read_text(errors="replace"), values))
def main() -> int:
parser: Final = argparse.ArgumentParser()
_ = parser.add_argument("--values", action="append", type=Path, required=True)
_ = parser.add_argument("--out", type=Path, required=True)
_ = parser.add_argument("files", nargs="*", type=Path)
args: Final = parser.parse_args()
values_files: Final = tuple(args.values)
out_dir: Final[Path] = args.out
sources: Final = tuple(args.files)
try:
values: Final = masked_values(values_files)
out_dir.mkdir(mode=0o700, exist_ok=True)
for source in sources:
write_redacted(source, out_dir, values)
except OSError as error:
_ = sys.stderr.write(f"could not redact {error.filename}\n")
return 1
_ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n")
return 0
if __name__ == "__main__":
sys.exit(main())

55
.github/e2e-stack/secrets_to_env.py vendored
View file

@ -1,55 +0,0 @@
import os
import re
import sys
from pathlib import Path
from typing import Final
from pydantic import TypeAdapter, ValidationError
secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
MIN_MASKED_LENGTH: Final = 8
ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
def main() -> int:
env_path: Final = Path(sys.argv[1])
try:
secrets: Final = {
key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
}
except (ValidationError, UnicodeError):
_ = sys.stderr.write("expected a JSON object containing string environment values\n")
return 1
unusable: Final = tuple(
key
for key, value in secrets.items()
if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
)
if unusable:
_ = sys.stderr.write(
f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
)
return 1
if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
_ = sys.stdout.write(
"".join(
f"::add-mask::{value.replace('%', '%25')}\n"
for value in secrets.values()
if len(value) >= MIN_MASKED_LENGTH
)
)
sys.stdout.flush()
lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
try:
with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
os.fchmod(handle.fileno(), 0o600)
_ = handle.write("\n".join(lines) + "\n")
except OSError:
_ = sys.stderr.write("could not write the environment file\n")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

52
.github/e2e-stack/select_tests.py vendored
View file

@ -1,52 +0,0 @@
import re
import sys
from typing import Final
SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
UNSUPPORTED: Final = re.compile(
r"^tests/e2e/(ui|claude_code|load|migrations)/"
r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$"
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
r"|^tests/e2e/logging/test_s3_log_e2e\.py$"
r"|^tests/e2e/secret_manager/"
)
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
r"|^tests/e2e/idp_realm\.json$"
r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
r"|^tests/e2e/coverage_registry/management_cases\.py$"
r"|^tests/e2e/gateway/"
r"|^\.github/e2e-stack/"
r"|^\.github/workflows/test-e2e-changed\.yml$"
)
UNEXPANDED: Final = re.compile(r"[*?\[]")
def is_selectable(path: str) -> bool:
return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
direct: Final = frozenset(path for path in changed if is_selectable(path))
harness_changed: Final = any(HARNESS.match(path) for path in changed)
canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
return tuple(sorted(direct | canary_tests))
def main() -> int:
canary: Final = tuple(sys.argv[1:])
unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
if unexpanded:
_ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
return 1
changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
_ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
return 0
if __name__ == "__main__":
sys.exit(main())

231
.github/e2e-stack/up.sh vendored
View file

@ -1,231 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
CERTS_DIR="${STACK_DIR}/certs"
LOGS_DIR="${STACK_DIR}/logs"
PIDS_DIR="${STACK_DIR}/pids"
POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
LB_PORT="${E2E_LB_PORT:-4000}"
GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
REDIS_PORT="${E2E_REDIS_PORT:-6379}"
DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}"
JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
chmod 755 "${CERTS_DIR}"
log() { printf 'e2e-stack: %s\n' "$*"; }
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
wait_for() {
local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
until eval "${check}"; do
if ((SECONDS >= deadline)); then
log "timed out waiting for ${label}"
exit 1
fi
sleep 2
done
log "${label} is up"
}
if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
set -a
source "${REPO_ROOT}/tests/e2e/.env"
set +a
fi
if [[ -z "${DD_API_KEY:-}" ]]; then
log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
exit 1
fi
export DD_SITE="${DD_SITE:-datadoghq.com}"
if ! port_open "${DATABASE_PORT}"; then
docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
-e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
"${POSTGRES_IMAGE}" >/dev/null
fi
wait_for "postgres" "port_open ${DATABASE_PORT}"
if ! port_open "${JAEGER_QUERY_PORT}"; then
docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
"${JAEGER_IMAGE}" >/dev/null
fi
wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
-subj "/CN=litellm-e2e-ca" \
-addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
-out "${CERTS_DIR}/ca.crt" 2>/dev/null
openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
-CAcreateserial -days 7 -sha256 \
-extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
-out "${CERTS_DIR}/server.crt" 2>/dev/null
chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
docker rm -f e2e-valkey >/dev/null 2>&1 || true
docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
"${VALKEY_IMAGE}" valkey-server \
--cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
--tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
--tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
CONFIG_PATH="${CONFIG_SOURCE}"
if [[ "${REDIS_PORT}" != "6379" ]]; then
CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
fi
SERVER_ENV=(
"LITELLM_MASTER_KEY=${MASTER_KEY}"
"DATABASE_HOST=${DATABASE_HOST}"
"DATABASE_PORT=${DATABASE_PORT}"
"DATABASE_USER=${DATABASE_USER}"
"DATABASE_PASSWORD=${DATABASE_PASSWORD}"
"DATABASE_NAME=${DATABASE_NAME}"
"DISABLE_SCHEMA_UPDATE=true"
"REDIS_HOST=127.0.0.1"
"REDIS_PORT=${REDIS_PORT}"
"REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
"CONFIG_FILE_PATH=${CONFIG_PATH}"
"STORE_MODEL_IN_DB=True"
"OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
"OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}"
"SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
"PYTHONPATH=${REPO_ROOT}"
"JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
"JWT_AUDIENCE=litellm-e2e"
)
if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
fi
cd "${REPO_ROOT}"
env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh
log "running migrations"
env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
start_server() {
local name="$1"; shift
env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
echo $! > "${PIDS_DIR}/${name}.pid"
}
if [[ "$(uname)" == "Linux" ]]; then
NGINX_UPSTREAM_HOST=127.0.0.1
NGINX_DOCKER_ARGS=(--network host)
else
NGINX_UPSTREAM_HOST=host.docker.internal
NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}")
fi
cat > "${STACK_DIR}/nginx.conf" <<EOF
events {}
http {
map \$http_upgrade \$connection_upgrade {
default upgrade;
'' close;
}
upstream litellm_gateways {
server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
}
server {
listen ${LB_PORT};
client_max_body_size 100m;
location / {
proxy_pass http://litellm_gateways;
proxy_http_version 1.1;
proxy_set_header Host \$host;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection \$connection_upgrade;
proxy_buffering off;
proxy_read_timeout 600s;
proxy_send_timeout 600s;
}
}
server {
listen ${JAEGER_OTLP_TLS_PORT} ssl;
ssl_certificate /certs/server.crt;
ssl_certificate_key /certs/server.key;
client_max_body_size 100m;
location / {
proxy_pass http://${NGINX_UPSTREAM_HOST}:${JAEGER_OTLP_PORT};
}
}
}
EOF
docker rm -f e2e-nginx >/dev/null 2>&1 || true
docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
-v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \
-v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null
wait_for "Jaeger OTLP TLS listener" \
"curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'"
start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
cat > "${STACK_DIR}/stack.env" <<EOF
LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
LITELLM_MASTER_KEY=${MASTER_KEY}
REDIS_HOST=127.0.0.1
REDIS_PORT=${REDIS_PORT}
E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
E2E_OTEL_EXPORTER_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}
E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
E2E_KEYCLOAK_ADMIN_USER=admin
E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
EOF
log "stack is up; pytest env written to ${STACK_DIR}/stack.env"

89
.github/scripts/assert_ci_coverage.py vendored
View file

@ -9,7 +9,6 @@ import sys
import warnings
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
import yaml
@ -22,11 +21,12 @@ TESTS_ROOT = REPO_ROOT / "tests"
ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"})
DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
@ -72,6 +72,17 @@ class Scalar:
value: str
@dataclass(frozen=True, slots=True)
class Selection:
included: frozenset[str]
ignored: frozenset[str]
def covers(self, relative_path: str) -> bool:
return any(_token_covers(token, relative_path) for token in self.included) and not any(
_token_covers(token, relative_path) for token in self.ignored
)
@dataclass(frozen=True, slots=True)
class Finding:
subject: str
@ -111,44 +122,32 @@ def _uncommented(value: str) -> str:
return COMMENT_RE.sub("", value)
def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
return frozenset(
match.group(0).rstrip("/")
def _selection_for_scalar(scalar: Scalar) -> Selection:
text: Final = _uncommented(scalar.value)
ignored: Final = frozenset().union(
*(
frozenset(token.rstrip("/") for token in TEST_TOKEN_RE.findall(ignored_argument))
for ignored_argument in IGNORE_ARG_RE.findall(text)
)
)
included: Final = frozenset(
match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(IGNORE_ARG_RE.sub("", text))
)
return Selection(included=included, ignored=ignored)
def _invoked_selections(scalars: Iterable[Scalar]) -> tuple[Selection, ...]:
selected_scalars: Final = tuple(
scalar
for scalar in scalars
if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
)
selections: Final = tuple(_selection_for_scalar(scalar) for scalar in selected_scalars)
return tuple(selection for selection in selections if selection.included)
SELECTION_ARM_RE = re.compile(r"(?ms)^\s*([A-Za-z0-9_|*-]+)\)\s*(.*?);;")
def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, frozenset[str]]:
script: Final = repo_root / ".circleci/scripts/unit_selection.sh"
if not script.is_file():
return MappingProxyType({})
text: Final = _uncommented(script.read_text())
return MappingProxyType(
{
label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body))
for label, body in SELECTION_ARM_RE.findall(text)
}
)
def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens)
def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value)
def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]:
wired: Final = _wired_unit_flags(scalars)
return _invoked_test_tokens(scalars) | frozenset(
token for label, tokens in arms.items() if label in wired for token in tokens
)
def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
return frozenset().union(*(selection.included for selection in _invoked_selections(scalars)))
def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
@ -211,11 +210,12 @@ def _dockerfiles() -> tuple[str, ...]:
)
def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
def _uncovered_tests(allowlist: Allowlist, selections: tuple[Selection, ...]) -> tuple[Finding, ...]:
uncovered = tuple(
relative_path
for relative_path in _test_files()
if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
if not any(selection.covers(relative_path) for selection in selections)
and not allowlist.covers_test(relative_path)
)
directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
return tuple(
@ -505,7 +505,7 @@ def _check_slices() -> int:
def _check_shards() -> int:
findings = _unassigned_shard_children(_shard_tokens(_all_scalars(), _unit_selection_arms()))
findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
if findings:
_report(
"test directories and files that no shard claims",
@ -568,6 +568,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes)
circle_path: Final = repo_root / ".circleci/config.yml"
circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
circle_test_path_tokens: Final = _invoked_test_tokens(
scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path"
)
steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
invoked: Final = any(
".circleci/scripts/run_integration.sh" in scalar.value
@ -609,9 +612,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
Finding(path, "GitHub-owned integration contract has no invoking workflow")
Finding(path, "GitHub-owned integration contract has no invoking job")
for path in sorted(github_files)
if not any(_token_covers(token, path) for token in gha_tokens)
if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens)
)
+ tuple(
Finding(path, "GitHub-owned integration file is missing")
@ -675,7 +678,11 @@ def main() -> int:
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(allowlist, _invoked_test_tokens(scalars) | _unit_selection_tokens() | integration_paths)
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))

2
.github/scripts/verify_linux_native_wheel.py vendored
View file

@ -214,7 +214,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 45_000_000
native_size_limit: Final = 48_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),

45
.github/workflows/_test-unit-base.yml vendored
View file

@ -13,16 +13,6 @@ on:
have its path existence-checked like any other token.
required: true
type: string
unit-flag:
description: >-
Codecov flag of the `.circleci/tests.yml` job that now owns part of
this shard. The shard also runs the files
`.circleci/scripts/unit_selection.sh` lists for the flag, on every
event, because the CircleCI pipeline is manual-only while the tests
migrate.
required: false
type: string
default: ""
workers:
description: "Number of pytest-xdist workers"
required: false
@ -79,12 +69,11 @@ on:
description: "Unique name for the coverage artifact (must be unique per run)"
required: true
type: string
legacy-mcp-peer:
description: "Install the isolated SDK1 peer for MCP compatibility tests"
rust-bridge-artifact:
description: "Prebuilt editable Rust bridge artifact"
required: false
type: boolean
default: false
type: string
default: ""
permissions:
contents: read
@ -134,25 +123,29 @@ jobs:
timeout-minutes: 5
uses: ./.github/actions/cache-uv-downloads
- name: Cache the Rust build
- name: Set up the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
uses: ./.github/actions/rust-bridge
with:
artifact: ${{ inputs.rust-bridge-artifact }}
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
env:
LEGACY_MCP_PEER: ${{ inputs.legacy-mcp-peer }}
RUST_BRIDGE_ARTIFACT: ${{ inputs.rust-bridge-artifact }}
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
if [ "$LEGACY_MCP_PEER" = "true" ]; then
uv venv --python "${UV_PYTHON}" .venv-mcp-peer
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "MCP_TEST_PEER_PYTHON=$GITHUB_WORKSPACE/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
fi
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
@ -171,7 +164,6 @@ jobs:
timeout-minutes: ${{ inputs.timeout-minutes }}
env:
TEST_PATH: ${{ inputs.test-path }}
UNIT_FLAG: ${{ inputs.unit-flag }}
MAX_FAILURES: ${{ inputs.max-failures }}
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
@ -181,9 +173,6 @@ jobs:
run: |
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
selection="${TEST_PATH}"
if [ -n "${UNIT_FLAG}" ]; then
selection="${TEST_PATH} $(bash .circleci/scripts/unit_selection.sh "${UNIT_FLAG}" | tr '\n' ' ')"
fi
if [ -z "${selection// /}" ]; then
echo "shard selection is empty; nothing to run"
exit 0

39
.github/workflows/image-scan.yml vendored
View file

@ -9,10 +9,12 @@ on:
paths:
- Dockerfile
- docker/Dockerfile.non_root
- docker/Dockerfile.database
- migrations/Dockerfile
- migrations/run.py
- gateway/Dockerfile
- gateway/main.py
- gateway/routes/allowlist.py
- backend/Dockerfile
- backend/main.py
- deploy/lens/**
@ -21,6 +23,10 @@ on:
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm/proxy/admin_mcp.py
- litellm/proxy/proxy_server.py
- backend/routes/allowlist.py
- pyproject.toml
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock
@ -142,6 +148,9 @@ jobs:
- name: Build runtime image
run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f docker/Dockerfile.non_root -t litellm-admin-mcp-schema:${{ github.sha }} .
# The prisma bake must migrate a fresh DB with no egress as an arbitrary
# non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
# `docker run` as the default uid with network hides a broken bake because
@ -155,9 +164,10 @@ jobs:
- name: Verify offline migration as a non-root uid
env:
LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
# Scans the whole shipped artifact: OS/apk plus every language package
# baked into the image, including ones no lockfile declares (e.g. prisma's
@ -176,21 +186,28 @@ jobs:
--output table
runtime-image:
name: runtime-image
name: runtime-image (${{ matrix.dockerfile }})
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
timeout-minutes: 45
permissions:
contents: read
strategy:
fail-fast: false
matrix:
dockerfile: [Dockerfile, docker/Dockerfile.database]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build runtime image
run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f "${{ matrix.dockerfile }}" -t litellm-runtime-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f "${{ matrix.dockerfile }}" -t litellm-admin-mcp-schema:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@ -200,11 +217,13 @@ jobs:
- name: Verify offline migration as a non-root uid
env:
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
- name: Verify the bundled Lens Compose installation and restart
if: matrix.dockerfile == 'Dockerfile'
env:
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
run: bash tests/e2e/migrations/lens_compose_smoke.sh
@ -266,9 +285,10 @@ jobs:
env:
LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
LITELLM_COMPONENT_PORT: "4000"
LITELLM_IMAGE_COMPONENT: gateway
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
ui-image:
name: ui-image
@ -316,6 +336,9 @@ jobs:
- name: Build backend image
run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f backend/Dockerfile -t litellm-admin-mcp-schema:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
@ -324,7 +347,9 @@ jobs:
- name: Verify the backend serves offline as a non-root uid
env:
LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
LITELLM_COMPONENT_PORT: "4001"
LITELLM_IMAGE_COMPONENT: backend
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_admin_mcp.py -v

61
.github/workflows/lens-worker.yml vendored
View file

@ -6,12 +6,14 @@ on:
paths:
- deploy/lens/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
push:
branches: [main]
paths:
- deploy/lens/**
- litellm/proxy/lens/**
- tests/proxy_behavior/lens/**
- .github/workflows/lens-worker.yml
workflow_dispatch:
@ -27,6 +29,7 @@ jobs:
permissions:
contents: read
packages: write
id-token: write
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@ -54,12 +57,66 @@ jobs:
with trace_store() as store:
assert store.count() == 0
'
- name: Verify recovery after temporary storage fills
- name: Prepare test-only coverage tool
run: |
coverage_directory=$(mktemp -d "$RUNNER_TEMP/lens-coverage.XXXXXX")
curl --fail --silent --show-error --location \
https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl \
--output "$coverage_directory/coverage.whl"
printf '%s %s\n' 8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1 \
"$coverage_directory/coverage.whl" | sha256sum --check
chmod 777 "$coverage_directory"
echo "LENS_COVERAGE_DIRECTORY=$coverage_directory" >> "$GITHUB_ENV"
- name: Verify confined Python execution
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_python_smoke.py:/app/python_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.python /app/python_smoke.py
- name: Verify workspace investigation and live review output
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_context_smoke.py:/app/context_smoke.py:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.context /app/context_smoke.py
- name: Verify default workspace recovery after Python scratch storage fills
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
-v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
--entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint python lens-worker:${{ github.sha }} \
-m coverage run --data-file=/coverage/.coverage.storage /app/storage_smoke.py
- name: Map native worker coverage to repository sources
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
run: |
docker run --rm --network none --read-only --cap-drop ALL \
--security-opt no-new-privileges -w /workspace \
-v "$PWD/litellm/proxy/lens:/workspace/litellm/proxy/lens:ro" \
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
--entrypoint /bin/sh lens-worker:${{ github.sha }} \
-c 'python -m coverage combine && python -m coverage xml'
- name: Upload native worker coverage
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: ${{ env.LENS_COVERAGE_DIRECTORY }}/lens-worker.xml
root_dir: ${{ github.workspace }}
flags: lens-worker
fail_ci_if_error: false
- name: Publish versioned Lens worker
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
env:

265
.github/workflows/test-e2e-changed.yml vendored
View file

@ -1,265 +0,0 @@
name: e2e-changed-tests
on:
pull_request:
concurrency:
group: e2e-changed-${{ github.event.pull_request.number }}
cancel-in-progress: true
permissions: {}
jobs:
detect:
name: Detect changed e2e tests
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: read
outputs:
tests: ${{ steps.changed.outputs.tests }}
any: ${{ steps.changed.outputs.any }}
steps:
- name: Checkout the selector and the canary suite
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: |
.github/e2e-stack
tests/e2e/access_control
tests/e2e/management/test_jwt_management_e2e.py
tests/e2e/other/test_jwt_auth_e2e.py
persist-credentials: false
ref: ${{ github.sha }}
- name: List the e2e test files this PR added or modified
id: changed
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
--jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
| grep -Fxq "${HEAD_SHA}"
files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
--jq '.[] | select(.status != "removed") | .filename')"
gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
tests="$(printf '%s\n' "${files}" \
| python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \
tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)"
echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
if [ -n "${tests}" ]; then
echo "any=true" >> "${GITHUB_OUTPUT}"
echo "selected e2e tests: ${tests}"
else
echo "any=false" >> "${GITHUB_OUTPUT}"
echo "no changed e2e test files supported by this stack; nothing to run"
fi
run:
name: Run changed e2e tests against the stage-mirror stack
needs: detect
if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
timeout-minutes: 90
environment: e2e-changed
permissions:
contents: read
id-token: write
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: litellm
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U litellm"
--health-interval 5s
--health-timeout 5s
--health-retries 10
jaeger:
image: jaegertracing/jaeger:2.10.0
ports:
- 4318:4318
- 16686:16686
steps:
- name: Validate configuration
env:
ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
- name: Checkout
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
ref: ${{ github.sha }}
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.13"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen \
--extra proxy --extra proxy-runtime --extra extra_proxy \
--extra semantic-router --extra bedrock-realtime \
--group ci --group proxy-dev --group e2e-dev
uv pip install "pipecat-ai[openai]==1.4.0"
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Install Playwright chromium
run: uv run --no-sync playwright install --with-deps chromium
- name: Configure AWS credentials
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
with:
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
aws-region: us-east-1
role-session-name: litellm-e2e-changed-${{ github.run_id }}
role-duration-seconds: 900
output-env-credentials: false
output-credentials: true
- name: Fetch provider credentials from AWS Secrets Manager
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
AWS_DEFAULT_REGION: us-east-1
run: |
umask 077
aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
--query SecretString --output text \
| uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
--query SecretString --output text \
| jq -R -s '{"LITELLM_LICENSE": .}' \
| uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
- name: Boot the stage-mirror stack
id: boot
run: |
umask 077
if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
echo "::error::stage-mirror stack failed to boot; raw logs are not published"
exit 1
fi
- name: Export stack environment
run: |
master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
echo "::add-mask::${master_key}"
cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
- name: Run the selected tests three times
env:
TESTS: ${{ needs.detect.outputs.tests }}
E2E_FIXTURE_MODE: live
E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
E2E_OWNED_GATEWAY: '1'
COLUMNS: '400'
run: |
umask 077
read -r -a test_files <<< "${TESTS}"
for pass in 1 2 3; do
report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
echo "::group::pass ${pass} of 3"
set +e
uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
-o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
status=$?
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
verified=$?
set -e
grep -E '^(FAILED|ERROR) ' "${log}" || true
grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
echo "::endgroup::"
if [ "${status}" = "5" ]; then
echo "::error::the selected files collected no runnable tests, so nothing was verified"
exit 1
fi
if [ "${status}" != "0" ]; then
echo "::error::pass ${pass} of 3 failed with exit code ${status}"
exit "${status}"
fi
if [ "${verified}" != "0" ]; then
echo "::error::pass ${pass} of 3 did not verify every selected file"
exit 1
fi
echo "pass ${pass} of 3 passed"
done
- name: Redact the pytest output
if: always() && steps.boot.outcome == 'success'
run: |
umask 077
shopt -s nullglob
uv run --no-sync python .github/e2e-stack/redact_output.py \
--values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \
--out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
- name: Keep the redacted pytest output
if: always() && steps.boot.outcome == 'success'
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: e2e-changed-pytest-output-${{ github.run_attempt }}
path: ${{ runner.temp }}/e2e-redacted
retention-days: 14
if-no-files-found: ignore
- name: Stop the stack
if: always() && steps.boot.outcome != 'skipped'
run: bash .github/e2e-stack/down.sh
- name: Remove credentials and raw output
if: always()
run: |
rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted"
gate:
name: e2e-changed-tests
needs: [detect, run]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Require three successful passes when tests changed
env:
DETECT_RESULT: ${{ needs.detect.result }}
ANY_TESTS: ${{ needs.detect.outputs.any }}
RUN_RESULT: ${{ needs.run.result }}
run: |
if [ "${DETECT_RESULT}" != "success" ]; then
echo "::error::changed-test detection did not succeed"
exit 1
fi
if [ "${ANY_TESTS}" = "false" ]; then
echo "no changed e2e test files supported by this stack; nothing to run"
exit 0
fi
if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
exit 1
fi

21
.github/workflows/test-mcp-dependency-resolution.yml vendored
View file

@ -5,6 +5,27 @@ on:
branches:
- main
- "litellm_**"
paths:
- "**/pyproject.toml"
- "uv.lock"
- "uv.toml"
- ".python-version"
- "rust-toolchain.toml"
- "litellm-rust/**"
- "litellm/__init__.py"
- "litellm/proxy/proxy_server.py"
- "litellm/**/*mcp*"
- "litellm/**/*mcp*/**"
- "litellm/integrations/arize/**"
- "scripts/check_mcp_sdk_install.py"
- "tests/base_sdk_tests/**"
- ".github/workflows/test-mcp-dependency-resolution.yml"
- ".github/actions/detect-changes/**"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/detect_changes.sh"
- ".github/scripts/uv_sync_with_retries.sh"
- ".circleci/scripts/classify_changes.sh"
permissions:
contents: read

185
.github/workflows/test-postgres.yml vendored
View file

@ -1,185 +0,0 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- "litellm_**"
push:
branches:
- main
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
id-token: write
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: roi-database
test-path: "tests/integration/database/test_roi_observed.py"
seed: none
workers: 0
timeout-minutes: 10
job-timeout-minutes: 35
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
timeout-minutes: 5
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || matrix.shard == 'roi-database' && '--cov=./litellm --cov-report=xml:coverage-roi-postgres.xml' || '' }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi
- name: Upload Lens database coverage
if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
use_oidc: true
version: v11.3.1
root_dir: ${{ github.workspace }}
files: coverage-lens-postgres.xml
flags: lens-postgres
fail_ci_if_error: true
- name: Upload ROI database coverage
if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'roi-database' && !cancelled()
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
use_oidc: true
version: v11.3.1
root_dir: ${{ github.workspace }}
files: coverage-roi-postgres.xml
flags: roi-postgres
fail_ci_if_error: true

106
.github/workflows/test-redis-compat.yml vendored
View file

@ -1,106 +0,0 @@
name: "Unit Tests: Redis Client Version Compatibility"
on:
pull_request:
branches:
- main
- "litellm_**"
paths:
- "litellm/_redis.py"
- "litellm/_redis_credential_provider.py"
- "litellm/caching/redis_cache.py"
- "litellm/caching/evicted_client_closer.py"
- "tests/unit/test_redis.py"
- "tests/local_testing/test_caching.py"
- "tests/unit/caching/test_redis_connection_pool.py"
- "tests/unit/caching/test_redis_cluster_cache.py"
- "tests/unit/caching/test_evicted_client_closer.py"
- ".github/workflows/test-redis-compat.yml"
- "pyproject.toml"
- "uv.lock"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
redis-compat:
name: "redis-py ${{ matrix.redis-version }}"
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
# 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the
# newer legs prove the inspect.signature introspection in litellm/_redis.py
# keeps extracting kwargs on the redis-py releases people actually run now.
# Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra)
# specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in
# for the 6.x line.
redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router
- name: Pin redis-py to the matrix version
env:
REDIS_VERSION: ${{ matrix.redis-version }}
run: |
uv pip install "redis==${REDIS_VERSION:?}"
uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)"
- name: Build Redis for cluster authentication tests
run: |
curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz"
echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check
tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP"
make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server
echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH"
- name: Run redis unit tests
run: |
redis-server --version
uv run --no-sync pytest \
tests/unit/test_redis.py \
tests/unit/caching/test_redis_connection_pool.py \
tests/unit/caching/test_redis_cluster_cache.py \
tests/unit/caching/test_evicted_client_closer.py \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \
tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \
--tb=short -vv \
--reruns 2 \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm --cov-report=xml:coverage-redis.xml
- name: Upload Redis coverage
if: matrix.redis-version == '5.3.1'
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
with:
use_oidc: true
files: coverage-redis.xml
flags: redis-compat
fail_ci_if_error: false

100
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -20,13 +20,6 @@ concurrency:
# rather than alphabetical letter ranges. Adding a new test file means adding it
# to whichever group it belongs to, not reshuffling slices.
#
# `.circleci/tests.yml` runs each group's files on same-repo events under the
# `proxy-db-<group>` Codecov flag; `.circleci/scripts/unit_selection.sh` holds
# the file lists. That pipeline is manual-only while the tests migrate, so
# `unit-flag` makes the shard run that list on every event. `test-path` keeps
# the files that still reach real providers and never left
# tests/proxy_unit_tests.
#
# Design targets:
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
# Most of a shard's time is pytest plugin load + xdist worker imports +
@ -78,93 +71,136 @@ jobs:
include:
# Must run serially — event-loop conflict with the logging worker.
- test-group: key-generation
test-path: ""
unit-flag: proxy-db-key-generation
test-path: >-
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
workers: 0
dist: loadscope
timeout: 20
# ---- auth: split into 2 shards ----
- test-group: auth-checks
test-path: ""
unit-flag: proxy-db-auth-checks
test-path: >-
tests/unit/proxy/auth/test_auth_checks.py
tests/unit/proxy/auth/test_user_api_key_auth.py
tests/unit/proxy/test_credential_slot_registry.py
tests/unit/proxy/test_deprecated_key_grace_period.py
workers: 4
dist: loadscope
timeout: 15
- test-group: jwt-and-keys
test-path: ""
unit-flag: proxy-db-jwt-and-keys
test-path: >-
tests/unit/proxy/auth/test_jwt.py
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
tests/unit/proxy/test_proxy_custom_auth.py
workers: 4
dist: loadscope
timeout: 15
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
- test-group: proxy-utils
test-path: ""
unit-flag: proxy-db-proxy-utils
test-path: >-
tests/unit/proxy/test_proxy_utils.py
workers: 4
dist: worksteal
timeout: 15
# ---- proxy server: split into 2 shards ----
- test-group: proxy-server-core
test-path: "tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py"
unit-flag: proxy-db-proxy-server-core
test-path: >-
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
tests/unit/proxy/test__lazy_features.py
tests/unit/proxy/test_aproxy_startup.py
tests/unit/proxy/test_proxy_server.py
workers: 4
dist: loadscope
timeout: 15
- test-group: proxy-runtime
test-path: ""
unit-flag: proxy-db-proxy-runtime
test-path: >-
tests/unit/proxy/auth/test_multipart_bypass_repro.py
tests/unit/proxy/auth/test_proxy_routes.py
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/test_proxy_config_unit_test.py
tests/unit/proxy/test_proxy_token_counter.py
tests/unit/proxy/test_server_root_path.py
workers: 4
dist: loadscope
timeout: 15
# ---- logging: split into 2 shards ----
- test-group: custom-logging
test-path: "tests/proxy_unit_tests/test_proxy_custom_logger.py"
unit-flag: proxy-db-custom-logging
test-path: >-
tests/proxy_unit_tests/test_proxy_custom_logger.py
tests/unit/proxy/test_custom_callback_input.py
tests/unit/proxy/test_custom_logger_s3_gcs.py
workers: 4
dist: loadscope
timeout: 15
- test-group: logging-misc
test-path: ""
unit-flag: proxy-db-logging-misc
test-path: >-
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/test_proxy_reject_logging.py
workers: 4
dist: loadscope
timeout: 15
- test-group: db-and-spend
test-path: ""
unit-flag: proxy-db-db-and-spend
test-path: >-
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/test_db_schema_changes.py
tests/unit/proxy/test_prisma_client_backoff_retry.py
tests/unit/proxy/test_update_spend.py
tests/unit/skills/test_skills_db.py
workers: 4
dist: loadscope
timeout: 15
# ---- guardrails + budget + hooks: split into 2 ----
- test-group: guardrails-hooks
test-path: ""
unit-flag: proxy-db-guardrails-hooks
test-path: >-
tests/unit/proxy/hooks/test_banned_keyword_list.py
tests/unit/proxy/test_proxy_setting_guardrails.py
tests/unit/proxy/test_unit_test_proxy_hooks.py
workers: 4
dist: loadscope
timeout: 15
- test-group: budgets
test-path: ""
unit-flag: proxy-db-budgets
test-path: >-
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
dist: loadscope
timeout: 15
- test-group: endpoints-and-responses
test-path: "tests/proxy_unit_tests/test_proxy_exception_mapping.py"
unit-flag: proxy-db-endpoints-and-responses
test-path: >-
tests/proxy_unit_tests/test_proxy_exception_mapping.py
tests/unit/proxy/lens
tests/unit/proxy/auth/test_models_fallback_endpoint.py
tests/unit/proxy/common_utils/test_check_batch_cost.py
tests/unit/proxy/common_utils/test_check_responses_cost.py
tests/unit/proxy/common_utils/test_realtime_cache.py
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/response_polling
tests/unit/proxy/test_custom_tokenizer_bug.py
tests/unit/proxy/test_get_favicon.py
tests/unit/proxy/test_get_image.py
tests/unit/proxy/test_prompt_test_endpoint.py
tests/unit/proxy/test_reducto_ocr_route.py
tests/unit/proxy/test_response_polling_pre_call_checks.py
tests/unit/proxy/test_ui_path_detection.py
workers: 4
dist: loadscope
timeout: 15
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
unit-flag: ${{ matrix.unit-flag }}
workers: ${{ matrix.workers }}
reruns: 2
timeout-minutes: ${{ matrix.timeout }}

332
.github/workflows/test-unit.yml vendored
View file

@ -21,10 +21,8 @@ concurrency:
# files that each wrapped a single call to _test-unit-base.yml. Adding a shard is
# now one matrix entry rather than a new file.
#
# `name` is the shard id and nothing else, so each check reports as
# "<shard> / Run tests" exactly as it did when the shard had its own file. Those
# strings are the branch ruleset's required contexts, so they are load-bearing:
# renaming an entry renames a required check and the ruleset stops matching it.
# `name` is the shard id, and each check reports as "<shard> / Run tests".
# Unit shard names are not required ruleset contexts, so matrix entries can be split freely.
#
# Every entry states its timeouts even when they equal the base workflow's
# defaults. An absent matrix key renders as an empty string, which is not a
@ -35,13 +33,102 @@ concurrency:
# a matrix and carries a shard-coverage guard that reads that file by name.
# Folding it in here is a follow-up, together with generalising that guard into
# assert_ci_coverage.py.
#
# `unit-flag` names the `.circleci/tests.yml` job that now runs part of the
# shard under the same Codecov flag. That pipeline is manual-only while the
# tests migrate, so the shard also runs those files on every event.
jobs:
rust-bridge:
name: Build the Rust bridge
outputs:
artifact: ${{ steps.rust-bridge-artifact.outputs.name }}
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
env:
UV_PYTHON: "3.12"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Define editable Rust bridge cache key
id: rust-bridge-key
if: steps.changes.outputs.decision != 'skip'
env:
RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'pyproject.toml') }}
run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT"
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Restore editable Rust bridge
id: rust-bridge-cache
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: rust-bridge-dist
key: ${{ steps.rust-bridge-key.outputs.key }}
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Build editable Rust bridge
if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 15
run: |
mkdir -p rust-bridge-dist
uv run --no-project --with maturin==1.15.0 python -c 'import maturin; maturin.build_editable("rust-bridge-dist")'
strip --strip-debug litellm/rust_bridge/_native.abi3.so
mkdir -p rust-bridge-dist/litellm/rust_bridge
cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so
- name: Upload Rust bridge artifact
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 10
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1
with:
name: rust-bridge-${{ github.run_id }}-${{ github.run_attempt }}
path: rust-bridge-dist/
retention-days: 1
- name: Save editable Rust bridge
if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main' && steps.rust-bridge-cache.outputs.cache-hit != 'true'
timeout-minutes: 10
uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: rust-bridge-dist
key: ${{ steps.rust-bridge-key.outputs.key }}
- name: Expose the Rust bridge artifact
id: rust-bridge-artifact
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 1
env:
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
run: echo "name=rust-bridge-${RUN_ID}-${RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
unit:
name: ${{ matrix.shard }}
needs: rust-bridge
if: ${{ !cancelled() }}
permissions:
contents: read
id-token: write
@ -50,29 +137,35 @@ jobs:
fail-fast: false
matrix:
include:
- shard: mcp-integration
artifact-name: mcp-integration
test-path: "tests/mcp_tests"
unit-flag: mcp-integration
workers: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: core-utils
artifact-name: core-utils
test-path: tests/unit/decisions
unit-flag: core-utils
workers: 2
reruns: 1
test-path: >-
tests/unit/decisions
tests/unit/litellm_core_utils
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-routing
artifact-name: enterprise-routing
test-path: ""
unit-flag: enterprise-routing
workers: 2
test-path: >-
tests/unit/google_genai
tests/unit/router_strategy
tests/unit/router_utils
tests/unit/proxy/common_utils/test_cache_aware_routing.py
tests/unit/enterprise/enterprise_callbacks/send_emails
tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py
tests/unit/enterprise/proxy/test_batch_retrieve_input_file_id.py
tests/unit/enterprise/proxy/test_batch_retrieve_registers_missing_output_file_id.py
tests/unit/enterprise/proxy/test_batch_retrieve_returns_unified_input_file_id.py
tests/unit/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
tests/unit/enterprise/proxy/test_deleted_file_returns_403_not_404.py
tests/unit/enterprise/proxy/test_enterprise_routes.py
tests/unit/enterprise/proxy/test_file_deletion_blocking.py
tests/unit/enterprise/proxy/test_managed_files_access_check.py
tests/unit/enterprise/proxy/test_managed_files_hook.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
@ -82,27 +175,41 @@ jobs:
test-path: >-
tests/test_litellm/integrations
tests/test_litellm/tracing
unit-flag: integrations
workers: 2
reruns: 3
tests/unit/integrations
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: Vertex AI
artifact-name: llm-vertex-ai
test-path: ""
unit-flag: llm-vertex-ai
workers: 1
reruns: 2
test-path: >-
tests/unit/llms/vertex_ai
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: All Other Providers
artifact-name: llm-other-providers
test-path: ""
unit-flag: llm-other-providers
workers: 2
reruns: 2
test-path: >-
tests/unit/llms
--ignore=tests/unit/llms/vertex_ai
--ignore=tests/unit/llms/openai
--ignore=tests/unit/llms/meta
--ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: OpenAI and Meta Providers
artifact-name: llm-openai-meta
test-path: >-
tests/unit/llms/openai
tests/unit/llms/meta
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
@ -110,19 +217,44 @@ jobs:
artifact-name: misc
test-path: >-
tests/test_litellm/test_*.py
unit-flag: misc
workers: 2
tests/unit/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: misc-dirs
artifact-name: misc-dirs
test-path: >-
tests/unit/test_router
tests/unit/a2a_protocol
tests/unit/batches
tests/unit/chat_completions
tests/unit/completion_extras
tests/unit/containers
tests/unit/embeddings
tests/unit/endpoints
tests/unit/files
tests/unit/harness
tests/unit/images
tests/unit/interactions
tests/unit/messages
tests/unit/rag
tests/unit/rerank_api
tests/unit/rust_bridge
tests/unit/secret_managers
tests/unit/vector_stores
tests/unit/videos
--ignore=tests/unit/rust_bridge/native_route_wheel_test.py
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-auth
artifact-name: proxy-auth
test-path: >-
tests/unit/proxy/auth
tests/unit/proxy/hooks
tests/unit/proxy/policy_engine
tests/unit/proxy/client
--ignore=tests/unit/proxy/auth/test_auth_checks.py
--ignore=tests/unit/proxy/auth/test_user_api_key_auth.py
--ignore=tests/unit/proxy/auth/test_default_end_user_budget_simple.py
@ -130,35 +262,56 @@ jobs:
--ignore=tests/unit/proxy/auth/test_models_fallback_endpoint.py
--ignore=tests/unit/proxy/auth/test_multipart_bypass_repro.py
--ignore=tests/unit/proxy/auth/test_proxy_routes.py
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-hooks-client
artifact-name: proxy-hooks-client
test-path: >-
tests/unit/proxy/hooks
tests/unit/proxy/policy_engine
tests/unit/proxy/client
--ignore=tests/unit/proxy/hooks/test_banned_keyword_list.py
--ignore=tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
workers: 2
reruns: 2
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-endpoints
artifact-name: proxy-endpoints
test-path: >-
tests/unit/proxy/management_endpoints
tests/unit/proxy/management_helpers
tests/unit/proxy/list_api
tests/unit/proxy/analytics_endpoints
tests/unit/proxy/decisions_endpoints
tests/unit/proxy/management_endpoints
tests/unit/proxy/list_api
tests/unit/proxy/memory
tests/unit/proxy/guardrails
tests/unit/proxy/management_helpers
tests/unit/proxy/agent_endpoints
tests/unit/proxy/openai_files_endpoint
tests/unit/proxy/health_endpoints
tests/unit/proxy/batches_endpoints
--ignore=tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
--ignore=tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
--ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
--ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-feature-endpoints
artifact-name: proxy-feature-endpoints
test-path: >-
tests/unit/proxy/guardrails
--ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
--ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
--ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
--ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
tests/unit/proxy/anthropic_endpoints
tests/unit/proxy/google_endpoints
tests/unit/proxy/openai_files_endpoint
tests/unit/proxy/batches_endpoints
tests/unit/proxy/container_endpoints
tests/unit/proxy/fine_tuning_endpoints
tests/unit/proxy/vector_store_files_endpoints
@ -168,11 +321,9 @@ jobs:
tests/unit/proxy/ocr_endpoints
tests/unit/proxy/search_endpoints
tests/unit/proxy/vector_store_endpoints
tests/unit/proxy/agent_endpoints
tests/unit/proxy/a2a
tests/unit/proxy/credential_endpoints
tests/unit/proxy/discovery_endpoints
tests/unit/proxy/health_endpoints
tests/unit/proxy/shutdown
tests/unit/proxy/public_endpoints
tests/unit/proxy/prompts
@ -183,7 +334,7 @@ jobs:
tests/unit/proxy/config_resolvers
tests/unit/proxy/utils
workers: 4
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
@ -191,7 +342,7 @@ jobs:
artifact-name: proxy-server
test-path: "tests/unit/proxy/proxy_server"
workers: 4
reruns: 2
reruns: 0
timeout-minutes: 60
job-timeout-minutes: 100
@ -218,9 +369,12 @@ jobs:
tests/unit/proxy/enterprise_billing
tests/unit/proxy/types_utils
tests/unit/proxy/logging_endpoints
unit-flag: proxy-infra
tests/unit/gateway
tests/unit/proxy/management
tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
tests/unit/proxy/roi_calculator
workers: 4
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
@ -254,52 +408,92 @@ jobs:
--ignore=tests/unit/proxy/test_update_spend.py
--ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py
workers: 4
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: ""
unit-flag: caching-local
test-path: >-
tests/unit/caching
workers: 2
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-extras
artifact-name: proxy-extras
test-path: ""
unit-flag: proxy-extras
test-path: >-
tests/unit/litellm_proxy_extras
workers: 2
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-package
artifact-name: enterprise-package
test-path: ""
unit-flag: enterprise-package
test-path: >-
tests/unit/enterprise/integrations
tests/unit/enterprise/proxy/auth
tests/unit/enterprise/proxy/guardrails
tests/unit/enterprise/proxy/management_endpoints
tests/unit/enterprise/proxy/test_audit_logging_endpoints.py
tests/unit/enterprise/proxy/test_liteadmin.py
tests/unit/enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py
workers: 4
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-managed-files
artifact-name: enterprise-managed-files
test-path: >-
tests/unit/enterprise/proxy/hooks
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: load
- shard: responses-caching-types
artifact-name: responses-caching-types
test-path: ""
unit-flag: responses-caching-types
test-path: >-
tests/unit/responses
tests/unit/types
--ignore=tests/unit/responses/mcp
workers: 2
reruns: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
- shard: unit
artifact-name: unit
test-path: >-
tests/unit/anthropic_interface
tests/unit/compression
tests/unit/enterprise/enterprise_callbacks/test_callback_controls.py
tests/unit/enterprise/enterprise_callbacks/test_llm_guard.py
tests/unit/enterprise/enterprise_callbacks/test_secret_detection.py
tests/unit/integration_support
tests/unit/models
tests/unit/ocr
tests/unit/passthrough
tests/unit/realtime_api
tests/unit/repositories
tests/unit/sandbox
tests/unit/skills/test_skills_main.py
tests/unit/tracing
workers: 2
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
uses: ./.github/workflows/_test-unit-base.yml
with:
rust-bridge-artifact: ${{ needs.rust-bridge.outputs.artifact }}
test-path: ${{ matrix.test-path }}
unit-flag: ${{ matrix.unit-flag || '' }}
workers: ${{ matrix.workers }}
reruns: ${{ matrix.reruns }}
timeout-minutes: ${{ matrix.timeout-minutes }}
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
dist: ${{ matrix.dist || 'loadscope' }}
artifact-name: ${{ matrix.artifact-name }}
legacy-mcp-peer: ${{ matrix.shard == 'mcp-integration' }}

2
Dockerfile
View file

@ -79,6 +79,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
@ -101,6 +102,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \

2
backend/Dockerfile
View file

@ -44,6 +44,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
@ -56,6 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \

2
backend/routes/allowlist.py
View file

@ -85,6 +85,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
"/analytics/",
"/lens/",
"/v1/traces",
"/v1/logs",
"/global/",
"/user_agent",
"/usage/",
@ -160,6 +161,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/admin",
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}

11
deploy/lens/Dockerfile
View file

@ -6,23 +6,30 @@ FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
RUN apk add --no-cache python-3.13 build-base libseccomp-dev
ENV UV_PYTHON_DOWNLOADS=0 UV_LINK_MODE=copy
WORKDIR /app
COPY deploy/lens/requirements.lock /tmp/requirements.lock
RUN uv venv --python python3.13 /app/.venv && \
uv pip sync --python /app/.venv/bin/python --require-hashes --only-binary :all: /tmp/requirements.lock
COPY deploy/lens/python_policy.c /tmp/python_policy.c
RUN cc -std=c11 -D_GNU_SOURCE -O2 -Wall -Wextra -Werror /tmp/python_policy.c -lseccomp -o /tmp/python-policy && \
/tmp/python-policy /app/python.seccomp
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG LITELLM_RELEASE_TAG=""
RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
RUN apk add --no-cache python-3.13
RUN apk add --no-cache python-3.13 setpriv
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} \
PATH="/app/.venv/bin:${PATH}" \
PYTHONDONTWRITEBYTECODE=1
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
COPY litellm/proxy/lens/context_pipeline.py litellm/proxy/lens/agent_review.py litellm/proxy/lens/agent_runtime.py litellm/proxy/lens/agent_workspace.py litellm/proxy/lens/python_tool.py litellm/proxy/lens/activity.py litellm/proxy/lens/agent_context.py /app/lens/
COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
COPY --from=builder /app/python.seccomp /app/lens/python.seccomp
COPY deploy/lens/python_runtime.py /tmp/python_runtime.py
RUN python3.13 -S /tmp/python_runtime.py /app/lens/python-runtime.json && rm /tmp/python_runtime.py
USER 65532:65532
CMD ["python", "-m", "lens.worker"]

55
deploy/lens/README.md
View file

@ -100,9 +100,9 @@ docker compose --env-file /path/to/lens.env -f compose.yaml up -d
To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. Python reports storage failures to the reviewer and cleans up temporary files, so the reviewer can retry a smaller computation or report insufficient evidence. The worker remains available for other scans. Existing workers must be recreated with the new image and mount options
The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles up to three investigations concurrently and can serve multiple lenses. For more throughput, start another worker with a separate credential
If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
@ -116,7 +116,7 @@ Describe how the agent should behave and optionally add specific checks. Select
Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every two seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
## Read the results
@ -132,11 +132,13 @@ The proxy selects executions received or updated within the configured lookback
A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence
The worker prepares a workspace containing the selected execution metadata and reviews executions in parallel. Reviewers receive their assignment and use catalog, read, search and optional Python tools to inspect evidence, including nested agents and other sampled executions. Tools retrieve original content from the gateway when requested; the worker does not preload the sampled traces or inject them into each model request. Python receives selected evidence as streamed input. Completed reviews retain cited excerpts and metadata. Observation batches are grouped in parallel, reconciled, and investigated against the original evidence. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs
There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools
There is no fixed total run, span, candidate or investigation-turn cutoff. Agents can replace their active conversation with working notes. If a request exceeds the configured model's context window, the worker compacts the conversation automatically and resumes with references to its archived tool history. Original evidence remains accessible through the gateway while it is available and retained. Tool results and working notes remain accessible during the investigation; character ranges make even a single oversized result readable in pieces. A review reports an error if the task or its replacement notes cannot fit. Context windows, the configured budget, worker resources and recorded evidence still bound practical work. The investigator has no browsing, code-editing or production-action tools
Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
The live review drawer shows loading, trace review, parallel grouping, reconciliation and candidate investigation. It reports current model and tool operations, including context compaction, and retains tool-call counts on completed trace reviews. These counts describe attempted calls, not successful executions. This progress channel contains operation metadata, not Python code or tool output. Preliminary observations remain separate from final findings; the final finding format and evidence links are unchanged
Each model response must match its JSON schema. A malformed response gets one repair attempt through the same budget controls. A session review that remains invalid or cannot fit marks that execution unassessable while other reviews continue. Broken evidence pagination or missing content pages return tool errors so the agent can inspect narrower spans or other evidence. Unreadable citations receive repair feedback. Verified excerpts remain available without fetching their source again. The affected source counts as partial, including failures discovered during later investigations, while the reviewer owns its assessment. Candidate investigation errors preserve completed findings. Source and analysis errors remain visible and mark the final scan as failed; transport errors, cancellation and budget exhaustion stop the scan. Both the worker and proxy validate quoted evidence against original content. Per-run issue assessments follow supporting citations, including evidence found by another run's reviewer; counterexamples do not mark a run affected. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
@ -215,7 +217,7 @@ python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
The default workspace retrieves trace content on demand. Python calls have temporary scratch space that is removed after execution. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates
@ -242,3 +244,42 @@ The hourly development pipeline pins all component images to the same selected c
## Worker dependencies
The worker uses the same digest-pinned Wolfi base and Python version as the component images. Python dependencies and their hashes are locked in `deploy/lens/requirements.lock`. To update them, edit `deploy/lens/requirements.in`, then run `uv pip compile --universal --python-version 3.13 --generate-hashes --no-emit-index-url deploy/lens/requirements.in -o deploy/lens/requirements.lock`. The image installs only the locked wheels with hash verification. CI builds and scans both native architectures
## Python analysis boundary
The `python` tool runs ordinary CPython with the standard library in a fresh child process inside the existing worker container. It receives the selected evidence as `data` over stdin and has its own temporary working directory. It creates no additional container or service. Read and search tools remain available independently of Python
The native worker image builds a syscall policy with libseccomp and includes the full `setpriv` launcher. Each child starts with no inherited worker secrets or open worker files, isolated Python startup, Landlock filesystem restrictions and a default-deny seccomp filter. It can read the Python runtime and its own scratch files. Worker source, installed worker packages, other jobs' files and `/proc` contents are unavailable. Network sockets, child processes, cross-process memory operations, signals to other processes and filesystem metadata mutation are denied, including calls made through `ctypes`. Some metadata inspection, such as `stat`, `access` and `readlink` of known paths, remains possible
Python execution requires a native Linux worker with Landlock ABI 3 or later and seccomp filtering. Build the image for the host architecture. Missing policy files, an incompatible kernel, or an unsupported host such as a macOS source worker returns a clear tool error. There is no unrestricted execution fallback. Keep the container's non-root user, dropped capabilities, no-new-privileges setting, read-only root and writable temporary mount
The worker permits two Python children at once across all investigations. Set `LENS_PYTHON_CONCURRENCY` to a positive integer to change this worker-wide pool. Queued calls consume no child process or scratch directory; cancelling a queued call does not start it. Model, read and search concurrency are separate
| Per-call resource | Default |
| --- | --- |
| Elapsed execution time | 60 seconds |
| CPU time | 30 seconds |
| Process address space | 512 MiB |
| Captured stdout or stderr | 8 MiB per stream |
| Individual scratch file size | 16 MiB |
| Monitored scratch storage | 64 MiB |
| Monitored scratch entries | 2,048 |
| Scratch directory depth | 128 |
| Open file descriptors | 64 |
Evidence is streamed from gateway pages into the confined child without building another complete selection in worker memory. The child decodes the selected data under its memory limit before running the code. The execution wall clock starts after input delivery; gateway fetches keep their HTTP timeouts and remain cancellable. CPU, address-space and file-size limits apply during input decoding as well as computation. Scratch usage is monitored every 50 milliseconds, so a call can temporarily overshoot its scratch allowance. The worker's shared temporary mount supplies the hard aggregate storage ceiling, 1 GiB by default. Accounting includes unlinked open files and files retained only by memory mappings. A mapped scratch inode without an open descriptor or directory entry is conservatively charged at the individual file-size limit, which may overcount small files. Cancellation and limit failures kill and reap the child before removing its scratch directory
Results include `stdout`, `stderr`, `exit_code`, `error` and `output_complete`. Nonzero interpreter exits, confinement failures and resource failures set `error` and `output_complete=false`. Available traceback output is retained. An output-size failure delivers no partial stdout/stderr; the agent can narrow its computation and retry. A successful result retains all captured output without truncation
This is a process boundary sharing the worker's Linux kernel. The checked-in smoke test verifies useful Python operations, filesystem and process restrictions, raw syscall attempts, resource failures, mapping accounting, cleanup and cancellation in the actual image. Run it on the deployment's native architecture and kernel:
```bash
docker build --build-arg LITELLM_RELEASE_TAG=lens-python-test \
-f deploy/lens/Dockerfile -t lens-worker:python-test .
docker run --rm --pull never --read-only --cap-drop ALL \
--security-opt no-new-privileges --network none \
--tmpfs /tmp:rw,noexec,nosuid,size=1g --entrypoint python -i \
lens-worker:python-test - < tests/proxy_behavior/lens/worker_python_smoke.py
```
The same checks can run through pytest by setting `LENS_TEST_WORKER_IMAGE` to an already-built native image. The worker image CI runs the standalone smoke without adding pytest to the production image

1
deploy/lens/compose.yaml
View file

@ -4,6 +4,7 @@ services:
environment:
LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
restart: unless-stopped
read_only: true
tmpfs:

63
deploy/lens/python_policy.c Normal file
View file

@ -0,0 +1,63 @@
#include <errno.h>
#include <fcntl.h>
#include <seccomp.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <unistd.h>
static int allow(scmp_filter_ctx policy, const char *name)
{
int number = seccomp_syscall_resolve_name(name);
return number < 0 ? 0 : seccomp_rule_add(policy, SCMP_ACT_ALLOW, number, 0);
}
int main(int argc, char **argv)
{
const char *calls[] = {
"read", "write", "readv", "writev", "pread64", "pwrite64", "close", "close_range",
"open", "openat", "openat2", "fstat", "stat", "lstat", "newfstatat", "statx",
"lseek", "getdents", "getdents64", "access", "faccessat", "faccessat2",
"readlink", "readlinkat", "getcwd", "chdir", "fchdir", "statfs", "fstatfs",
"mkdir", "mkdirat", "rmdir", "unlink", "unlinkat", "rename", "renameat", "renameat2",
"link", "linkat", "symlink", "symlinkat", "truncate", "ftruncate", "fsync", "fdatasync",
"mmap", "mmap2", "mprotect", "munmap", "mremap", "madvise", "brk",
"rt_sigaction", "rt_sigprocmask", "rt_sigreturn", "rt_sigsuspend", "rt_sigtimedwait", "sigaltstack",
"getpid", "getppid", "gettid", "getuid", "geteuid", "getgid", "getegid", "getgroups",
"clock_gettime", "clock_getres", "clock_nanosleep", "gettimeofday", "time", "nanosleep",
"futex", "futex_time64", "set_tid_address", "set_robust_list", "rseq", "arch_prctl",
"sched_getaffinity", "sched_yield", "getrandom", "getrlimit", "setrlimit", "getrusage", "umask",
"dup", "dup2", "dup3", "pipe", "pipe2", "poll", "ppoll", "select", "pselect6",
"epoll_create", "epoll_create1", "epoll_ctl", "epoll_wait", "epoll_pwait", "epoll_pwait2",
"capget", "capset", "prctl", "landlock_create_ruleset", "landlock_add_rule", "landlock_restrict_self",
"execve", "exit", "exit_group", "uname", "sysinfo", "restart_syscall"
};
const int commands[] = {F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_SETFD, F_GETFL, F_GETLK, F_SETLK, F_SETLKW};
if (argc != 2) {
fputs("Usage: python-policy OUTPUT\n", stderr);
return 1;
}
scmp_filter_ctx policy = seccomp_init(SCMP_ACT_ERRNO(EPERM));
if (!policy)
return 1;
int result = 0;
for (size_t i = 0; i < sizeof(calls) / sizeof(calls[0]); i++)
result |= allow(policy, calls[i]);
result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(prlimit64), 1, SCMP_A0(SCMP_CMP_EQ, 0));
for (size_t i = 0; i < sizeof(commands) / sizeof(commands[0]); i++)
result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 1, SCMP_A1(SCMP_CMP_EQ, commands[i]));
result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(fcntl), 2,
SCMP_A1(SCMP_CMP_EQ, F_SETFL), SCMP_A2(SCMP_CMP_MASKED_EQ, O_ASYNC, 0));
result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIOCLEX));
result |= seccomp_rule_add(policy, SCMP_ACT_ALLOW, SCMP_SYS(ioctl), 1, SCMP_A1(SCMP_CMP_EQ, FIONCLEX));
int output = open(argv[1], O_WRONLY | O_CREAT | O_TRUNC, 0444);
if (output < 0)
result = -1;
if (!result)
result = seccomp_export_bpf(policy, output);
if (output >= 0)
close(output);
seccomp_release(policy);
if (result)
fputs("Could not build the Python syscall policy\n", stderr);
return result ? 1 : 0;
}

41
deploy/lens/python_runtime.py Normal file
View file

@ -0,0 +1,41 @@
import json
import subprocess
import sys
import sysconfig
from itertools import chain
from pathlib import Path
from typing import Final
def dependencies(path: Path, loader: Path) -> tuple[Path, ...]:
result: Final = subprocess.run((str(loader), "--list", str(path)), capture_output=True, text=True, check=True)
if "not found" in result.stdout:
raise RuntimeError(f"Missing Python runtime library: {path}")
words: Final = tuple(result.stdout.split())
return tuple(Path(word).resolve() for word in words if word.startswith("/"))
def main() -> None:
stdlib: Final = Path(sysconfig.get_path("stdlib")).resolve()
executable: Final = Path(sys.executable).resolve()
loaders: Final = tuple(Path("/usr/lib").glob("ld-linux-*.so.*"))
if len(loaders) != 1:
raise RuntimeError("Expected one native glibc dynamic loader in the Lens worker image")
entries: Final = tuple(
path for path in stdlib.iterdir() if path.name not in ("site-packages", "dist-packages", "__pycache__")
)
extensions: Final = tuple((stdlib / "lib-dynload").glob("*.so"))
libraries: Final = frozenset(
chain.from_iterable(dependencies(binary, loaders[0]) for binary in (executable, *extensions))
)
manifest: Final = {
"executable": str(executable),
"directories": (str(stdlib),),
"read": tuple(sorted(str(path) for path in {*entries, *libraries})),
"execute": tuple(sorted(str(path) for path in (executable, *loaders))),
}
Path(sys.argv[1]).write_text(json.dumps(manifest), encoding="utf-8")
if __name__ == "__main__":
main()

1
deploy/lens/stack.yaml
View file

@ -40,6 +40,7 @@ services:
environment:
LITELLM_URL: http://litellm:4000
LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
LENS_PYTHON_CONCURRENCY: ${LENS_PYTHON_CONCURRENCY:-2}
depends_on: [litellm]
networks: [proxy]
restart: unless-stopped

2
docker/Dockerfile.database
View file

@ -77,6 +77,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
@ -99,6 +100,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.
RUN uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \

2
docker/Dockerfile.non_root
View file

@ -81,6 +81,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
@ -108,6 +109,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \

26
docker/README.md
View file

@ -70,6 +70,32 @@ To stop the running containers, use the following command:
docker compose down
```
## Embedded LiteAdmin MCP
Source builds containing embedded LiteAdmin MCP can serve it at `/admin/mcp` on the existing LiteLLM port. This capability is unreleased. Keep your existing database, master key, and proxy configuration, then add these settings to the serving container's environment:
```bash
LITELLM_ENABLE_ADMIN_MCP=true
LITELLM_LICENSE="your-enterprise-license"
PROXY_BASE_URL=https://gateway.example.com
```
For the unified source deployment described above, put them in its `.env` file and rebuild:
```bash
docker compose up -d --build
```
In componentized deployments, set the flag and license on the backend container and route `/admin/mcp` to the backend service. The gateway component excludes this endpoint. The unified, database, non-root, and backend image builds bundle the connector
Hosting is disabled by default. Opting in requires a valid base Enterprise license; an unlicensed opt-in or invalid flag value prevents startup. Enabling it reserves `/admin`, so rename any MCP server alias called `admin` first
With native key authentication, connect with a personal proxy-admin bearer key. When `enable_oauth2_proxy_auth` is enabled, the existing trusted-proxy identity headers select the user instead; the MCP bearer is required by the connector but does not select the native user. The resolved user must have the stored `proxy_admin` role, and `trusted_proxy_ranges` applies to the original caller's direct peer
Embedded responses default to `full`; selecting `LITELLM_ADMIN_RESPONSE_VIEW=compact` requires subsequent saved-result reads to reach the same worker process, including within a multi-worker pod
See the [LiteAdmin MCP guide](https://docs.litellm.ai/docs/proxy/liteadmin_mcp#run-liteadmin-mcp-inside-litellm) for client configuration, tool restrictions, and verification
## Hardened / Offline Testing
To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:

2
docker/component_entrypoint.sh
View file

@ -3,7 +3,7 @@
# stale samples from a previous container incarnation would be summed into the aggregate
if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db
rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_*
fi
case "$USE_DDTRACE" in

2
enterprise/litellm_enterprise/enterprise_callbacks/callback_controls.py
View file

@ -7,7 +7,6 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.llm_request_utils import (
get_proxy_server_request_headers,
)
from litellm.proxy._types import CommonProxyErrors
from litellm.types.utils import StandardCallbackDynamicParams
@ -86,6 +85,7 @@ class EnterpriseCallbackControls:
@staticmethod
def _should_allow_dynamic_callback_disabling():
import litellm
from litellm.proxy._types import CommonProxyErrors
from litellm.proxy.proxy_server import premium_user
# Check if admin has disabled this feature

6
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -46,6 +46,8 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
WebhookEvent,
)
from litellm.repositories.table_repositories import InvitationLinkRepository
from litellm.repositories.user_repository import UserRepository
from litellm.secret_managers.main import get_secret_bool
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
@ -844,7 +846,7 @@ class BaseEmailLogger(CustomLogger):
)
return None
user_row = await prisma_client.db.litellm_usertable.find_unique(
user_row = await UserRepository(prisma_client).table.find_unique(
where={"user_id": user_id}
)
@ -929,7 +931,7 @@ class BaseEmailLogger(CustomLogger):
try:
# Try to get existing invitation
existing_invitations = (
await prisma_client.db.litellm_invitationlink.find_many(
await InvitationLinkRepository(prisma_client).table.find_many(
where={"user_id": user_id},
order={"created_at": "desc"},
)

18
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -15,6 +15,10 @@ from litellm.constants import (
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
)
from litellm.repositories.table_repositories import ManagedObjectRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
if TYPE_CHECKING:
from prisma import models as prisma_models
@ -58,25 +62,19 @@ class _ManagedObjectRow(Protocol):
def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
return table
return ManagedObjectRepository(prisma_client).table
def _user_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_UserTable]":
table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = prisma_client.db.litellm_usertable
return table
return UserRepository(prisma_client).table
def _token_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = (
prisma_client.db.litellm_verificationtoken
)
return table
return VerificationTokenRepository(prisma_client).table
def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = prisma_client.db.litellm_teamtable
return table
return TeamRepository(prisma_client).table
class CheckBatchCost:

4
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -16,6 +16,7 @@ from litellm.constants import (
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
from litellm.repositories.table_repositories import ManagedObjectRepository
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
@ -43,8 +44,7 @@ class _ManagedObjectRow(Protocol):
def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_ManagedObjectRow]":
table: Final[TableActions[_ManagedObjectRow]] = prisma_client.db.litellm_managedobjecttable
return table
return ManagedObjectRepository(prisma_client).table
class CheckResponsesCost:

4
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -1647,7 +1647,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
owner_filter: Final = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return FileListPage(**build_list_page([]))
return FileListPage.model_validate(build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
@ -1686,7 +1686,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
def _is_batch_polling_enabled(self) -> bool:
"""

7
enterprise/litellm_enterprise/proxy/liteadmin.py
View file

@ -15,11 +15,12 @@ from urllib.parse import urlencode, urlsplit
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse, Response
from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
from pydantic import ConfigDict, Field, SecretStr, TypeAdapter, ValidationError
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.proxy._experimental.mcp_server.oauth_utils import get_request_base_url
from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
router: Final = APIRouter()
@ -34,14 +35,14 @@ _HEADERS: Final = {
}
class LinkDetails(BaseModel):
class LinkDetails(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, strict=True, extra="forbid")
workspace_id: str = Field(min_length=1, max_length=64)
slack_user_id: str = Field(min_length=1, max_length=64)
email: str = Field(min_length=1, max_length=320)
class AdminSession(BaseModel):
class AdminSession(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
user_id: str
credential: SecretStr

2
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -22,7 +22,7 @@ from litellm._uuid import uuid
from litellm.proxy._types import *
from litellm.proxy.auth.auth_checks import delete_cached_project_object
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management.teams.access import is_team_admin
from litellm.proxy.management.teams.authz import is_team_admin
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
from litellm.proxy.management_helpers.utils import (

13
enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py
View file

@ -1,12 +1,13 @@
import enum
from typing import Dict, List, Optional
from pydantic import BaseModel, Field
from pydantic import Field
from litellm.proxy._types import WebhookEvent
from litellm.types.llms.base import LiteLLMBaseModel
class EmailParams(BaseModel):
class EmailParams(LiteLLMBaseModel):
logo_url: str
support_contact: str
base_url: str
@ -39,14 +40,14 @@ class EmailEvent(str, enum.Enum):
soft_budget_crossed = "Soft Budget Crossed"
max_budget_alert = "Max Budget Alert"
class EmailEventSettings(BaseModel):
class EmailEventSettings(LiteLLMBaseModel):
event: EmailEvent
enabled: bool
class EmailEventSettingsUpdateRequest(BaseModel):
class EmailEventSettingsUpdateRequest(LiteLLMBaseModel):
settings: List[EmailEventSettings]
class EmailEventSettingsResponse(BaseModel):
class EmailEventSettingsResponse(LiteLLMBaseModel):
settings: List[EmailEventSettings]
class DefaultEmailSettings(BaseModel):
class DefaultEmailSettings(LiteLLMBaseModel):
"""Default settings for email events"""
settings: Dict[EmailEvent, bool] = Field(
default_factory=lambda: {

8
enterprise/litellm_enterprise/types/proxy/audit_logging_endpoints.py
View file

@ -1,10 +1,12 @@
from datetime import datetime
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
from pydantic import Field
from litellm.types.llms.base import LiteLLMBaseModel
class AuditLogResponse(BaseModel):
class AuditLogResponse(LiteLLMBaseModel):
"""Response model for a single audit log entry"""
id: str
@ -18,7 +20,7 @@ class AuditLogResponse(BaseModel):
updated_values: Optional[Dict[str, Any]] = None
class PaginatedAuditLogResponse(BaseModel):
class PaginatedAuditLogResponse(LiteLLMBaseModel):
"""Response model for paginated audit logs"""
audit_logs: List[AuditLogResponse]

2
gateway/Dockerfile
View file

@ -60,6 +60,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \
@ -72,6 +73,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \

1
gateway/routes/allowlist.py
View file

@ -76,6 +76,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/containers",
"/v1/evals",
"/v1/traces",
"/v1/logs",
"/v1/memory",
"/queue/chat/",
# Google data plane (v1beta is the Google AI Studio version)

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_add_end_user_models/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_EndUserTable" ADD COLUMN IF NOT EXISTS "models" TEXT[] DEFAULT ARRAY[]::TEXT[];

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230000_lens_trace_findings_indexes/migration.sql Normal file
View file

@ -0,0 +1,3 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_LensRun_completed_executions_idx"
ON "LiteLLM_LensRun" USING GIN ((data->'sample'->'executions') jsonb_path_ops)
WHERE data->>'status'='completed';

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261005230100_lens_current_findings_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_Lens_jobs_idx"
ON "LiteLLM_Lens" USING GIN ((data->'jobs') jsonb_path_ops);

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -656,6 +656,7 @@ model LiteLLM_EndUserTable {
spend Float @default(0.0)
allowed_model_region String? // require all user requests to use models in this specific region
default_model String? // use along with 'allowed_model_region'. if no available model in region, default to this model.
models String[] @default([])
budget_id String?
object_permission_id String?
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])

66
litellm-rust/.agents/skills/rust-string-enums/SKILL.md Normal file
View file

@ -0,0 +1,66 @@
---
name: rust-string-enums
description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
---
# Rust string enums
Use this skill when adding or changing enums represented by a single string, or surveying handwritten string conversions
## Choose the representation
For an enum with fixed spellings and an unknown-string fallback, prefer `strum::EnumString` and `strum::Display` together with `serde_with::DeserializeFromStr` and `serde_with::SerializeDisplay`. Keep each wire spelling in the Strum attributes instead of repeating it in a handwritten Serde match
Strum implements string conversion traits, not Serde traits. `EnumString` implements `FromStr`; `Display` formats the wire string. The two `serde_with` derives connect those traits to Serde. `AsRefStr` provides a borrowed string accessor and is optional
```rust
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::AsRefStr,
strum::EnumString,
strum::Display,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
pub enum EventType {
#[strum(serialize = "event.created")]
Created,
#[strum(default, transparent)]
Other(String),
}
```
Use `#[strum(serialize_all = "snake_case")]` or another supported case style when it exactly matches the contract. Use explicit variant spellings otherwise. With multiple accepted aliases, set `to_string` to the existing canonical output: Strum Display otherwise selects the longest `serialize` spelling
Use only the derives the contract needs. A deserialize-only type should remain deserialize-only. Do not add an unknown variant to a closed enum, or derive Serde for a type that currently has no serialization contract
Plain Serde derives with `rename` or `rename_all` remain appropriate for closed unit enums. Adding Strum and serde_with solely to replace working Serde derives adds little value. When a type has both Serde and Strum parsing, compare their accepted inputs before sharing the parser: case-insensitive Strum parsing must not silently make strict JSON parsing case-insensitive
## Preserve behavior during migration
Read the type, its callers, and existing tests before changing it. Preserve canonical output, accepted aliases, case sensitivity, whitespace handling, unknown values, malformed-input rejection, and existing public conversion APIs. Keep conversions needed by callers or compatibility even when Serde no longer uses them
Use the workspace dependencies and enable `serde_with.workspace = true` in a crate only when needed. Check the versions and enabled features in `Cargo.toml` and `Cargo.lock` rather than upgrading dependencies for this refactor
Remove replaced manual Serde implementations and obsolete Serde conversion attributes. Do not combine the new derives with `wire_type`, `request_type`, or `response_type` aliases that already derive the same Serde traits. Expand the necessary non-Serde derives and schema attributes locally rather than changing shared aliases for unrelated types
Preserve the generated schema, including titles and definition names. Open string enums need a string schema, including unknown values. When replacing Serde `from`/`into` attributes that previously supplied that schema, retain their schema behavior with `#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]` and compare the full generated result. `schemars(with = "String")` also makes a string schema, but changes the schema name and title, so keep it only where it already matches the contract
Keep custom `FromStr` and `Display` implementations for structured strings or validation that Strum does not express faithfully. Their Serde adapters can still use `DeserializeFromStr` and `SerializeDisplay`. Do not replace JSON visitors, tagged payload enums, permissive value wrappers, or domain transformations with string parsing
For a requested survey, document candidates and exceptions without migrating source. If the user asks to approve a bulk migration, present the concrete scope and wait for that approval
## Verify the contract
Extend existing mapped tests with named `rstest` cases. Assert both parsing into the expected variant and serialization to the expected wire string. Include unknown and empty strings for open enums, accepted aliases when present, and rejection of non-string JSON. A decode-only assertion does not prove a round trip
Test structured parsers with valid and invalid payloads, including their existing error behavior. For types with schema support, check the string schema and run the affected crate tests with the schema feature enabled. Run affected downstream checks when conversion APIs or derive aliases change
## Upstream references
The workspace used Strum 0.28.0 and serde_with 3.16.1 when this guidance was written. Consult the matching version of the [EnumString docs](https://docs.rs/strum/0.28.0/strum/derive.EnumString.html), [Display docs](https://docs.rs/strum_macros/0.28.0/strum_macros/derive.Display.html), [DeserializeFromStr docs](https://docs.rs/serde_with/3.16.1/serde_with/derive.DeserializeFromStr.html), and [SerializeDisplay docs](https://docs.rs/serde_with/3.16.1/serde_with/derive.SerializeDisplay.html). Schemars documents [schema overrides and Serde conversion attributes](https://docs.rs/schemars/1.2.2/schemars/derive.JsonSchema.html)

2
litellm-rust/AGENTS.md
View file

@ -2,6 +2,8 @@
For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.agents/skills/rust-tracing/SKILL.md)
For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md)
## Test placement
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`

44
litellm-rust/Cargo.lock generated
View file

@ -3711,7 +3711,7 @@ dependencies = [
"litellm-cache-memory",
"litellm-cache-response",
"litellm-core-utils",
"litellm-framing",
"litellm-framer",
"litellm-host",
"litellm-host-native",
"litellm-http",
@ -3796,7 +3796,7 @@ dependencies = [
]
[[package]]
name = "litellm-framing"
name = "litellm-framer"
version = "0.1.0"
dependencies = [
"aws-smithy-eventstream",
@ -4053,7 +4053,7 @@ dependencies = [
"litellm-auth-azure",
"litellm-auth-gcp",
"litellm-core-utils",
"litellm-framing",
"litellm-framer",
"litellm-host",
"litellm-http",
"litellm-llms-types",
@ -4085,26 +4085,6 @@ dependencies = [
"strum",
]
[[package]]
name = "litellm-migrate"
version = "0.1.0"
dependencies = [
"litellm-migrate-macros",
"rstest",
]
[[package]]
name = "litellm-migrate-macros"
version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
"rstest",
"syn 2.0.119",
"tempfile",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-model-catalog"
version = "0.1.0"
@ -4132,16 +4112,9 @@ dependencies = [
"litellm-auth",
"litellm-auth-aws",
"litellm-cache",
"litellm-cache-azure-blob",
"litellm-cache-disk",
"litellm-cache-gcs",
"litellm-cache-memory",
"litellm-cache-qdrant-semantic",
"litellm-cache-redis",
"litellm-cache-redis-semantic",
"litellm-cache-response",
"litellm-cache-s3",
"litellm-cache-valkey-semantic",
"litellm-callbacks-legacy-python",
"litellm-core",
"litellm-core-utils",
@ -4162,20 +4135,18 @@ dependencies = [
"prost",
"pyo3",
"pyo3-async-runtimes",
"qdrant-client",
"redis",
"reqwest 0.12.28",
"rstest",
"serde",
"serde_json",
"serde_with",
"sha2 0.10.9",
"sqlx",
"strum",
"thiserror 2.0.19",
"tokio",
"tokio-tungstenite",
"tracing",
"url",
"veil",
"wiremock",
]
@ -4368,6 +4339,9 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"serde_with",
"sqlx",
"testcontainers-modules",
"thiserror 2.0.19",
"tokio",
"url",
@ -4466,6 +4440,8 @@ dependencies = [
"schemars 1.2.2",
"serde",
"serde_json",
"serde_with",
"sha2 0.10.9",
"strum",
"thiserror 2.0.19",
"time",
@ -4498,7 +4474,6 @@ dependencies = [
"hmac 0.12.1",
"jsonschema",
"litellm-http",
"litellm-migrate",
"litellm-storage-clickhouse",
"litellm-traces",
"litellm-traces-cache",
@ -4509,6 +4484,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"sqlx",
"strum",
"testcontainers-modules",
"thiserror 2.0.19",

4
litellm-rust/Cargo.toml
View file

@ -16,8 +16,6 @@ litellm-traces = { path = "crates/traces" }
litellm-traces-cache = { path = "crates/traces-cache" }
litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
litellm-migrate = { path = "crates/migrate" }
litellm-migrate-macros = { path = "crates/migrate-macros" }
litellm-core = { path = "crates/core" }
litellm-gateway-mcp = { path = "crates/gateway-mcp" }
litellm-gateway = { path = "crates/gateway" }
@ -30,7 +28,7 @@ litellm-host = { path = "crates/host" }
litellm-host-http = { path = "crates/host-http" }
litellm-host-native = { path = "crates/host-native" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-framing = { path = "crates/framer" }
litellm-framer = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }
litellm-auth-types = { path = "crates/auth-types" }
litellm-auth-aws = { path = "crates/auth-aws" }

2
litellm-rust/crates/core/AGENTS.md
View file

@ -28,7 +28,7 @@ The workspace `Error definitions` rules shape each crate's error; this section d
A failure is declared once, by the lowest crate that raises it. Every crate above nests that error unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`. `RouteError` collects route failures and never re-declares a variant a lower crate raises
Scope follows the concept, not the first caller. An error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it. A failure two providers or two routes can hit, such as wire framing, stream event decoding, or a malformed provider response, belongs to the crate that owns the concept: `litellm-framing` for framing, `litellm_llms::Error` for the transformation layer
Scope follows the concept, not the first caller. An error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it. A failure two providers or two routes can hit, such as wire framing, stream event decoding, or a malformed provider response, belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer
`litellm_llms::Error` (`crates/llms/src/error.rs`) is the one transformation error for every provider and API. `base_llm/ocr/error.rs` is the recorded exception until OCR folds into it

2
litellm-rust/crates/core/Cargo.toml
View file

@ -8,7 +8,7 @@ repository.workspace = true
[dependencies]
litellm-cache.workspace = true
litellm-cache-response.workspace = true
litellm-framing.workspace = true
litellm-framer.workspace = true
tokio-util = { version = "0.7", features = ["codec"] }
litellm-secrets.workspace = true
litellm-llms-types.workspace = true

2
litellm-rust/crates/core/src/caching.rs
View file

@ -341,7 +341,7 @@ fn now() -> Duration {
fn successful_stream(text: &str, terminal: &str) -> bool {
let mut pending = BytesMut::from(text.as_bytes());
let mut codec = litellm_framing::sse::SseCodec::default();
let mut codec = litellm_framer::sse::SseCodec::default();
let mut complete = false;
loop {
let event = match codec.decode(&mut pending) {

2
litellm-rust/crates/framer/Cargo.toml
View file

@ -1,5 +1,5 @@
[package]
name = "litellm-framing"
name = "litellm-framer"
version = "0.1.0"
edition.workspace = true
license.workspace = true

2
litellm-rust/crates/framer/tests/aws_event_stream.rs
View file

@ -6,7 +6,7 @@ use std::io;
use bytes::Bytes;
use futures_util::{StreamExt, TryStreamExt, stream};
use litellm_framing::{
use litellm_framer::{
EventStreamError,
aws_event_stream::{AwsEventStreamCodec, Header, HeaderValue, Message},
frames,

2
litellm-rust/crates/framer/tests/chaining.rs
View file

@ -4,7 +4,7 @@ mod support;
use bytes::Bytes;
use futures_util::{StreamExt, TryStreamExt};
use litellm_framing::{
use litellm_framer::{
EventStreamError, SseError,
aws_event_stream::{AwsEventStreamCodec, Message},
frames,

2
litellm-rust/crates/framer/tests/sse.rs
View file

@ -6,7 +6,7 @@ use std::io;
use bytes::Bytes;
use futures_util::{StreamExt, TryStreamExt, stream};
use litellm_framing::{
use litellm_framer::{
SseError, frames,
sse::{SseCodec, SseEvent},
};

15
litellm-rust/crates/llms-types/src/formats/messages/request.rs
View file

@ -18,9 +18,18 @@ pub enum MessageContent {
Blocks(Vec<ContentBlock>),
}
#[macro_rules_attribute::apply(wire_type)]
#[derive(Eq, strum::Display, strum::EnumString)]
#[serde(from = "String", into = "String")]
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::Display,
strum::EnumString,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
#[strum(serialize_all = "snake_case")]
pub enum ContentBlockType {
Text,

55
litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs
View file

@ -1,7 +1,16 @@
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Map, Value};
#[derive(Clone, Debug, PartialEq, Eq, strum::AsRefStr)]
#[derive(
Clone,
Debug,
PartialEq,
Eq,
strum::AsRefStr,
strum::EnumString,
strum::Display,
serde_with::DeserializeFromStr,
serde_with::SerializeDisplay,
)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schema", schemars(with = "String"))]
pub enum ResponsesWsEventType {
@ -27,33 +36,6 @@ impl ResponsesWsEventType {
}
}
impl Serialize for ResponsesWsEventType {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.as_str())
}
}
impl<'de> Deserialize<'de> for ResponsesWsEventType {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
Ok(match value.as_str() {
"response.create" => Self::ResponseCreate,
"response.created" => Self::ResponseCreated,
"response.completed" => Self::ResponseCompleted,
"response.failed" => Self::ResponseFailed,
"response.incomplete" => Self::ResponseIncomplete,
"error" => Self::Error,
_ => Self::Other(value),
})
}
}
#[macro_rules_attribute::apply(wire_type)]
pub struct ResponsesWsEvent {
#[serde(rename = "type")]
@ -114,21 +96,6 @@ mod tests {
use super::*;
#[rstest]
#[case::known("response.completed", ResponsesWsEventType::ResponseCompleted)]
#[case::unknown(
"response.output_text.delta",
ResponsesWsEventType::Other("response.output_text.delta".to_string())
)]
fn event_type_round_trips_known_and_unknown_values(
#[case] value: &str,
#[case] expected: ResponsesWsEventType,
) {
let actual: ResponsesWsEventType =
serde_json::from_str(&serde_json::to_string(value).unwrap()).expect("valid event type");
assert_eq!(actual, expected);
}
#[test]
fn error_frame_matches_proxy_shape() {
let frame = ResponsesErrorFrame::invalid_request("missing model");

21
litellm-rust/crates/llms-types/tests/messages_request.rs
View file

@ -2,6 +2,27 @@ use litellm_llms_types::formats::messages::{ContentBlock, ContentBlockType};
use rstest::rstest;
use serde_json::{Value, json};
#[rstest]
#[case::null(json!(null))]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!(["tool_use"]))]
#[case::object(json!({"type": "tool_use"}))]
fn content_block_type_rejects_non_string_json(#[case] value: Value) {
assert!(serde_json::from_value::<ContentBlockType>(value).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn content_block_type_schema_remains_a_string() {
let schema = schemars::schema_for!(ContentBlockType).to_value();
assert_eq!(schema.get("type"), Some(&json!("string")));
assert_eq!(
schema.get("title"),
Some(&json!(stringify!(ContentBlockType)))
);
}
#[rstest]
#[case::text("text", ContentBlockType::Text)]
#[case::thinking("thinking", ContentBlockType::Thinking)]

48
litellm-rust/crates/llms-types/tests/responses.rs Normal file
View file

@ -0,0 +1,48 @@
use litellm_llms_types::formats::responses::streaming_websocket::ResponsesWsEventType;
use rstest::rstest;
#[rstest]
#[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
#[case::created("response.created", ResponsesWsEventType::ResponseCreated)]
#[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)]
#[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)]
#[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)]
#[case::error("error", ResponsesWsEventType::Error)]
#[case::unknown(
"response.output_text.delta",
ResponsesWsEventType::Other("response.output_text.delta".to_string())
)]
#[case::empty("", ResponsesWsEventType::Other(String::new()))]
#[case::case_sensitive(
"Response.Completed",
ResponsesWsEventType::Other("Response.Completed".into())
)]
#[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))]
fn websocket_event_type_round_trips(#[case] wire: &str, #[case] expected: ResponsesWsEventType) {
let serialized = serde_json::to_string(&expected).unwrap();
assert_eq!(serialized, serde_json::to_string(wire).unwrap());
assert_eq!(
serde_json::from_str::<ResponsesWsEventType>(&serialized).unwrap(),
expected
);
}
#[rstest]
#[case::number("17")]
#[case::boolean("true")]
#[case::null("null")]
#[case::array("[]")]
#[case::object("{}")]
fn websocket_event_type_rejects_non_strings(#[case] wire: &str) {
assert!(serde_json::from_str::<ResponsesWsEventType>(wire).is_err());
}
#[cfg(feature = "schema")]
#[rstest]
fn websocket_event_type_schema_is_open_string() {
let schema = schemars::schema_for!(ResponsesWsEventType);
assert_eq!(
schema.to_value().get("type"),
Some(&serde_json::json!("string"))
);
}

2
litellm-rust/crates/llms/Cargo.toml
View file

@ -16,7 +16,7 @@ litellm-auth-aws.workspace = true
litellm-auth-azure.workspace = true
litellm-auth-gcp.workspace = true
litellm-host.workspace = true
litellm-framing.workspace = true
litellm-framer.workspace = true
litellm-http.workspace = true
litellm-secrets.workspace = true
litellm-python-compat.workspace = true

72
litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs
View file

@ -6,7 +6,7 @@ use litellm_auth_azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAME
use litellm_core_utils::{call_arguments::CallArguments, url_utils::ApiUrl};
use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
use reqwest::Url;
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
use tokio::time::Instant;
@ -54,39 +54,22 @@ pub enum DocumentIntelligenceRequest {
},
}
#[derive(Clone, Debug, PartialEq)]
#[derive(
Clone, Debug, PartialEq, strum::EnumString, strum::Display, serde_with::DeserializeFromStr,
)]
enum OperationStatus {
#[strum(serialize = "succeeded")]
Succeeded,
#[strum(serialize = "running")]
Running,
#[strum(serialize = "notStarted")]
NotStarted,
#[strum(serialize = "failed")]
Failed,
#[strum(default, transparent)]
Unknown(String),
}
impl<'de> Deserialize<'de> for OperationStatus {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
Ok(match String::deserialize(deserializer)?.as_str() {
"succeeded" => Self::Succeeded,
"running" => Self::Running,
"notStarted" => Self::NotStarted,
"failed" => Self::Failed,
value => Self::Unknown(value.to_string()),
})
}
}
impl std::fmt::Display for OperationStatus {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Succeeded => "succeeded",
Self::Running => "running",
Self::NotStarted => "notStarted",
Self::Failed => "failed",
Self::Unknown(value) => value,
})
}
}
#[derive(Clone, Debug, Deserialize)]
pub struct AzureDocumentIntelligenceOperation {
status: Option<OperationStatus>,
@ -671,6 +654,43 @@ mod tests {
use super::*;
#[rstest]
#[case::succeeded("succeeded", OperationStatus::Succeeded)]
#[case::running("running", OperationStatus::Running)]
#[case::not_started("notStarted", OperationStatus::NotStarted)]
#[case::failed("failed", OperationStatus::Failed)]
fn operation_status_parses_known_values(
#[case] input: &str,
#[case] expected: OperationStatus,
) {
let parsed = serde_json::from_value::<OperationStatus>(json!(input)).unwrap();
assert_eq!(parsed, expected);
assert_eq!(parsed.to_string(), input);
}
#[rstest]
#[case::unknown("queued")]
#[case::case_sensitive("NotStarted")]
#[case::escaped("future\"\\\n")]
#[case::empty("")]
fn operation_status_preserves_unknown_values(#[case] input: &str) {
let parsed = serde_json::from_value::<OperationStatus>(json!(input)).unwrap();
assert_eq!(parsed, OperationStatus::Unknown(input.into()));
assert_eq!(parsed.to_string(), input);
}
#[rstest]
#[case::number(json!(1))]
#[case::boolean(json!(true))]
#[case::array(json!([]))]
#[case::null(Value::Null)]
#[case::object(json!({"status": "succeeded"}))]
fn operation_status_rejects_non_string_json(#[case] input: Value) {
assert!(serde_json::from_value::<OperationStatus>(input).is_err());
}
fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
let arguments = serde_json::from_value(value).unwrap();
AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")

2
litellm-rust/crates/llms/src/base_llm/messages/streaming.rs
View file

@ -1,6 +1,6 @@
use bytes::Bytes;
use futures_util::{StreamExt, stream::BoxStream};
use litellm_framing::{frames, sse::SseCodec};
use litellm_framer::{frames, sse::SseCodec};
use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent;
use crate::Error;

2
litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs
View file

@ -1,7 +1,7 @@
use base64::Engine;
use bytes::Buf;
use futures_util::{Stream, StreamExt};
use litellm_framing::{
use litellm_framer::{
aws_event_stream::{AwsEventStreamCodec, Message},
frames,
};

19
litellm-rust/crates/migrate-macros/Cargo.toml
View file

@ -1,19 +0,0 @@
[package]
name = "litellm-migrate-macros"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[lib]
proc-macro = true
[dependencies]
proc-macro2.workspace = true
quote.workspace = true
syn = { workspace = true, features = ["parsing", "printing", "proc-macro"] }
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
tempfile.workspace = true

21
litellm-rust/crates/migrate-macros/src/error.rs
View file

@ -1,21 +0,0 @@
use std::io;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("could not read migrations directory `{path}`")]
ReadDirectory {
path: String,
#[source]
source: io::Error,
},
#[error(
"migration name `{name}` must be `<digits>_<description>.sql` with a `[a-z0-9_]` description"
)]
InvalidName { name: String },
#[error("migration version `{version}` is declared more than once")]
DuplicateVersion { version: u64 },
#[error("migrations directory `{path}` contains no migrations")]
Empty { path: String },
#[error("migration path `{path}` is not valid UTF-8")]
NonUtf8Path { path: String },
}

199
litellm-rust/crates/migrate-macros/src/lib.rs
View file

@ -1,199 +0,0 @@
mod error;
use std::path::{Path, PathBuf};
use error::Error;
use proc_macro::TokenStream;
use quote::quote;
use syn::LitStr;
struct Entry {
version: u64,
description: String,
path: PathBuf,
}
fn resolve(dir: &Path) -> Result<Vec<Entry>, Error> {
let mut entries = Vec::new();
let files = std::fs::read_dir(dir).map_err(|source| Error::ReadDirectory {
path: dir.display().to_string(),
source,
})?;
for file in files {
let file = file.map_err(|source| Error::ReadDirectory {
path: dir.display().to_string(),
source,
})?;
let path = file.path();
let name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| Error::NonUtf8Path {
path: path.display().to_string(),
})?
.to_owned();
let invalid = || Error::InvalidName { name: name.clone() };
let stem = name
.strip_suffix(".sql")
.filter(|_| file.file_type().is_ok_and(|kind| kind.is_file()))
.and_then(|stem| stem.split_once('_'))
.filter(|(version, description)| {
!version.is_empty()
&& version.bytes().all(|b| b.is_ascii_digit())
&& !description.is_empty()
&& description
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
})
.ok_or_else(invalid)?;
let version = stem.0.parse::<u64>().map_err(|_| invalid())?;
entries.push(Entry {
version,
description: stem.1.to_owned(),
path,
});
}
if entries.is_empty() {
return Err(Error::Empty {
path: dir.display().to_string(),
});
}
entries.sort_by_key(|entry| entry.version);
for pair in entries.windows(2) {
if pair[0].version == pair[1].version {
return Err(Error::DuplicateVersion {
version: pair[0].version,
});
}
}
Ok(entries)
}
fn resolve_input(lit: &LitStr) -> Result<Vec<Entry>, Error> {
let root = std::env::var("CARGO_MANIFEST_DIR")
.map(PathBuf::from)
.unwrap_or_default();
let dir = root.join(lit.value());
let dir = dir.canonicalize().map_err(|source| Error::ReadDirectory {
path: dir.display().to_string(),
source,
})?;
if dir.to_str().is_none() {
return Err(Error::NonUtf8Path {
path: dir.display().to_string(),
});
}
resolve(&dir)
}
#[proc_macro]
pub fn migrate(input: TokenStream) -> TokenStream {
let lit = syn::parse_macro_input!(input as LitStr);
match resolve_input(&lit) {
Ok(entries) => {
let migrations = entries.iter().map(|entry| {
let version = entry.version;
let description = &entry.description;
let path = entry
.path
.to_str()
.expect("canonical migration path is UTF-8");
quote! {
::litellm_migrate::Migration {
version: #version,
description: #description,
sql: ::core::include_str!(#path),
}
}
});
quote! { &[#(#migrations),*] }.into()
}
Err(err) => syn::Error::new(lit.span(), err).to_compile_error().into(),
}
}
#[cfg(test)]
mod tests {
use std::fs;
use rstest::rstest;
use tempfile::TempDir;
use super::{Error, resolve};
fn migrations_dir(files: &[&str]) -> TempDir {
let dir = TempDir::new().expect("tempdir");
for file in files {
fs::write(dir.path().join(file), "SELECT 1").expect("write fixture");
}
dir
}
#[rstest]
fn orders_versions_numerically() {
let dir = migrations_dir(&["10_tenth.sql", "2_second.sql", "1_first.sql"]);
let entries = resolve(dir.path()).expect("resolves");
let versions: Vec<u64> = entries.iter().map(|entry| entry.version).collect();
let descriptions: Vec<&str> = entries
.iter()
.map(|entry| entry.description.as_str())
.collect();
assert_eq!(versions, [1, 2, 10]);
assert_eq!(descriptions, ["first", "second", "tenth"]);
}
#[rstest]
#[case::dash_in_version(&["0001-dash.sql"])]
#[case::not_sql(&["notes.txt"])]
#[case::empty_description(&["0001_.sql"])]
#[case::non_digit_version(&["x_name.sql"])]
#[case::uppercase_description(&["0001_Upper.sql"])]
#[case::no_underscore(&["0001.sql"])]
#[case::plus_sign_version(&["+10_add.sql"])]
fn rejects_invalid_names(#[case] files: &[&str]) {
let dir = migrations_dir(files);
assert!(matches!(
resolve(dir.path()),
Err(Error::InvalidName { .. })
));
}
#[rstest]
fn rejects_subdirectories() {
let dir = migrations_dir(&["0001_a.sql"]);
fs::create_dir(dir.path().join("0002_b.sql")).expect("subdir");
assert!(matches!(
resolve(dir.path()),
Err(Error::InvalidName { .. })
));
}
#[cfg(unix)]
#[rstest]
fn rejects_symlinks() {
let dir = migrations_dir(&["0001_a.sql"]);
let target = TempDir::new().expect("tempdir");
let target_file = target.path().join("real.sql");
fs::write(&target_file, "SELECT 2").expect("write fixture");
std::os::unix::fs::symlink(&target_file, dir.path().join("0002_b.sql")).expect("symlink");
assert!(matches!(
resolve(dir.path()),
Err(Error::InvalidName { .. })
));
}
#[rstest]
fn rejects_duplicate_versions() {
let dir = migrations_dir(&["0001_a.sql", "1_b.sql"]);
assert!(matches!(
resolve(dir.path()),
Err(Error::DuplicateVersion { version: 1 })
));
}
#[rstest]
fn rejects_empty_directory() {
let dir = migrations_dir(&[]);
assert!(matches!(resolve(dir.path()), Err(Error::Empty { .. })));
}
}

12
litellm-rust/crates/migrate/Cargo.toml
View file

@ -1,12 +0,0 @@
[package]
name = "litellm-migrate"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-migrate-macros.workspace = true
[dev-dependencies]
rstest.workspace = true

5
litellm-rust/crates/migrate/README.md
View file

@ -1,5 +0,0 @@
# Migrations
`litellm-migrate` exports the `Migration` struct and the `migrate!` macro that embeds a directory of `<digits>_<description>.sql` files at compile time, sorted by numeric version
The crate does not apply or track migrations; callers decide how and when the embedded SQL runs

8
litellm-rust/crates/migrate/src/lib.rs
View file

@ -1,8 +0,0 @@
pub use litellm_migrate_macros::migrate;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Migration {
pub version: u64,
pub description: &'static str,
pub sql: &'static str,
}

1
litellm-rust/crates/migrate/tests/fixtures/migrations/10_tenth.sql vendored
View file

@ -1 +0,0 @@
SELECT 10;

1
litellm-rust/crates/migrate/tests/fixtures/migrations/1_first.sql vendored
View file

@ -1 +0,0 @@
SELECT 1;

1
litellm-rust/crates/migrate/tests/fixtures/migrations/2_second.sql vendored
View file

@ -1 +0,0 @@
SELECT 2;

21
litellm-rust/crates/migrate/tests/migrate.rs
View file

@ -1,21 +0,0 @@
use litellm_migrate::Migration;
use rstest::rstest;
const MIGRATIONS: &[Migration] = litellm_migrate::migrate!("tests/fixtures/migrations");
#[rstest]
#[case::first(0, 1, "first", include_str!("fixtures/migrations/1_first.sql"))]
#[case::second(1, 2, "second", include_str!("fixtures/migrations/2_second.sql"))]
#[case::tenth(2, 10, "tenth", include_str!("fixtures/migrations/10_tenth.sql"))]
fn embeds_every_file_sorted_by_numeric_version(
#[case] index: usize,
#[case] version: u64,
#[case] description: &str,
#[case] sql: &str,
) {
assert_eq!(MIGRATIONS.len(), 3);
let migration = &MIGRATIONS[index];
assert_eq!(migration.version, version);
assert_eq!(migration.description, description);
assert_eq!(migration.sql, sql);
}

11
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -29,17 +29,9 @@ litellm-host.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-cache.workspace = true
litellm-cache-azure-blob.workspace = true
litellm-cache-memory.workspace = true
litellm-cache-redis.workspace = true
litellm-cache-s3.workspace = true
litellm-cache-gcs.workspace = true
litellm-cache-disk.workspace = true
litellm-cache-redis-semantic.workspace = true
litellm-cache-response.workspace = true
litellm-cache-qdrant-semantic.workspace = true
qdrant-client.workspace = true
litellm-cache-valkey-semantic = { path = "../cache-valkey-semantic" }
serde.workspace = true
litellm-auth.workspace = true
litellm-auth-aws.workspace = true
@ -63,7 +55,6 @@ strum.workspace = true
veil.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "sync"] }
url.workspace = true
[dev-dependencies]
tracing.workspace = true
@ -74,7 +65,7 @@ serde_with.workspace = true
criterion.workspace = true
futures-util.workspace = true
rstest.workspace = true
sha2.workspace = true
sqlx = { workspace = true, features = ["migrate"] }
tokio-tungstenite.workspace = true
wiremock.workspace = true
aws-sdk-secretsmanager = "1.117.0"

12
litellm-rust/crates/python-bridge/src/cache/AGENTS.md vendored
View file

@ -1,13 +1,7 @@
# Cache boundary
This folder owns Python cache API compatibility: argument projection, facade identity, public result construction, Python embedding calls and per-operation composition of native backends. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates
This folder owns how Rust inference reaches the selected cache: global cache selection, route admission, delegation to a Python cache and the experimental V2 native handles. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates
`mod.rs` exposes the cache boundary to routes and module registration; adapter directories remain private. `selection.rs` owns global cache selection, route admission and inference protocol composition for both adapters. `runtime.rs` exposes the Python-facing runtime that can wrap either native storage or a Python callback. `future.rs` converts cache results into ready Futures
`mod.rs` exposes the cache boundary to routes and module registration; adapter directories remain private. `selection.rs` owns global cache selection, route admission and inference protocol composition for both adapters
`python/` delegates operations to the selected Python cache without discovering configuration. `native/` owns native backend construction, configuration projection, facade validation, embedding and storage bindings, including experimental V2 handles. Neither adapter depends on shared selection or the other adapter. Shared composition depends on the adapters, and routes use only the parent module's exports
`SemanticExecution` belongs here because its steps select cache operations and invoke the Python embedder. Use the shared `Execution` handle and inline lifecycle driver; do not duplicate coroutine state validation, runtime waiting or GIL machinery. Python embedding awaits stay in the caller's task, and cancellation must prevent later backend or batch operations from starting
Resolved asyncio Future construction is generic host machinery. Use `litellm-host-python::ready_future` with an already constructed Python value. Keep cache-specific conversion and disabled-cache return values here. Preserve the Future-returning API and running-loop requirement
Tests for Future mechanics belong in `host-python`; tests for disabled-cache values, embedding failure policy, batch sequencing and cancellation belong with this cache adapter. Assert public behavior rather than the location or name of a helper
`python/` delegates operations to the selected Python cache without discovering configuration. `native/` owns the experimental V2 native cache handles. Neither adapter depends on shared selection or the other adapter. Shared composition depends on the adapters, and routes use only the parent module's exports

13
litellm-rust/crates/python-bridge/src/cache/future.rs vendored
View file

@ -1,13 +0,0 @@
use litellm_host_python::{ready_future, to_py};
use pyo3::prelude::*;
pub(super) fn ready_none(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
ready_value(py, &())
}
pub(super) fn ready_value<'py, T: serde::Serialize>(
py: Python<'py>,
value: &T,
) -> PyResult<Bound<'py, PyAny>> {
ready_future(py, to_py(py, value)?.bind(py))
}

3
litellm-rust/crates/python-bridge/src/cache/mod.rs vendored
View file

@ -1,12 +1,9 @@
mod future;
mod native;
mod python;
mod runtime;
mod selection;
pub(crate) use native::NativeCacheHandle;
pub(crate) use python::{CacheCall, PythonCache};
pub(crate) use runtime::ResolvedCache;
pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native};
use litellm_cache::Error;

8
litellm-rust/crates/python-bridge/src/cache/native/AGENTS.md vendored
View file

@ -1,9 +1,9 @@
# Native cache bindings
This directory constructs and exposes Rust cache backends to Python. It owns backend configuration projection, facade validation, native request conversion, semantic embedding integration and experimental V2 handles. Cache algorithms and storage protocols remain in their cache crates
This directory exposes the experimental V2 native cache handles (`NativeCacheHandle`) to Python. Cache algorithms and storage protocols remain in their cache crates
Accept the cache object or projected configuration selected by the parent module. Do not read global `litellm.cache`, decide route admission, or select the Python cache adapter here
Accept the cache object selected by the parent module. Do not read global `litellm.cache`, decide route admission, or select the Python cache adapter here
Keep Python-facing cache classes and method signatures stable when reorganizing modules. Native internals stay private to this directory unless the shared cache boundary or Python module registration needs them. Python embedding awaits use the existing inline lifecycle driver, preserving caller task identity and cancellation
Keep Python-facing cache classes and method signatures stable when reorganizing modules. Native internals stay private to this directory unless the shared cache boundary or Python module registration needs them
Verify changes with the existing backend and facade tests using a freshly built extension. Test cache behavior, not module paths or file structure
Verify changes with the V2 cache tests using a freshly built extension. Test cache behavior, not module paths or file structure

112
litellm-rust/crates/python-bridge/src/cache/native/activation.rs vendored
View file

@ -1,112 +0,0 @@
use crate::cache::cache_error;
use crate::execution::run_sync_value;
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use litellm_cache_redis_semantic::RedisSemanticConfig;
use litellm_host_python::release_gil;
use litellm_http::ClientVariant;
use pyo3::prelude::*;
use super::{
backend::NativeResponseCache,
config::{CacheBackendConfig, NativeCacheConfig, UnsupportedCacheConfig},
embedder::PythonEmbedder,
};
use crate::errors::RustBridgeDeclined;
use crate::http::host_client;
fn declined(reason: UnsupportedCacheConfig) -> PyErr {
RustBridgeDeclined::new_err(reason.message())
}
/// Builds the native backend a `Cache` facade's projected configuration describes. `backend` is
/// the facade's `.cache` object, which owns embedding for the Python-embedded semantic caches.
pub(in crate::cache) fn activate(
py: Python<'_>,
backend: &Bound<'_, PyAny>,
config: NativeCacheConfig,
) -> PyResult<NativeResponseCache> {
let policy = config.policy;
let service = match config.backend {
CacheBackendConfig::Memory(memory) => {
NativeResponseCache::memory(memory.capacity, memory.default_ttl, memory.max_entry_bytes)
}
CacheBackendConfig::Redis(redis) => {
let url = redis.connection.native_url().map_err(declined)?;
let flush_size = policy.redis_flush_size.map(|_| redis.flush_size);
release_gil(py, move || {
NativeResponseCache::redis(
&url,
&redis.topology,
Some(redis.default_ttl),
redis.namespace,
)
})
.map_err(cache_error)?
.with_redis_flush_size(flush_size)
}
CacheBackendConfig::S3(s3) => {
let http = host_client(py, ClientVariant::NoRedirect)?;
run_sync_value(
py,
async move { Ok(NativeResponseCache::s3(*s3, http).await) },
)?
}
CacheBackendConfig::Gcs(gcs) => NativeResponseCache::gcs(
GcsConfig {
bucket_name: gcs.bucket_name,
gcs_path: Some(gcs.key_prefix),
path_service_account: gcs.path_service_account,
endpoint: DEFAULT_ENDPOINT.to_owned(),
},
host_client(py, ClientVariant::NoRedirect)?,
None,
),
CacheBackendConfig::Disk(disk) => {
release_gil(py, move || NativeResponseCache::disk(&disk.directory))
.map_err(cache_error)?
}
CacheBackendConfig::AzureBlob(azure) => {
let http = host_client(py, ClientVariant::NoRedirect)?;
run_sync_value(py, async move {
NativeResponseCache::azure_blob(&azure.account_url, &azure.container, http)
.await
.map_err(cache_error)
})?
}
CacheBackendConfig::RedisSemantic(semantic) => {
let url = semantic.native_url().map_err(declined)?.to_owned();
let embedder = PythonEmbedder::new(backend.clone().unbind());
let semantic_config = RedisSemanticConfig {
index_name: semantic.index_name,
similarity_threshold: semantic.similarity_threshold as f32,
};
release_gil(py, move || {
NativeResponseCache::redis_semantic(&url, embedder, semantic_config)
})
.map_err(cache_error)?
}
CacheBackendConfig::ValkeySemantic(valkey) => {
let url = valkey.connection.native_url().map_err(declined)?;
let embedder = PythonEmbedder::new(backend.clone().unbind());
release_gil(py, move || {
NativeResponseCache::valkey_semantic(
&url,
valkey.similarity_threshold,
valkey.index_name,
embedder,
)
})
.map_err(cache_error)?
}
CacheBackendConfig::QdrantSemantic(qdrant) => {
let client = host_client(py, ClientVariant::Provider)?;
run_sync_value(py, async move {
let runtime = tokio::runtime::Handle::current();
NativeResponseCache::qdrant_semantic(*qdrant, client, runtime)
.await
.map_err(cache_error)
})?
}
};
Ok(service.with_scope(policy.semantic_cache_scope))
}

690
litellm-rust/crates/python-bridge/src/cache/native/backend.rs vendored
View file

@ -1,690 +0,0 @@
use crate::cache::cache_error;
use std::{sync::Arc, time::Duration};
use litellm_cache::{CacheCodec, CacheConnectionResult, Error, semantic::SemanticLookup};
use litellm_cache_azure_blob::AzureBlobCache;
use litellm_cache_disk::DiskCache;
use litellm_cache_gcs::{GcsCache, GcsConfig, StaticTokenSource};
use litellm_cache_memory::InMemoryCache;
use litellm_cache_qdrant_semantic::{OpenAiEmbedder, QdrantSemanticCache};
use litellm_cache_redis::{RedisCache, RedisTopology};
use litellm_cache_redis_semantic::{RedisSemanticCache, RedisSemanticConfig};
use litellm_cache_response::{
ConnectionProbe, ExactResponseCache, PartialHits, ResponseCache, ResponseCacheCodec,
WriteBuffer,
};
use litellm_cache_s3::{S3Cache, S3CacheConfig};
use litellm_cache_valkey_semantic::{ValkeySemanticCache, ValkeySemanticConfig};
use pyo3::prelude::*;
use serde_json::Value;
use super::{
config::QdrantSemanticCacheConfig,
embedder::PythonEmbedder,
identity::BackendIdentity,
request::{NativeRequest, now},
semantic::{EmbeddingFailure, SemanticExecution, SemanticOperation, drive},
};
/// What the Python embedder receives for one semantic request.
pub(in crate::cache) struct EmbeddingInput {
pub(in crate::cache) prompt: String,
pub(in crate::cache) metadata: Option<Value>,
}
/// An exact-match backend behind one pointer, with the identity its facade must reproduce.
pub(in crate::cache) struct ExactService {
cache: Arc<dyn ExactResponseCache>,
probe: Option<Arc<dyn ConnectionProbe>>,
buffer: Option<WriteBuffer>,
identity: BackendIdentity,
}
#[derive(Clone)]
pub(in crate::cache) enum NativeResponseCache {
Exact(Arc<ExactService>),
ValkeySemantic {
cache: Arc<ResponseCache<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>>,
embedder: PythonEmbedder,
scope: String,
},
RedisSemantic {
cache: Arc<ResponseCache<RedisSemanticCache<PythonEmbedder, ResponseCacheCodec>>>,
embedder: PythonEmbedder,
},
QdrantSemantic(Arc<ResponseCache<QdrantSemanticCache<OpenAiEmbedder, ResponseCacheCodec>>>),
}
impl NativeResponseCache {
pub fn memory(capacity: usize, ttl: Duration, max_entry_bytes: usize) -> Self {
let backend = InMemoryCache::with_clock_and_size_measurement(
Some(capacity),
Some(ttl),
Some(max_entry_bytes),
Some(Arc::new(|entry| {
ResponseCacheCodec.encode(entry).map(|bytes| bytes.len())
})),
now,
);
let identity = BackendIdentity::Memory {
capacity: backend.max_size_in_memory(),
max_entry_bytes: backend.max_entry_bytes(),
default_ttl: None,
};
Self::exact(ResponseCache::new(Arc::new(backend)), identity)
}
pub fn redis(
url: &str,
topology: &RedisTopology,
ttl: Option<Duration>,
namespace: Option<String>,
) -> Result<Self, Error> {
let backend =
RedisCache::connect(url, topology, ttl, ResponseCacheCodec)?.with_namespace(namespace);
let identity = BackendIdentity::Redis {
topology: backend.topology().clone(),
namespace: backend.namespace().map(str::to_owned),
default_ttl: None,
};
Ok(Self::exact_probed(
ResponseCache::new(Arc::new(backend)),
identity,
))
}
pub async fn s3(config: S3CacheConfig, http: litellm_http::Client) -> Self {
let runtime = tokio::runtime::Handle::current();
let backend = S3Cache::new(config, http, ResponseCacheCodec, runtime);
let identity = BackendIdentity::S3 {
bucket: backend.bucket().to_owned(),
key_prefix: backend.key_prefix().to_owned(),
region: backend.region().to_owned(),
endpoint: backend.endpoint().map(str::to_owned),
};
Self::exact(ResponseCache::new(Arc::new(backend)), identity)
}
pub fn disk(directory: impl AsRef<std::path::Path>) -> Result<Self, Error> {
let backend = DiskCache::open(directory, ResponseCacheCodec)?;
let identity = BackendIdentity::Disk {
directory: backend.directory().to_path_buf(),
};
Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity))
}
pub fn gcs(config: GcsConfig, client: litellm_http::Client, token: Option<String>) -> Self {
let backend = match token {
Some(token) => GcsCache::with_token_source(
config,
client,
ResponseCacheCodec,
Arc::new(StaticTokenSource(token)),
),
None => GcsCache::new(config, client, ResponseCacheCodec),
};
let identity = BackendIdentity::Gcs {
bucket_name: backend.bucket_name().to_owned(),
key_prefix: backend.key_prefix().to_owned(),
path_service_account: backend.path_service_account().map(str::to_owned),
};
Self::exact(ResponseCache::new(Arc::new(backend)), identity)
}
pub async fn azure_blob(
account_url: &str,
container: &str,
http: litellm_http::Client,
) -> Result<Self, Error> {
let backend = AzureBlobCache::connect(
account_url,
container,
http,
ResponseCacheCodec,
tokio::runtime::Handle::current(),
)
.await?;
let identity = BackendIdentity::AzureBlob {
account_url: backend.account_url().to_owned(),
container: backend.container_name().to_owned(),
};
Ok(Self::exact(ResponseCache::new(Arc::new(backend)), identity))
}
/// Wraps a built exact backend; the TTL a facade must match comes from the built cache.
fn exact<B>(cache: ResponseCache<B>, identity: BackendIdentity) -> Self
where
ResponseCache<B>: ExactResponseCache + 'static,
B: litellm_cache::BaseCache<Value = litellm_cache_response::CacheEntry>,
B::Context: Default + PartialEq,
{
Self::exact_service(Arc::new(cache), None, identity)
}
/// Wraps an exact backend whose Python class defines `test_connection`.
fn exact_probed<B>(cache: ResponseCache<B>, identity: BackendIdentity) -> Self
where
ResponseCache<B>: ExactResponseCache + ConnectionProbe + 'static,
B: litellm_cache::BaseCache<Value = litellm_cache_response::CacheEntry>,
B::Context: Default + PartialEq,
{
let cache = Arc::new(cache);
Self::exact_service(cache.clone(), Some(cache), identity)
}
fn exact_service(
cache: Arc<dyn ExactResponseCache>,
probe: Option<Arc<dyn ConnectionProbe>>,
identity: BackendIdentity,
) -> Self {
let default_ttl = cache.default_ttl();
let identity = match identity {
BackendIdentity::Memory {
capacity,
max_entry_bytes,
..
} => BackendIdentity::Memory {
capacity,
max_entry_bytes,
default_ttl,
},
BackendIdentity::Redis {
topology,
namespace,
..
} => BackendIdentity::Redis {
topology,
namespace,
default_ttl,
},
other => other,
};
Self::Exact(Arc::new(ExactService {
cache,
probe,
buffer: None,
identity,
}))
}
pub fn valkey_semantic(
url: &str,
similarity_threshold: f64,
index_name: String,
embedder: PythonEmbedder,
) -> Result<Self, Error> {
let backend = ValkeySemanticCache::new(
url,
embedder.clone(),
ResponseCacheCodec,
ValkeySemanticConfig {
similarity_threshold,
index_name,
},
)?;
Ok(Self::ValkeySemantic {
cache: Arc::new(ResponseCache::new(Arc::new(backend))),
embedder,
scope: String::from("key"),
})
}
pub fn redis_semantic(
url: &str,
embedder: PythonEmbedder,
config: RedisSemanticConfig,
) -> Result<Self, Error> {
let backend = RedisSemanticCache::new(url, embedder.clone(), ResponseCacheCodec, config)?;
Ok(Self::RedisSemantic {
cache: Arc::new(ResponseCache::new(Arc::new(backend))),
embedder,
})
}
pub(super) async fn qdrant_semantic(
config: QdrantSemanticCacheConfig,
client: litellm_http::Client,
runtime: tokio::runtime::Handle,
) -> Result<Self, Error> {
let qdrant = qdrant_client::Qdrant::from_url(&config.grpc_url)
.skip_compatibility_check()
.api_key(config.api_key.as_deref())
.build()
.map_err(|_| Error::Unavailable)?;
let qdrant_config = config.to_qdrant_config();
let embedder = OpenAiEmbedder::new(client, config.embedding);
let cache = QdrantSemanticCache::connect(
qdrant,
embedder,
ResponseCacheCodec,
qdrant_config,
runtime,
)
.await?;
Ok(Self::QdrantSemantic(Arc::new(ResponseCache::new(
Arc::new(cache),
))))
}
pub fn identity(&self) -> BackendIdentity {
match self {
Self::Exact(service) => service.identity.clone(),
Self::ValkeySemantic { cache, .. } => BackendIdentity::ValkeySemantic {
index_name: cache.backend().index_name().to_owned(),
similarity_threshold: cache.backend().similarity_threshold(),
},
Self::RedisSemantic { cache, .. } => BackendIdentity::RedisSemantic {
index_name: cache.backend().index_name().to_owned(),
similarity_threshold: cache.backend().similarity_threshold(),
},
Self::QdrantSemantic(cache) => BackendIdentity::QdrantSemantic {
collection_name: cache.backend().collection_name().to_owned(),
similarity_threshold: cache.backend().similarity_threshold(),
vector_size: cache.backend().vector_size(),
embedding_model: cache.backend().embedder().model().to_owned(),
},
}
}
pub fn with_redis_flush_size(self, flush_size: Option<usize>) -> Self {
match self {
Self::Exact(service) if matches!(service.identity, BackendIdentity::Redis { .. }) => {
Self::Exact(Arc::new(ExactService {
cache: Arc::clone(&service.cache),
probe: service.probe.clone(),
buffer: flush_size.map(WriteBuffer::new),
identity: service.identity.clone(),
}))
}
value => value,
}
}
pub fn with_scope(self, scope: String) -> Self {
match self {
Self::ValkeySemantic {
cache, embedder, ..
} => Self::ValkeySemantic {
cache,
embedder,
scope,
},
value => value,
}
}
pub fn embedder_object(&self) -> Option<&Py<PyAny>> {
match self {
Self::RedisSemantic { embedder, .. } => Some(embedder.object()),
_ => None,
}
}
/// The prompt and metadata this backend would embed for `request`, if it has a prompt.
pub(in crate::cache) fn embedding_input(
&self,
request: &NativeRequest,
) -> Option<EmbeddingInput> {
let context = match self {
Self::ValkeySemantic { scope, .. } => request.scoped_semantic(scope).context,
Self::RedisSemantic { .. } => request.semantic().context,
Self::Exact(_) | Self::QdrantSemantic(_) => return None,
};
let prompt = litellm_cache::semantic::prompt_from_context(&context)?;
Some(EmbeddingInput {
prompt,
metadata: context.metadata,
})
}
/// Drives a semantic operation whose embedding comes from Python.
fn python_semantic<'py>(
&self,
py: Python<'py>,
operation: SemanticOperation,
) -> PyResult<Bound<'py, PyAny>> {
let (embedder, failure) = match self {
Self::ValkeySemantic { embedder, .. } => (embedder, EmbeddingFailure::Propagate),
Self::RedisSemantic { embedder, .. } => (embedder, EmbeddingFailure::Unavailable),
Self::Exact(_) | Self::QdrantSemantic(_) => {
return Err(pyo3::exceptions::PyRuntimeError::new_err(
"semantic execution requires a Python-embedded backend",
));
}
};
drive(
py,
SemanticExecution::new(self.clone(), embedder.clone(), failure, operation),
)
}
pub fn lookup(&self, request: &NativeRequest, now: Duration) -> Result<Option<Value>, Error> {
match self {
Self::Exact(service) => service.cache.lookup(&request.exact(), now),
Self::ValkeySemantic { cache, scope, .. } => {
cache.lookup(&request.scoped_semantic(scope), now)
}
Self::RedisSemantic { cache, .. } => cache.lookup(&request.semantic(), now),
Self::QdrantSemantic(cache) => cache.lookup(&request.semantic(), now),
}
}
/// `lookup` plus the similarity Python's semantic backend writes to the request metadata.
/// Exact backends report none.
pub fn lookup_semantic(
&self,
request: &NativeRequest,
now: Duration,
) -> Result<SemanticLookup<Value>, Error> {
match self {
Self::Exact(service) => service
.cache
.lookup(&request.exact(), now)
.map(exact_lookup),
Self::ValkeySemantic { cache, scope, .. } => {
redis_family(cache.lookup_semantic(&request.scoped_semantic(scope), now))
}
Self::RedisSemantic { cache, .. } => {
redis_family(cache.lookup_semantic(&request.semantic(), now))
}
Self::QdrantSemantic(cache) => cache.lookup_semantic(&request.semantic(), now),
}
}
pub fn store(
&self,
request: &NativeRequest,
response: Value,
now: Duration,
) -> Result<(), Error> {
match self {
Self::Exact(service) => service.cache.store(&request.exact(), response, now),
Self::ValkeySemantic { cache, scope, .. } => {
cache.store(&request.scoped_semantic(scope), response, now)
}
Self::RedisSemantic { cache, .. } => cache.store(&request.semantic(), response, now),
Self::QdrantSemantic(cache) => cache.store(&request.semantic(), response, now),
}
}
pub fn lookup_batch(
&self,
requests: &[NativeRequest],
now: Duration,
) -> Result<PartialHits, Error> {
match self {
Self::Exact(service) => service.cache.lookup_batch(&exact_requests(requests), now),
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => {
Err(Error::UnsupportedOperation)
}
}
}
pub async fn async_lookup(
&self,
request: &NativeRequest,
now: Duration,
) -> Result<Option<Value>, Error> {
match self {
Self::Exact(service) => service.cache.async_lookup(&request.exact(), now).await,
Self::ValkeySemantic { cache, scope, .. } => {
cache
.async_lookup(&request.scoped_semantic(scope), now)
.await
}
Self::RedisSemantic { cache, .. } => cache.async_lookup(&request.semantic(), now).await,
Self::QdrantSemantic(cache) => cache.async_lookup(&request.semantic(), now).await,
}
}
pub async fn async_lookup_semantic(
&self,
request: &NativeRequest,
now: Duration,
) -> Result<SemanticLookup<Value>, Error> {
match self {
Self::Exact(service) => service
.cache
.async_lookup(&request.exact(), now)
.await
.map(exact_lookup),
Self::ValkeySemantic { cache, scope, .. } => redis_family(
cache
.async_lookup_semantic(&request.scoped_semantic(scope), now)
.await,
),
Self::RedisSemantic { cache, .. } => {
redis_family(cache.async_lookup_semantic(&request.semantic(), now).await)
}
Self::QdrantSemantic(cache) => {
cache.async_lookup_semantic(&request.semantic(), now).await
}
}
}
pub(in crate::cache) fn async_lookup_semantic_py<'py>(
&self,
py: Python<'py>,
request: NativeRequest,
) -> PyResult<Bound<'py, PyAny>> {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
crate::execution::run_async(
py,
async move {
service
.async_lookup_semantic(&request, now())
.await
.map(SemanticReply::from)
},
cache_error,
)
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => {
self.python_semantic(py, SemanticOperation::LookupSemantic(request))
}
}
}
pub(in crate::cache) fn async_lookup_py<'py>(
&self,
py: Python<'py>,
request: NativeRequest,
) -> PyResult<Bound<'py, PyAny>> {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
crate::execution::run_async(
py,
async move { service.async_lookup(&request, now()).await },
cache_error,
)
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => {
self.python_semantic(py, SemanticOperation::Lookup(request))
}
}
}
pub async fn async_store(
&self,
request: &NativeRequest,
response: Value,
now: Duration,
) -> Result<(), Error> {
match self {
Self::Exact(service) => match &service.buffer {
None => {
service
.cache
.async_store(&request.exact(), response, now)
.await
}
Some(buffer) => {
buffer
.async_store(service.cache.as_ref(), &request.exact(), response, now)
.await
}
},
Self::ValkeySemantic { cache, scope, .. } => {
cache
.async_store(&request.scoped_semantic(scope), response, now)
.await
}
Self::RedisSemantic { cache, .. } => {
cache.async_store(&request.semantic(), response, now).await
}
Self::QdrantSemantic(cache) => {
cache.async_store(&request.semantic(), response, now).await
}
}
}
pub(in crate::cache) fn async_store_py<'py>(
&self,
py: Python<'py>,
request: NativeRequest,
response: Value,
) -> PyResult<Bound<'py, PyAny>> {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
crate::execution::run_async(
py,
async move { service.async_store(&request, response, now()).await },
cache_error,
)
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => {
self.python_semantic(py, SemanticOperation::Store(request, response))
}
}
}
pub async fn async_lookup_batch(
&self,
requests: &[NativeRequest],
now: Duration,
) -> Result<PartialHits, Error> {
match self {
Self::Exact(service) => {
service
.cache
.async_lookup_batch(&exact_requests(requests), now)
.await
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => {
Err(Error::UnsupportedOperation)
}
}
}
pub async fn async_store_batch(
&self,
entries: Vec<(NativeRequest, Value)>,
now: Duration,
) -> Result<(), Error> {
match self {
Self::Exact(service) => {
let entries = entries
.into_iter()
.map(|(request, value)| (request.exact(), value))
.collect();
service.cache.async_store_batch(entries, now).await
}
Self::ValkeySemantic { cache, scope, .. } => {
let entries = entries
.into_iter()
.map(|(request, value)| (request.scoped_semantic(scope), value))
.collect();
cache.async_store_batch(entries, now).await
}
Self::RedisSemantic { .. } => Err(Error::UnsupportedOperation),
Self::QdrantSemantic(cache) => {
let entries = entries
.into_iter()
.map(|(request, value)| (request.semantic(), value))
.collect();
cache.async_store_batch(entries, now).await
}
}
}
pub(in crate::cache) fn async_store_batch_py<'py>(
&self,
py: Python<'py>,
entries: Vec<(NativeRequest, Value)>,
) -> PyResult<Bound<'py, PyAny>> {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
crate::execution::run_async(
py,
async move { service.async_store_batch(entries, now()).await },
cache_error,
)
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } => {
self.python_semantic(py, SemanticOperation::StoreBatch(entries.into()))
}
}
}
pub async fn async_flush(&self) -> Result<(), Error> {
match self {
Self::Exact(service) => {
if let Some(buffer) = &service.buffer {
buffer.clear()?;
}
service.cache.async_flush().await
}
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => {
Err(Error::UnsupportedOperation)
}
}
}
pub async fn test_connection(&self) -> Result<CacheConnectionResult, Error> {
match self {
Self::Exact(service) => match &service.probe {
Some(probe) => probe.test_connection().await,
None => Err(Error::UnsupportedOperation),
},
Self::ValkeySemantic { .. } | Self::RedisSemantic { .. } | Self::QdrantSemantic(_) => {
Err(Error::UnsupportedOperation)
}
}
}
}
fn exact_requests(requests: &[NativeRequest]) -> Vec<litellm_cache_response::ResponseCacheRequest> {
requests.iter().map(NativeRequest::exact).collect()
}
/// What `lookup_semantic` hands Python: the response and the similarity to stamp, if any.
#[derive(serde::Serialize)]
pub(in crate::cache) struct SemanticReply(
pub(in crate::cache) Option<Value>,
pub(in crate::cache) Option<f64>,
);
impl From<SemanticLookup<Value>> for SemanticReply {
fn from(lookup: SemanticLookup<Value>) -> Self {
Self(lookup.value, lookup.similarity)
}
}
fn exact_lookup(value: Option<Value>) -> SemanticLookup<Value> {
SemanticLookup {
value,
similarity: None,
}
}
/// Python's Redis and Valkey semantic caches catch every lookup failure and stamp `0.0`.
fn redis_family(
lookup: Result<SemanticLookup<Value>, Error>,
) -> Result<SemanticLookup<Value>, Error> {
Ok(lookup.unwrap_or_else(|_| SemanticLookup::miss(Some(0.0))))
}

1715
litellm-rust/crates/python-bridge/src/cache/native/config.rs vendored
View file

File diff suppressed because it is too large Load diff

129
litellm-rust/crates/python-bridge/src/cache/native/embedder.rs vendored
View file

@ -1,129 +0,0 @@
use std::future::Future;
use litellm_cache::Error;
use litellm_host_python::to_py;
use pyo3::{PyTraverseError, PyVisit, prelude::*, types::PyDict};
use serde_json::Value;
tokio::task_local! {
static PREPARED_EMBEDDING: Result<Vec<f32>, Error>;
}
/// Runs `future` with the vector the Python embedder already produced, so the backend's
/// `async_embed` never has to call back into Python from the runtime.
pub(in crate::cache) fn with_prepared_embedding<F: Future>(
vector: Result<Vec<f32>, Error>,
future: F,
) -> impl Future<Output = F::Output> {
PREPARED_EMBEDDING.scope(vector, future)
}
/// The Python object that owns embedding for a semantic backend.
pub(in crate::cache) struct PythonEmbedder(Py<PyAny>);
impl Clone for PythonEmbedder {
fn clone(&self) -> Self {
Python::attach(|py| Self(self.0.clone_ref(py)))
}
}
impl PythonEmbedder {
pub(in crate::cache) fn new(object: Py<PyAny>) -> Self {
Self(object)
}
pub(in crate::cache) fn object(&self) -> &Py<PyAny> {
&self.0
}
pub(in crate::cache) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.0)
}
fn metadata_kwargs<'py>(
py: Python<'py>,
metadata: Option<&Value>,
) -> PyResult<Bound<'py, PyDict>> {
let kwargs = PyDict::new(py);
kwargs.set_item("metadata", to_py(py, &metadata)?)?;
Ok(kwargs)
}
/// The awaitable of `_get_async_embedding(prompt, metadata=...)`, to run in the caller's loop.
pub(in crate::cache) fn async_embedding(
&self,
py: Python<'_>,
prompt: &str,
metadata: Option<&Value>,
) -> PyResult<Py<PyAny>> {
let kwargs = Self::metadata_kwargs(py, metadata)?;
self.0
.bind(py)
.call_method("_get_async_embedding", (prompt,), Some(&kwargs))
.map(Bound::unbind)
}
pub(in crate::cache) fn extract(vector: Bound<'_, PyAny>) -> PyResult<Vec<f32>> {
Ok(vector
.extract::<Vec<f64>>()?
.into_iter()
.map(|value| value as f32)
.collect())
}
fn embed_sync(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
Python::attach(|py| {
let kwargs = Self::metadata_kwargs(py, metadata)?;
Self::extract(self.0.bind(py).call_method(
"_get_embedding",
(prompt,),
Some(&kwargs),
)?)
})
.map_err(|_| Error::Unavailable)
}
fn seeded_embedding() -> Result<Vec<f32>, Error> {
PREPARED_EMBEDDING
.try_with(Clone::clone)
.unwrap_or(Err(Error::Unavailable))
}
}
impl litellm_cache::semantic::Embedder for PythonEmbedder {
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
self.embed_sync(prompt, metadata)
}
fn async_embed(
&self,
_prompt: &str,
_metadata: Option<&Value>,
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
std::future::ready(Self::seeded_embedding())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn async_embed_returns_the_seeded_vector_or_unavailable() {
Python::initialize();
let object = Python::attach(|py| py.None());
let embedder = PythonEmbedder::new(object);
let scoped_embedder = embedder.clone();
let scoped = with_prepared_embedding(Ok(vec![0.25]), async move {
litellm_cache::semantic::Embedder::async_embed(&scoped_embedder, "prompt", None).await
});
assert_eq!(scoped.await, Ok(vec![0.25]));
let unscoped =
litellm_cache::semantic::Embedder::async_embed(&embedder, "prompt", None).await;
assert_eq!(unscoped, Err(Error::Unavailable));
let valkey = with_prepared_embedding(Ok(vec![0.5]), async move {
litellm_cache::semantic::Embedder::async_embed(&embedder, "prompt", None).await
});
assert_eq!(valkey.await, Ok(vec![0.5]));
}
}

502
litellm-rust/crates/python-bridge/src/cache/native/facade.rs vendored
View file

@ -1,502 +0,0 @@
use litellm_cache_redis::RedisTopology;
use litellm_host_python::from_py;
use pyo3::{
PyTraverseError, PyVisit,
exceptions::PyTypeError,
prelude::*,
types::{PyDict, PyTuple, PyType},
};
use serde_json::Value;
use super::{
backend::NativeResponseCache,
config::{CacheConfigProjection, NativeCacheConfig},
identity::BackendIdentity,
};
struct ClassGuard {
class: Py<PyType>,
attributes: Vec<(String, Py<PyAny>)>,
}
struct ObjectGuard {
reference: Py<PyAny>,
classes: Vec<ClassGuard>,
config_names: &'static [&'static str],
config: Vec<Value>,
}
struct RedisPoolGuard {
reference: Py<PyAny>,
connection_class: Py<PyAny>,
connection_kwargs: Py<PyAny>,
max_connections: Option<usize>,
client_name: &'static str,
attributes: RedisPoolAttributes,
}
struct DiskStoreGuard {
reference: Py<PyAny>,
directory: String,
}
struct AzureBlobClientGuard {
sync_client: Py<PyAny>,
async_client: Py<PyAny>,
url: String,
container_name: String,
}
struct S3ClientGuard {
reference: Py<PyAny>,
}
enum ConnectionGuard {
None,
RedisPool(RedisPoolGuard),
AzureBlob(AzureBlobClientGuard),
S3(S3ClientGuard),
}
#[derive(Clone, Copy)]
struct RedisPoolAttributes {
pool: &'static str,
connection_class: &'static str,
max_connections: Option<&'static str>,
}
const STANDALONE_POOL: RedisPoolAttributes = RedisPoolAttributes {
pool: "connection_pool",
connection_class: "connection_class",
max_connections: Some("max_connections"),
};
const CLUSTER_POOL: RedisPoolAttributes = RedisPoolAttributes {
pool: "nodes_manager",
connection_class: "connection_pool_class",
max_connections: None,
};
const VALKEY_POOL: RedisPoolAttributes = STANDALONE_POOL;
/// Class-level defaults an instance overwrites with its own state rather than behavior:
/// `Cache._native_cache` holds the runtime `Cache.__init__` resolved.
const INSTANCE_STATE: &[&str] = &["_native_cache"];
pub(in crate::cache) struct FacadeGuard {
outer: ObjectGuard,
backend: ObjectGuard,
disk_store: Option<DiskStoreGuard>,
connection: ConnectionGuard,
}
impl ObjectGuard {
fn capture(
py: Python<'_>,
object: &Bound<'_, PyAny>,
config_names: &'static [&'static str],
) -> PyResult<Self> {
let classes = object
.get_type()
.getattr("__mro__")?
.cast_into::<PyTuple>()?
.iter()
.map(|class| {
let class = class.cast_into::<PyType>()?;
let attributes = class
.getattr("__dict__")?
.call_method0("items")?
.try_iter()?
.map(|item| item?.extract::<(String, Py<PyAny>)>())
.collect::<PyResult<Vec<_>>>()?;
Ok(ClassGuard {
class: class.unbind(),
attributes,
})
})
.collect::<PyResult<Vec<_>>>()?;
let guard = Self {
reference: py
.import("weakref")?
.getattr("ref")?
.call1((object,))?
.unbind(),
classes,
config_names,
config: Self::config(object, config_names)?,
};
if !guard.matches(py, object)? {
return Err(PyTypeError::new_err(
"native facade registration requires unmodified built-in methods",
));
}
Ok(guard)
}
fn config(object: &Bound<'_, PyAny>, names: &[&str]) -> PyResult<Vec<Value>> {
names
.iter()
.map(|name| match object.getattr(*name) {
Ok(value) => from_py(&value),
Err(error)
if error.is_instance_of::<pyo3::exceptions::PyAttributeError>(object.py()) =>
{
Ok(Value::Null)
}
Err(error) => Err(error),
})
.collect()
}
fn matches(&self, py: Python<'_>, object: &Bound<'_, PyAny>) -> PyResult<bool> {
if !self.reference.bind(py).call0()?.is(object) {
return Ok(false);
}
let mro = object
.get_type()
.getattr("__mro__")?
.cast_into::<PyTuple>()?;
if mro.len() != self.classes.len() {
return Ok(false);
}
let instance = object.getattr("__dict__")?.cast_into::<PyDict>()?;
for (class, expected) in mro.iter().zip(&self.classes) {
if !class.is(expected.class.bind(py)) {
return Ok(false);
}
let attributes = class.getattr("__dict__")?;
if attributes.len()? != expected.attributes.len() {
return Ok(false);
}
for (name, value) in &expected.attributes {
if (instance.contains(name)?
&& !self.config_names.contains(&name.as_str())
&& !INSTANCE_STATE.contains(&name.as_str()))
|| !attributes.get_item(name)?.is(value.bind(py))
{
return Ok(false);
}
}
}
Ok(Self::config(object, self.config_names)? == self.config)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reference)?;
for class in &self.classes {
visit.call(&class.class)?;
for (_, value) in &class.attributes {
visit.call(value)?;
}
}
Ok(())
}
}
impl RedisPoolGuard {
fn capture(
backend: &Bound<'_, PyAny>,
client_name: &'static str,
attributes: RedisPoolAttributes,
) -> PyResult<Self> {
let pool = backend.getattr(client_name)?.getattr(attributes.pool)?;
Ok(Self {
reference: pool.clone().unbind(),
connection_class: pool.getattr(attributes.connection_class)?.unbind(),
connection_kwargs: pool
.getattr("connection_kwargs")?
.call_method0("copy")?
.unbind(),
max_connections: attributes
.max_connections
.map(|name| pool.getattr(name)?.extract::<usize>())
.transpose()?,
client_name,
attributes,
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
let pool = backend
.getattr(self.client_name)?
.getattr(self.attributes.pool)?;
Ok(self.reference.bind(py).is(&pool)
&& self
.connection_class
.bind(py)
.is(&pool.getattr(self.attributes.connection_class)?)
&& self.max_connections
== self
.attributes
.max_connections
.map(|name| pool.getattr(name)?.extract::<usize>())
.transpose()?
&& self
.connection_kwargs
.bind(py)
.eq(pool.getattr("connection_kwargs")?)?)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reference)?;
visit.call(&self.connection_class)?;
visit.call(&self.connection_kwargs)
}
}
impl DiskStoreGuard {
fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
let store = backend.getattr("disk_cache")?;
Ok(Self {
reference: store.clone().unbind(),
directory: store.getattr("directory")?.extract()?,
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
let store = backend.getattr("disk_cache")?;
Ok(self.reference.bind(py).is(&store)
&& self.directory == store.getattr("directory")?.extract::<String>()?)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reference)
}
}
impl AzureBlobClientGuard {
fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
let sync_client = backend.getattr("container_client")?;
Ok(Self {
url: sync_client.getattr("url")?.extract::<String>()?,
container_name: sync_client.getattr("container_name")?.extract::<String>()?,
sync_client: sync_client.unbind(),
async_client: backend.getattr("async_container_client")?.unbind(),
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
let sync_client = backend.getattr("container_client")?;
Ok(self.sync_client.bind(py).is(&sync_client)
&& self
.async_client
.bind(py)
.is(&backend.getattr("async_container_client")?)
&& self.url == sync_client.getattr("url")?.extract::<String>()?
&& self.container_name == sync_client.getattr("container_name")?.extract::<String>()?)
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.sync_client)?;
visit.call(&self.async_client)
}
}
impl S3ClientGuard {
fn capture(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
Ok(Self {
reference: backend.getattr("s3_client")?.unbind(),
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
Ok(self.reference.bind(py).is(&backend.getattr("s3_client")?))
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.reference)
}
}
impl ConnectionGuard {
fn capture(kind: &str, cluster: bool, backend: &Bound<'_, PyAny>) -> PyResult<Self> {
Ok(match (kind, cluster) {
("redis", false) => Self::RedisPool(RedisPoolGuard::capture(
backend,
"redis_client",
STANDALONE_POOL,
)?),
("redis", true) => Self::RedisPool(RedisPoolGuard::capture(
backend,
"redis_client",
CLUSTER_POOL,
)?),
("valkey-semantic", _) => Self::RedisPool(RedisPoolGuard::capture(
backend,
"sync_client",
VALKEY_POOL,
)?),
("disk", _) => Self::None,
("azure-blob", _) => Self::AzureBlob(AzureBlobClientGuard::capture(backend)?),
("s3", _) => Self::S3(S3ClientGuard::capture(backend)?),
_ => Self::None,
})
}
fn matches(&self, py: Python<'_>, backend: &Bound<'_, PyAny>) -> PyResult<bool> {
match self {
Self::None => Ok(true),
Self::RedisPool(guard) => guard.matches(py, backend),
Self::AzureBlob(guard) => guard.matches(py, backend),
Self::S3(guard) => guard.matches(py, backend),
}
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
match self {
Self::None => Ok(()),
Self::RedisPool(guard) => guard.traverse(visit),
Self::AzureBlob(guard) => guard.traverse(visit),
Self::S3(guard) => guard.traverse(visit),
}
}
}
impl FacadeGuard {
pub(in crate::cache) fn capture(
py: Python<'_>,
facade: &Bound<'_, PyAny>,
service: &NativeResponseCache,
) -> PyResult<Self> {
let identity = service.identity();
let kind = identity.kind();
let cache_type = py.import("litellm.caching.caching")?.getattr("Cache")?;
if !facade.get_type().is(&cache_type) {
return Err(PyTypeError::new_err(
"only exact built-in Cache facades can be registered",
));
}
let cluster = matches!(
identity,
BackendIdentity::Redis {
topology: RedisTopology::Cluster { .. },
..
}
);
let (module, name) = match (kind, cluster) {
("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache"),
("redis", false) => ("litellm.caching.redis_cache", "RedisCache"),
("redis", true) => ("litellm.caching.redis_cluster_cache", "RedisClusterCache"),
("redis_semantic", _) => ("litellm.caching.redis_semantic_cache", "RedisSemanticCache"),
("qdrant_semantic", _) => (
"litellm.caching.qdrant_semantic_cache",
"QdrantSemanticCache",
),
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache"),
("valkey-semantic", _) => (
"litellm.caching.valkey_semantic_cache",
"ValkeySemanticCache",
),
("disk", _) => ("litellm.caching.disk_cache", "DiskCache"),
("azure-blob", _) => ("litellm.caching.azure_blob_cache", "AzureBlobCache"),
("s3", _) => ("litellm.caching.s3_cache", "S3Cache"),
_ => unreachable!(),
};
let cache_kind = identity.cache_type();
let backend = facade.getattr("cache")?;
if facade.getattr("type")?.extract::<String>()? != cache_kind
|| !backend.get_type().is(&py.import(module)?.getattr(name)?)
{
return Err(PyTypeError::new_err(
"facade and native backend types must match",
));
}
let config = match NativeCacheConfig::project(facade)? {
CacheConfigProjection::Native(config) => *config,
CacheConfigProjection::Unsupported(reason) => {
return Err(PyTypeError::new_err(reason.message()));
}
};
if let Some(message) = config.service_mismatch(service) {
return Err(PyTypeError::new_err(message));
}
if kind == "redis_semantic"
&& service
.embedder_object()
.is_none_or(|embedder| !backend.is(embedder.bind(py)))
{
return Err(PyTypeError::new_err(
"facade backend must be the native embedder",
));
}
Ok(Self {
outer: ObjectGuard::capture(
py,
facade,
&[
"type",
"mode",
"ttl",
"namespace",
"supported_call_types",
"redis_flush_size",
"semantic_cache_scope",
],
)?,
backend: ObjectGuard::capture(
py,
&backend,
&[
"namespace",
"default_ttl",
"max_size_in_memory",
"max_size_per_item",
"redis_kwargs",
"redis_flush_size",
"similarity_threshold",
"distance_threshold",
"embedding_model",
"embedding_max_input_tokens",
"embedding_timeout",
"qdrant_api_base",
"qdrant_api_key",
"collection_name",
"vector_size",
"_index_name",
"_redis_url",
"similarity_threshold",
"embedding_model",
"index_name",
"embedding_max_input_tokens",
"embedding_timeout",
"bucket_name",
"key_prefix",
"path_service_account",
],
)?,
disk_store: (kind == "disk")
.then(|| DiskStoreGuard::capture(&backend))
.transpose()?,
connection: ConnectionGuard::capture(kind, cluster, &backend)?,
})
}
pub(in crate::cache) fn matches(
&self,
py: Python<'_>,
facade: &Bound<'_, PyAny>,
) -> PyResult<bool> {
if !self.outer.matches(py, facade)? {
return Ok(false);
}
let backend = facade.getattr("cache")?;
if !self.backend.matches(py, &backend)? {
return Ok(false);
}
if let Some(guard) = &self.disk_store
&& !guard.matches(py, &backend)?
{
return Ok(false);
}
self.connection.matches(py, &backend)
}
pub(in crate::cache) fn traverse(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
self.outer.traverse(&visit)?;
self.backend.traverse(&visit)?;
if let Some(guard) = &self.disk_store {
guard.traverse(&visit)?;
}
self.connection.traverse(&visit)
}
}

510
litellm-rust/crates/python-bridge/src/cache/native/identity.rs vendored
View file

@ -1,510 +0,0 @@
use std::{path::PathBuf, time::Duration};
use litellm_cache_redis::RedisTopology;
/// What makes a native backend the one a Python facade describes: the configuration a user can
/// observe on the Python object, captured once so facade projection and native construction
/// compare plain data instead of reaching into each backend type.
#[derive(Clone, Debug, PartialEq)]
pub(in crate::cache) enum BackendIdentity {
Memory {
capacity: usize,
max_entry_bytes: Option<usize>,
default_ttl: Option<Duration>,
},
Redis {
topology: RedisTopology,
namespace: Option<String>,
default_ttl: Option<Duration>,
},
S3 {
bucket: String,
key_prefix: String,
region: String,
endpoint: Option<String>,
},
Gcs {
bucket_name: String,
key_prefix: String,
path_service_account: Option<String>,
},
Disk {
directory: PathBuf,
},
AzureBlob {
account_url: String,
container: String,
},
RedisSemantic {
index_name: String,
/// The backend stores the threshold as `f32`; a facade's `f64` is compared at that width.
similarity_threshold: f32,
},
ValkeySemantic {
index_name: String,
similarity_threshold: f64,
},
QdrantSemantic {
collection_name: String,
similarity_threshold: f64,
vector_size: u64,
embedding_model: String,
},
}
const TYPES: &str = "facade and native backend types must match";
impl BackendIdentity {
pub(in crate::cache) fn kind(&self) -> &'static str {
match self {
Self::Memory { .. } => "memory",
Self::Redis { .. } => "redis",
Self::S3 { .. } => "s3",
Self::Gcs { .. } => "gcs",
Self::ValkeySemantic { .. } => "valkey-semantic",
Self::RedisSemantic { .. } => "redis_semantic",
Self::QdrantSemantic { .. } => "qdrant_semantic",
Self::Disk { .. } => "disk",
Self::AzureBlob { .. } => "azure-blob",
}
}
/// The `LiteLLMCacheType` value a facade of this backend carries in `Cache.type`.
pub(in crate::cache) fn cache_type(&self) -> &'static str {
match self {
Self::Memory { .. } => "local",
Self::Redis { .. } => "redis",
Self::S3 { .. } => "s3",
Self::Gcs { .. } => "gcs",
Self::ValkeySemantic { .. } => "valkey-semantic",
Self::RedisSemantic { .. } => "redis-semantic",
Self::QdrantSemantic { .. } => "qdrant-semantic",
Self::Disk { .. } => "disk",
Self::AzureBlob { .. } => "azure-blob",
}
}
/// The first difference between the facade's configuration (`self`) and the native
/// backend (`native`), in the order Python users see the attributes.
pub(in crate::cache) fn mismatch(&self, native: &Self) -> Option<&'static str> {
let mut differences: Vec<(bool, &'static str)> = Vec::new();
let mut differs = |condition: bool, message: &'static str| {
differences.push((condition, message));
};
match (self, native) {
(
Self::Memory {
capacity,
max_entry_bytes,
default_ttl,
},
Self::Memory {
capacity: native_capacity,
max_entry_bytes: native_max_entry_bytes,
default_ttl: native_default_ttl,
},
) => {
differs(
default_ttl != native_default_ttl,
"facade and native backend default TTLs must match",
);
differs(
capacity != native_capacity,
"facade and native backend capacities must match",
);
differs(
max_entry_bytes != native_max_entry_bytes,
"facade and native backend item limits must match",
);
}
(
Self::Redis {
topology,
namespace,
default_ttl,
},
Self::Redis {
topology: native_topology,
namespace: native_namespace,
default_ttl: native_default_ttl,
},
) => {
differs(
default_ttl != native_default_ttl,
"facade and native backend default TTLs must match",
);
differs(
topology != native_topology,
"facade and native backend topologies must match",
);
differs(
namespace != native_namespace,
"facade and native backend namespaces must match",
);
}
(
Self::S3 {
bucket,
key_prefix,
region,
endpoint,
},
Self::S3 {
bucket: native_bucket,
key_prefix: native_key_prefix,
region: native_region,
endpoint: native_endpoint,
},
) => {
differs(
bucket != native_bucket,
"facade and native backend buckets must match",
);
differs(
key_prefix != native_key_prefix,
"facade and native backend key prefixes must match",
);
differs(
region != native_region,
"facade and native backend regions must match",
);
differs(
endpoint != native_endpoint,
"facade and native backend endpoints must match",
);
}
(
Self::Gcs {
bucket_name,
key_prefix,
path_service_account,
},
Self::Gcs {
bucket_name: native_bucket_name,
key_prefix: native_key_prefix,
path_service_account: native_path_service_account,
},
) => {
differs(
bucket_name != native_bucket_name,
"facade and native backend buckets must match",
);
differs(
key_prefix != native_key_prefix,
"facade and native backend key prefixes must match",
);
differs(
path_service_account != native_path_service_account,
"facade and native backend credentials must match",
);
}
(
Self::Disk { directory },
Self::Disk {
directory: native_directory,
},
) => {
let canonical = |path: &PathBuf| std::fs::canonicalize(path).ok();
differs(
canonical(directory) != canonical(native_directory),
"facade and native backend directories must match",
);
}
(
Self::AzureBlob {
account_url,
container,
},
Self::AzureBlob {
account_url: native_account_url,
container: native_container,
},
) => {
differs(
account_url != native_account_url || container != native_container,
"facade and native backend containers must match",
);
}
(
Self::RedisSemantic {
index_name,
similarity_threshold,
},
Self::RedisSemantic {
index_name: native_index_name,
similarity_threshold: native_similarity_threshold,
},
) => {
differs(
index_name != native_index_name,
"facade and native backend index names must match",
);
differs(
similarity_threshold != native_similarity_threshold,
"facade and native backend similarity thresholds must match",
);
}
(
Self::ValkeySemantic {
index_name,
similarity_threshold,
},
Self::ValkeySemantic {
index_name: native_index_name,
similarity_threshold: native_similarity_threshold,
},
) => {
differs(
index_name != native_index_name
|| similarity_threshold != native_similarity_threshold,
"facade and native semantic settings must match",
);
}
(
Self::QdrantSemantic {
collection_name,
similarity_threshold,
vector_size,
embedding_model,
},
Self::QdrantSemantic {
collection_name: native_collection_name,
similarity_threshold: native_similarity_threshold,
vector_size: native_vector_size,
embedding_model: native_embedding_model,
},
) => {
differs(
collection_name != native_collection_name,
"facade and native backend collections must match",
);
differs(
similarity_threshold != native_similarity_threshold,
"facade and native backend similarity thresholds must match",
);
differs(
vector_size != native_vector_size,
"facade and native backend vector sizes must match",
);
differs(
embedding_model != native_embedding_model,
"facade and native backend embedding models must match",
);
}
_ => return Some(TYPES),
}
differences
.into_iter()
.find_map(|(condition, message)| condition.then_some(message))
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use litellm_cache_redis::{RedisNode, RedisTopology};
use super::BackendIdentity;
fn memory() -> BackendIdentity {
BackendIdentity::Memory {
capacity: 200,
max_entry_bytes: Some(1024),
default_ttl: Some(Duration::from_secs(60)),
}
}
fn redis() -> BackendIdentity {
BackendIdentity::Redis {
topology: RedisTopology::Standalone,
namespace: Some("team".into()),
default_ttl: Some(Duration::from_secs(60)),
}
}
fn s3() -> BackendIdentity {
BackendIdentity::S3 {
bucket: "bucket".into(),
key_prefix: "cache/".into(),
region: "us-east-1".into(),
endpoint: None,
}
}
fn gcs() -> BackendIdentity {
BackendIdentity::Gcs {
bucket_name: "bucket".into(),
key_prefix: "cache/".into(),
path_service_account: Some("credentials.json".into()),
}
}
fn azure() -> BackendIdentity {
BackendIdentity::AzureBlob {
account_url: "https://account.blob.core.windows.net".into(),
container: "cache".into(),
}
}
fn redis_semantic() -> BackendIdentity {
BackendIdentity::RedisSemantic {
index_name: "idx".into(),
similarity_threshold: 0.8,
}
}
#[test]
fn redis_semantic_thresholds_compare_at_backend_precision() {
let facade = BackendIdentity::RedisSemantic {
index_name: "idx".into(),
similarity_threshold: 0.8_f64 as f32,
};
assert_eq!(facade.mismatch(&redis_semantic()), None);
}
fn valkey_semantic() -> BackendIdentity {
BackendIdentity::ValkeySemantic {
index_name: "idx".into(),
similarity_threshold: 0.8,
}
}
fn qdrant() -> BackendIdentity {
BackendIdentity::QdrantSemantic {
collection_name: "collection".into(),
similarity_threshold: 0.8,
vector_size: 1536,
embedding_model: "text-embedding-3-small".into(),
}
}
#[test]
fn identical_identities_have_no_mismatch() {
for identity in [
memory(),
redis(),
s3(),
gcs(),
azure(),
redis_semantic(),
valkey_semantic(),
qdrant(),
BackendIdentity::Disk {
directory: std::env::temp_dir(),
},
] {
assert_eq!(identity.mismatch(&identity), None, "{identity:?}");
}
}
#[test]
fn different_kinds_report_a_type_mismatch() {
assert_eq!(
memory().mismatch(&redis()),
Some("facade and native backend types must match")
);
assert_eq!(
redis_semantic().mismatch(&valkey_semantic()),
Some("facade and native backend types must match")
);
}
#[test]
fn the_first_differing_field_names_the_mismatch() {
let BackendIdentity::Memory { capacity, .. } = memory() else {
unreachable!()
};
assert_eq!(
memory().mismatch(&BackendIdentity::Memory {
capacity: capacity + 1,
max_entry_bytes: Some(1),
default_ttl: Some(Duration::from_secs(60)),
}),
Some("facade and native backend capacities must match")
);
assert_eq!(
memory().mismatch(&BackendIdentity::Memory {
capacity,
max_entry_bytes: Some(1),
default_ttl: Some(Duration::from_secs(61)),
}),
Some("facade and native backend default TTLs must match")
);
assert_eq!(
redis().mismatch(&BackendIdentity::Redis {
topology: RedisTopology::Cluster {
startup_nodes: vec![RedisNode {
host: "node".into(),
port: 7000,
}],
},
namespace: None,
default_ttl: Some(Duration::from_secs(60)),
}),
Some("facade and native backend topologies must match")
);
assert_eq!(
s3().mismatch(&BackendIdentity::S3 {
bucket: "bucket".into(),
key_prefix: "cache/".into(),
region: "us-east-1".into(),
endpoint: Some("http://localhost:9000".into()),
}),
Some("facade and native backend endpoints must match")
);
assert_eq!(
gcs().mismatch(&BackendIdentity::Gcs {
bucket_name: "bucket".into(),
key_prefix: "cache/".into(),
path_service_account: None,
}),
Some("facade and native backend credentials must match")
);
assert_eq!(
azure().mismatch(&BackendIdentity::AzureBlob {
account_url: "https://account.blob.core.windows.net".into(),
container: "other".into(),
}),
Some("facade and native backend containers must match")
);
assert_eq!(
valkey_semantic().mismatch(&BackendIdentity::ValkeySemantic {
index_name: "idx".into(),
similarity_threshold: 0.9,
}),
Some("facade and native semantic settings must match")
);
assert_eq!(
qdrant().mismatch(&BackendIdentity::QdrantSemantic {
collection_name: "collection".into(),
similarity_threshold: 0.8,
vector_size: 1536,
embedding_model: "text-embedding-3-large".into(),
}),
Some("facade and native backend embedding models must match")
);
}
#[test]
fn disk_directories_compare_canonically() {
let directory = std::env::temp_dir();
let mut indirect = directory.clone();
indirect.push(".");
assert_eq!(
BackendIdentity::Disk {
directory: directory.clone()
}
.mismatch(&BackendIdentity::Disk {
directory: indirect
}),
None
);
assert_eq!(
BackendIdentity::Disk { directory }.mismatch(&BackendIdentity::Disk {
directory: "/definitely/missing".into()
}),
Some("facade and native backend directories must match")
);
}
}

8
litellm-rust/crates/python-bridge/src/cache/native/mod.rs vendored
View file

@ -1,11 +1,3 @@
pub(super) mod activation;
pub(super) mod backend;
pub(super) mod config;
mod embedder;
pub(super) mod facade;
mod identity;
pub(super) mod request;
mod semantic;
pub(super) mod v2;
pub(crate) use v2::NativeCacheHandle;

258
litellm-rust/crates/python-bridge/src/cache/native/request.rs vendored
View file

@ -1,258 +0,0 @@
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use litellm_cache::{ExactCacheContext, SemanticCacheContext};
use litellm_cache_response::{CacheControls, CacheKeyField, CacheKeyInput, ResponseCacheRequest};
use litellm_host_python::from_py;
use pyo3::{exceptions::PyValueError, prelude::*};
use serde::Deserialize;
use serde_json::Value;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct RequestInput {
key: CacheKeyInput,
controls: Option<CacheControls>,
ttl_seconds: Option<f64>,
max_age_seconds: Option<f64>,
messages: Option<Value>,
input: Option<Value>,
metadata: Option<Value>,
litellm_metadata: Option<Value>,
litellm_params: Option<Value>,
scope: Option<String>,
}
#[derive(Clone)]
pub(in crate::cache) struct NativeRequest {
pub(super) key: CacheKeyInput,
pub(super) controls: CacheControls,
pub(super) ttl: Option<Duration>,
pub(super) max_age: Option<Duration>,
pub(super) messages: Option<Value>,
pub(super) input: Option<Value>,
pub(super) metadata: Option<Value>,
pub(super) litellm_metadata: Option<Value>,
pub(super) litellm_params: Option<Value>,
pub(super) scope: Option<String>,
}
impl NativeRequest {
pub(super) fn exact(&self) -> ResponseCacheRequest<ExactCacheContext> {
ResponseCacheRequest {
key: self.key.clone(),
controls: self.controls,
context: ExactCacheContext { ttl: self.ttl },
max_age: self.max_age,
}
}
/// The request as a semantic backend that keys on the caller's scope sees it.
pub(super) fn semantic(&self) -> ResponseCacheRequest<SemanticCacheContext> {
self.semantic_with(self.key.clone(), self.scope.clone())
}
/// The request keyed the way Python's Valkey semantic cache keys it: prompt fields drop out
/// and the tenant identifiers for `scope` join the key.
pub(super) fn scoped_semantic(
&self,
scope: &str,
) -> ResponseCacheRequest<SemanticCacheContext> {
self.semantic_with(semantic_key(self, scope), Some(scope.to_owned()))
}
fn semantic_with(
&self,
key: CacheKeyInput,
scope: Option<String>,
) -> ResponseCacheRequest<SemanticCacheContext> {
ResponseCacheRequest {
key,
controls: self.controls,
context: SemanticCacheContext {
input: self.input.clone(),
messages: self.messages.clone(),
metadata: self.metadata.clone(),
scope,
ttl: self.ttl,
},
max_age: self.max_age,
}
}
}
fn semantic_key(request: &NativeRequest, scope: &str) -> CacheKeyInput {
let mut key = request.key.clone();
if key.preset.is_some() {
return key;
}
key.fields
.retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input"));
const TENANT: [&str; 3] = [
"user_api_key",
"user_api_key_team_id",
"user_api_key_org_id",
];
let end_user = (scope == "end_user").then_some("user_api_key_end_user_id");
for name in TENANT.into_iter().chain(end_user) {
let sources = [
request.metadata.as_ref(),
request.litellm_metadata.as_ref(),
request
.litellm_params
.as_ref()
.and_then(|params| params.get("metadata")),
request
.litellm_params
.as_ref()
.and_then(|params| params.get("litellm_metadata")),
];
let Some(value) = sources.into_iter().flatten().find_map(|source| {
source
.as_object()
.and_then(|values| values.get(name))
.filter(|value| !value.is_null())
}) else {
continue;
};
let value = match value {
Value::Null => continue,
Value::String(text) => text.clone(),
other => other.to_string(),
};
key.fields.push(CacheKeyField {
name: name.to_owned(),
value: Some(value),
api_parameter: true,
internal_parameter: false,
});
}
key
}
pub(in crate::cache) fn request(value: &Bound<'_, PyAny>) -> PyResult<NativeRequest> {
let input: RequestInput = from_py(value)?;
request_input(input)
}
fn request_input(input: RequestInput) -> PyResult<NativeRequest> {
let controls = input.controls.unwrap_or_else(|| {
ResponseCacheRequest::<ExactCacheContext>::new(input.key.clone()).controls
});
Ok(NativeRequest {
key: input.key,
controls,
ttl: input.ttl_seconds.map(duration).transpose()?,
max_age: input.max_age_seconds.map(duration).transpose()?,
messages: input.messages,
input: input.input,
metadata: input.metadata,
litellm_metadata: input.litellm_metadata,
litellm_params: input.litellm_params,
scope: input.scope,
})
}
pub(in crate::cache) fn requests(value: &Bound<'_, PyAny>) -> PyResult<Vec<NativeRequest>> {
from_py::<Vec<RequestInput>>(value)?
.into_iter()
.map(request_input)
.collect()
}
pub(super) fn duration(seconds: f64) -> PyResult<Duration> {
Duration::try_from_secs_f64(seconds)
.map_err(|_| PyValueError::new_err("cache durations must be finite and nonnegative"))
}
pub(in crate::cache) fn now() -> Duration {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key};
use serde_json::json;
use sha2::{Digest, Sha256};
use super::*;
fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest {
NativeRequest {
key,
controls: CacheControls::default(),
ttl: None,
max_age: None,
messages: Some(json!([{"role": "user", "content": "prompt"}])),
input: None,
metadata: Some(metadata),
litellm_metadata: None,
litellm_params: None,
scope: None,
}
}
#[test]
fn semantic_key_matches_python_scope_material() {
let key = CacheKeyInput {
fields: vec![
CacheKeyField {
name: "model".to_owned(),
value: Some("gpt-4.1".to_owned()),
api_parameter: true,
internal_parameter: false,
},
CacheKeyField {
name: "messages".to_owned(),
value: Some("prompt".to_owned()),
api_parameter: true,
internal_parameter: false,
},
],
..Default::default()
};
let request = native_request(
key,
json!({"user_api_key": "k1", "user_api_key_team_id": null}),
);
let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1"));
assert_eq!(cache_key(&semantic_key(&request, "key")), expected);
assert_eq!(cache_key(&request.scoped_semantic("key").key), expected);
let end_user_request = native_request(
request.key.clone(),
json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}),
);
let expected = format!(
"{:x}",
Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1")
);
assert_eq!(
cache_key(&semantic_key(&end_user_request, "end_user")),
expected
);
let preset_request = native_request(
CacheKeyInput {
preset: Some("preset-key".to_owned()),
..Default::default()
},
json!({"user_api_key": "k1"}),
);
assert_eq!(
semantic_key(&preset_request, "end_user").preset.as_deref(),
Some("preset-key")
);
assert!(semantic_key(&preset_request, "end_user").fields.is_empty());
assert_eq!(preset_request.semantic().context.scope, None);
assert_eq!(
preset_request
.scoped_semantic("end_user")
.context
.scope
.as_deref(),
Some("end_user")
);
}
}

211
litellm-rust/crates/python-bridge/src/cache/native/semantic.rs vendored
View file

@ -1,211 +0,0 @@
use crate::cache::cache_error;
use crate::execution::run_async;
use std::{collections::VecDeque, time::Duration};
use litellm_cache::Error;
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyException, PyRuntimeError},
prelude::*,
};
use serde_json::Value;
use super::{
backend::{NativeResponseCache, SemanticReply},
embedder::{PythonEmbedder, with_prepared_embedding},
request::{NativeRequest, now},
};
pub(super) enum SemanticOperation {
Lookup(NativeRequest),
/// A lookup that also reports the similarity, as `SemanticReply`.
LookupSemantic(NativeRequest),
Store(NativeRequest, Value),
StoreBatch(VecDeque<(NativeRequest, Value)>),
}
/// What an exception from the Python embedder means for the operation.
#[derive(Clone, Copy)]
pub(super) enum EmbeddingFailure {
/// Raise the Python exception unchanged.
Propagate,
/// Treat the embedding as unavailable and let the backend report that.
Unavailable,
}
enum Phase {
Start,
AwaitingEmbedding,
AwaitingBackend,
}
/// Runs a semantic cache operation whose embedding comes from Python: await the Python
/// embedder in the caller's event loop, seed the native backend with the vector, await the
/// backend, and repeat for each entry of a batch.
pub(super) struct SemanticExecution {
service: NativeResponseCache,
embedder: PythonEmbedder,
failure: EmbeddingFailure,
operation: SemanticOperation,
pending: Option<(NativeRequest, Option<Value>)>,
phase: Phase,
now: Duration,
}
impl SemanticExecution {
pub(super) fn new(
service: NativeResponseCache,
embedder: PythonEmbedder,
failure: EmbeddingFailure,
operation: SemanticOperation,
) -> Self {
Self {
service,
embedder,
failure,
operation,
pending: None,
phase: Phase::Start,
now: now(),
}
}
/// Takes the next entry of the operation; `None` once a batch is exhausted.
fn next_pending(&mut self) -> Option<(NativeRequest, Option<Value>)> {
match &mut self.operation {
SemanticOperation::Lookup(request) | SemanticOperation::LookupSemantic(request) => {
Some((request.clone(), None))
}
SemanticOperation::Store(request, response) => {
Some((request.clone(), Some(std::mem::take(response))))
}
SemanticOperation::StoreBatch(queue) => queue
.pop_front()
.map(|(request, response)| (request, Some(response))),
}
}
fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
let Some(pending) = self.next_pending() else {
return Ok(ExecutionStep::Return(py.None()));
};
let (request, response) = &pending;
let enabled = match response {
None => request.controls.reads(),
Some(_) => request.controls.writes(),
};
let input = enabled
.then(|| self.service.embedding_input(request))
.flatten();
self.pending = Some(pending);
let Some(input) = input else {
return self.backend_step(py, Err(Error::Unavailable));
};
let awaitable =
self.embedder
.async_embedding(py, &input.prompt, input.metadata.as_ref())?;
self.phase = Phase::AwaitingEmbedding;
Ok(ExecutionStep::Await(awaitable))
}
fn embedded(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<ExecutionStep> {
let seed = match result {
Ok(vector) => {
PythonEmbedder::extract(vector.into_bound(py)).map_err(|_| Error::Unavailable)
}
Err(error) => match self.embedding_failure() {
EmbeddingFailure::Propagate => return Err(error),
EmbeddingFailure::Unavailable if error.is_instance_of::<PyException>(py) => {
Err(Error::Unavailable)
}
EmbeddingFailure::Unavailable => return Err(error),
},
};
self.backend_step(py, seed)
}
/// Python's semantic lookups catch embedding errors and stamp a similarity of `0.0`.
fn embedding_failure(&self) -> EmbeddingFailure {
match self.operation {
SemanticOperation::LookupSemantic(_) => EmbeddingFailure::Unavailable,
_ => self.failure,
}
}
fn backend_step(
&mut self,
py: Python<'_>,
seed: Result<Vec<f32>, Error>,
) -> PyResult<ExecutionStep> {
self.phase = Phase::AwaitingBackend;
let (request, response) = self.pending.take().ok_or_else(|| {
PyRuntimeError::new_err("semantic execution resumed without a pending operation")
})?;
let service = self.service.clone();
let now = self.now;
let with_similarity = matches!(self.operation, SemanticOperation::LookupSemantic(_));
let future = async move {
match response {
None if with_similarity => service
.async_lookup_semantic(&request, now)
.await
.map(|lookup| Reply::Semantic(lookup.into())),
None => service.async_lookup(&request, now).await.map(Reply::Plain),
Some(response) => service
.async_store(&request, response, now)
.await
.map(|_| Reply::Plain(None)),
}
};
let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?;
Ok(ExecutionStep::Await(awaitable.unbind()))
}
fn resume_py(
&mut self,
py: Python<'_>,
result: Option<PyResult<Py<PyAny>>>,
) -> PyResult<ExecutionStep> {
match (&self.phase, result) {
(Phase::Start, None) => self.start(py),
(Phase::AwaitingEmbedding, Some(result)) => self.embedded(py, result),
(Phase::AwaitingBackend, Some(Err(error))) => Err(error),
(Phase::AwaitingBackend, Some(Ok(value))) => {
let more = matches!(
&self.operation,
SemanticOperation::StoreBatch(queue) if !queue.is_empty()
);
if more {
self.phase = Phase::Start;
return self.start(py);
}
Ok(ExecutionStep::Return(value))
}
_ => Err(PyRuntimeError::new_err(
"invalid semantic cache execution state",
)),
}
}
}
#[derive(serde::Serialize)]
#[serde(untagged)]
enum Reply {
Plain(Option<Value>),
Semantic(SemanticReply),
}
impl ExecutionBody for SemanticExecution {
fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
Python::attach(|py| self.resume_py(py, result))
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.embedder.traverse(visit)
}
}
pub(super) fn drive(py: Python<'_>, body: SemanticExecution) -> PyResult<Bound<'_, PyAny>> {
Execution::new(body, crate::lifecycle::binding).into_coroutine(py)
}

34
litellm-rust/crates/python-bridge/src/cache/native/v2.rs vendored
View file

@ -1,5 +1,8 @@
use crate::cache::cache_error;
use std::{sync::Arc, time::Duration};
use std::{
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use litellm_cache::{DeleteCache, DisconnectCache, PingCache};
use litellm_host_python::{from_py, release_gil, to_py};
@ -118,8 +121,8 @@ impl NativeCacheHandle {
fn get(&self, py: Python<'_>, key: String) -> PyResult<Py<PyAny>> {
self.check_process()?;
let request = request(key, None)?;
let value = release_gil(py, || self.backend.lookup(&request, super::request::now()))
.map_err(cache_error)?;
let value =
release_gil(py, || self.backend.lookup(&request, now())).map_err(cache_error)?;
to_py(py, &value)
}
@ -134,10 +137,7 @@ impl NativeCacheHandle {
self.check_process()?;
let request = request(key, ttl)?;
let value: Value = from_py(value)?;
release_gil(py, || {
self.backend.store(&request, value, super::request::now())
})
.map_err(cache_error)
release_gil(py, || self.backend.store(&request, value, now())).map_err(cache_error)
}
fn async_get<'py>(&self, py: Python<'py>, key: String) -> PyResult<Bound<'py, PyAny>> {
@ -146,7 +146,7 @@ impl NativeCacheHandle {
let backend = self.backend.clone();
crate::execution::run_async(
py,
async move { backend.async_lookup(&request, super::request::now()).await },
async move { backend.async_lookup(&request, now()).await },
cache_error,
)
}
@ -165,11 +165,7 @@ impl NativeCacheHandle {
let backend = self.backend.clone();
crate::execution::run_async(
py,
async move {
backend
.async_store(&request, value, super::request::now())
.await
},
async move { backend.async_store(&request, value, now()).await },
cache_error,
)
}
@ -190,11 +186,7 @@ impl NativeCacheHandle {
let backend = self.backend.clone();
crate::execution::run_async(
py,
async move {
backend
.async_store_batch(entries, super::request::now())
.await
},
async move { backend.async_store_batch(entries, now()).await },
cache_error,
)
}
@ -345,3 +337,9 @@ pub(in crate::cache) fn configured(
},
))
}
fn now() -> Duration {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
}

2
litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md vendored
View file

@ -1,6 +1,6 @@
# Python cache delegation
This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust. `callback.rs` provides Python cache delegation for the Python-facing cache runtime
This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust.
Keep the shared inference protocol wrapper and adapter selection in the parent module. Receive the configured cache and prepared arguments from the parent module. Do not discover global configuration, choose native backends, or move inference to Python. Core remains independent of Python objects and cache implementation details

Some files were not shown because too many files have changed in this diff Show more