mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-27 06:13:07 +00:00
ee03f153bd
Gateway replicas now share state through the coordination Redis the Python proxy uses (general_settings.coordination_redis, a redis response cache, or REDIS_*): - A deployment one replica cools down is written under Python's deployment:<id>:cooldown key and applied by every other replica's background poll (one MGET a second), so the fleet stops sending to a failing deployment together. allowed_fails counts failures across replicas, as Python's v1 rule does; the v2 failure-rate rule stays per replica, as in Python. Deployments are named by model_info.id, or an id derived from the config that is the same on every replica. - The virtual key cache follows the proxy's litellm_proxy.auth_cache_invalidation channel, so a key blocked, deleted or edited through the admin API (or its team or user) is re-read on every replica at once instead of after up to 60 s. general_settings.user_api_key_cache_ttl sets the key TTL. Nothing is added to the path of a successful request. Without Redis, or while it is down, each replica keeps its own state as before. Redis Cluster and Sentinel are refused at startup for now.
217 lines
7.3 KiB
YAML
217 lines
7.3 KiB
YAML
name: LiteLLM Rust
|
|
|
|
on:
|
|
push:
|
|
paths:
|
|
- "litellm-rust/**"
|
|
- "litellm/rust_bridge/**"
|
|
- "scripts/generate_trace_types.py"
|
|
- "scripts/trace_codegen/**"
|
|
- "scripts/lens_assets/**"
|
|
- "litellm/tracing/**"
|
|
- "tests/test_litellm_rust/**"
|
|
- "litellm/integrations/custom_logger.py"
|
|
- "litellm/litellm_core_utils/litellm_logging.py"
|
|
- "litellm/litellm_core_utils/logging_worker.py"
|
|
- "litellm/proxy/guardrails/**"
|
|
- "litellm/utils.py"
|
|
- "litellm/ocr/**"
|
|
- "litellm/chat_completions/dispatch.py"
|
|
- "litellm/embeddings/dispatch.py"
|
|
- "litellm/messages/dispatch.py"
|
|
- "litellm/responses/dispatch.py"
|
|
- "litellm/llms/base_llm/ocr/**"
|
|
- "litellm/llms/custom_httpx/llm_http_handler.py"
|
|
- "Makefile"
|
|
- ".cargo/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- "rust-toolchain.toml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
- ".github/scripts/smoke_test_native_wheel.py"
|
|
- ".github/scripts/verify_linux_native_wheel.py"
|
|
- "tests/unit/rust_bridge/native_route_wheel_test.py"
|
|
- ".github/workflows/test-rust.yml"
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- "litellm_**"
|
|
paths:
|
|
- "litellm-rust/**"
|
|
- "litellm/rust_bridge/**"
|
|
- "scripts/generate_trace_types.py"
|
|
- "scripts/trace_codegen/**"
|
|
- "scripts/lens_assets/**"
|
|
- "litellm/tracing/**"
|
|
- "tests/test_litellm_rust/**"
|
|
- "litellm/integrations/custom_logger.py"
|
|
- "litellm/litellm_core_utils/litellm_logging.py"
|
|
- "litellm/litellm_core_utils/logging_worker.py"
|
|
- "litellm/proxy/guardrails/**"
|
|
- "litellm/utils.py"
|
|
- "litellm/ocr/**"
|
|
- "litellm/chat_completions/dispatch.py"
|
|
- "litellm/embeddings/dispatch.py"
|
|
- "litellm/messages/dispatch.py"
|
|
- "litellm/responses/dispatch.py"
|
|
- "litellm/llms/base_llm/ocr/**"
|
|
- "litellm/llms/custom_httpx/llm_http_handler.py"
|
|
- "Makefile"
|
|
- ".cargo/**"
|
|
- "pyproject.toml"
|
|
- "uv.lock"
|
|
- "rust-toolchain.toml"
|
|
- ".github/actions/setup-uv-with-retries/**"
|
|
- ".github/scripts/smoke_test_native_wheel.py"
|
|
- ".github/scripts/verify_linux_native_wheel.py"
|
|
- "tests/unit/rust_bridge/native_route_wheel_test.py"
|
|
- ".github/workflows/test-rust.yml"
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
|
|
jobs:
|
|
rust-lint:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
defaults:
|
|
run:
|
|
working-directory: litellm-rust
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- run: rustup toolchain install --no-self-update
|
|
|
|
- run: cargo fmt --all --check
|
|
|
|
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
with:
|
|
workspaces: litellm-rust
|
|
cache-on-failure: true
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
|
|
|
rust-test:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
# The gateway's shared-state tests run against this Redis.
|
|
services:
|
|
redis:
|
|
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
|
|
ports:
|
|
- 6379:6379
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
# Trim debuginfo to keep the job's target dir within the runner's disk
|
|
env:
|
|
CARGO_PROFILE_DEV_DEBUG: line-tables-only
|
|
LITELLM_TEST_REDIS_URL: redis://127.0.0.1:6379
|
|
defaults:
|
|
run:
|
|
working-directory: litellm-rust
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install Python dependencies for the bridge tests
|
|
working-directory: .
|
|
run: |
|
|
uv sync --frozen --no-install-project
|
|
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
|
|
|
|
- run: rustup toolchain install --no-self-update
|
|
|
|
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
|
|
with:
|
|
tool: cargo-nextest@0.9.143
|
|
|
|
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
with:
|
|
workspaces: litellm-rust
|
|
cache-on-failure: true
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Check generated trace contracts
|
|
working-directory: .
|
|
run: uv run scripts/generate_trace_types.py --check
|
|
|
|
- run: cargo nextest run --workspace --locked
|
|
|
|
- run: cargo test --workspace --doc --locked
|
|
|
|
- name: Test token counter feature combinations
|
|
run: |
|
|
for features in '' fast huggingface tiktoken fast,huggingface fast,tiktoken huggingface,tiktoken fast,huggingface,tiktoken; do
|
|
cargo test -p litellm-token-counter --locked --no-default-features --features "$features"
|
|
cargo check -p litellm-python-bridge --locked --no-default-features --features "abi3${features:+,$features}"
|
|
done
|
|
|
|
- name: Test secret manager feature combinations
|
|
run: |
|
|
cargo test -p litellm-auth-gcp --locked --no-default-features
|
|
for features in '' aws google hashicorp azure cyberark aws,google aws,azure google,azure aws,google,azure aws,google,cyberark aws,google,azure,cyberark aws,google,hashicorp,azure,cyberark; do
|
|
cargo test -p litellm-secrets --locked --no-default-features --features "$features"
|
|
done
|
|
|
|
rust-wheel:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- run: rustup toolchain install --no-self-update
|
|
|
|
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
|
with:
|
|
workspaces: litellm-rust
|
|
cache-on-failure: true
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- run: uv build --wheel --out-dir dist
|
|
|
|
- run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
|
|
env:
|
|
RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
|
|
|
- run: python tests/unit/rust_bridge/native_route_wheel_test.py dist/*.whl
|
|
|
|
- name: Run pytest tests/test_litellm_rust with the compiled extension
|
|
run: make test-rust-extension
|
|
|
|
- run: >-
|
|
uv build --wheel --out-dir panic-dist
|
|
--config-setting "maturin.build-args=--features panic-test,extension-module"
|
|
|
|
- run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl
|