Merge remote-tracking branch 'origin/main' into litellm_worker_healthcheck_default
# Conflicts: # litellm/constants.py # litellm/proxy/proxy_cli.py # tests/unit/proxy/test_proxy_cli.py
52
.agents/skills/database-access/SKILL.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
name: database-access
|
||||
description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
|
||||
---
|
||||
|
||||
# Database access
|
||||
|
||||
## Adding new database access
|
||||
|
||||
- Put Python database access behind an operation in `litellm/repositories`
|
||||
- Give callers typed inputs and outputs, not Prisma clients, models, or query dictionaries
|
||||
- Keep query execution and backend selection inside the repository
|
||||
- Prisma is fine for the implementation, adding a query does not require a Rust port
|
||||
- Native Rust callers use `litellm-rust/crates/db` directly
|
||||
|
||||
## Asked to move existing access behind a repository
|
||||
|
||||
- Identify the calls being moved and their transaction boundary
|
||||
- Move the existing Prisma implementation into the relevant repository
|
||||
- Replace those direct database calls with repository operations
|
||||
- Preserve return values, errors, and transaction behavior, with focused regression tests
|
||||
- Keep this an extraction, not a backend migration or unrelated cleanup
|
||||
|
||||
## An equivalent Rust or SQLx query already exists
|
||||
|
||||
- Leave it and the current backend selection unchanged unless integration is explicitly requested
|
||||
- Its existence alone does not justify switching a Prisma caller to Rust
|
||||
- Repository extraction can keep using Prisma even when the Rust query is available
|
||||
|
||||
## Asked to port a Prisma query to Rust or SQLx
|
||||
|
||||
- Identify the existing inputs, outputs, errors, and transaction boundary
|
||||
- Check `litellm-rust/crates/db` for an equivalent query and reuse it if its behavior matches
|
||||
- Otherwise follow `litellm-rust/crates/db/AGENTS.md`
|
||||
- Put static SQL in `queries/` and compile-check it with SQLx
|
||||
- Keep typed inputs and reusable row projections in `src/`
|
||||
- Refresh SQLx query metadata after SQL changes
|
||||
- Keep Python conversion in the Python bridge, not in `litellm-rust/crates/db`
|
||||
- When caller integration is requested, expose the Rust operation through the bridge and call it from the repository
|
||||
- A query-only port can land without switching callers
|
||||
- Verify behavior against real PostgreSQL before switching callers
|
||||
- Check relevant null handling, conflicts, ordering, errors, and transaction semantics
|
||||
- Remove the old implementation only when no callers need it
|
||||
|
||||
## Migration context and boundaries
|
||||
|
||||
- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations incrementally with Rust APIs
|
||||
- Mixed Prisma and Rust implementations are expected during the transition, unrelated existing calls can stay as they are
|
||||
- Both backends may have separate pools through PgBouncer, size them together and verify pooling-mode compatibility
|
||||
- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction
|
||||
- Never retry a failed Rust write through Prisma when it may already have committed
|
||||
- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port
|
||||
2624
.circleci/config.yml
|
|
@ -1,12 +1,11 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
|
||||
category="${1:?usage: classify_changes.sh <backend|client|ui|cost-map-only|mcp-dependencies|windows-release|redis-compat>}"
|
||||
|
||||
has_client=false
|
||||
has_backend=false
|
||||
has_ci=false
|
||||
has_provider_harness=false
|
||||
has_cost_map=false
|
||||
has_mcp_dependencies=false
|
||||
has_windows_release=false
|
||||
|
|
@ -15,7 +14,7 @@ outside_cost_map_set=false
|
|||
while IFS= read -r file || [ -n "$file" ]; do
|
||||
[ -n "$file" ] || continue
|
||||
case "$file" in
|
||||
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock)
|
||||
litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock)
|
||||
has_redis_compat=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
|
|
@ -23,11 +22,6 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py)
|
||||
has_mcp_dependencies=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
tests/e2e/*/*.py) : ;;
|
||||
tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/unit/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
|
||||
has_provider_harness=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*)
|
||||
has_windows_release=true ;;
|
||||
|
|
@ -39,9 +33,9 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
*) has_backend=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
|
||||
has_cost_map=true ;;
|
||||
tests/test_litellm/* | tests/proxy_unit_tests/* | tests/unit/proxy/*) : ;;
|
||||
tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
esac
|
||||
done
|
||||
|
|
@ -53,9 +47,6 @@ case "$category" in
|
|||
cost-map-only)
|
||||
{ [ "$has_cost_map" = true ] && [ "$outside_cost_map_set" = false ]; } && echo run || echo skip
|
||||
;;
|
||||
provider-harness)
|
||||
[ "$has_provider_harness" = true ] && echo run || echo skip
|
||||
;;
|
||||
windows-release)
|
||||
[ "$has_windows_release" = true ] && echo run || echo skip
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
category="${1:?usage: path_filter.sh <backend|client|provider-harness|redis-compat>}"
|
||||
category="${1:?usage: path_filter.sh <backend|client|windows-release>}"
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
run_full() {
|
||||
|
|
|
|||
|
|
@ -128,14 +128,31 @@ fi
|
|||
sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
|
||||
awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
|
||||
|
||||
setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
|
||||
.venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
|
||||
upstream_pid=$!
|
||||
if [ "$mode" = parity ]; then
|
||||
export INTEGRATION_WORKERS=1
|
||||
else
|
||||
case "$suite" in
|
||||
management|accounting) export INTEGRATION_WORKERS=2 ;;
|
||||
providers|extensions) export INTEGRATION_WORKERS=4 ;;
|
||||
esac
|
||||
fi
|
||||
if [ "$suite" = cost ]; then
|
||||
export INTEGRATION_WORKERS=8
|
||||
fi
|
||||
if [ "$suite" = security ]; then
|
||||
export INTEGRATION_WORKERS=4
|
||||
fi
|
||||
if [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_WORKERS=4 INTEGRATION_COVERAGE=1
|
||||
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
|
||||
fi
|
||||
isolated_workers=false
|
||||
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
|
||||
isolated_workers=true
|
||||
fi
|
||||
if [ "$isolated_workers" = false ]; then
|
||||
setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
|
||||
.venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
|
||||
upstream_pid=$!
|
||||
fi
|
||||
coverage_data="$PWD/$results/coverage/data"
|
||||
proxy_command=(.venv/bin/python -m integration._support.proxy)
|
||||
|
|
@ -177,26 +194,28 @@ start_proxy() {
|
|||
> "$results/$log_name" 2>&1 &
|
||||
launched_pid=$!
|
||||
}
|
||||
start_proxy 4000 proxy.log
|
||||
proxy_pid="$launched_pid"
|
||||
.venv/bin/python .circleci/scripts/wait_integration_services.py
|
||||
curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
|
||||
-d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json"
|
||||
if [ "$suite" = management ] || [ "$suite" = mcp ]; then
|
||||
export INTEGRATION_PEER_URL=http://127.0.0.1:4001
|
||||
start_proxy 4001 peer.log
|
||||
peer_pid="$launched_pid"
|
||||
if [ "$isolated_workers" = false ]; then
|
||||
start_proxy 4000 proxy.log
|
||||
proxy_pid="$launched_pid"
|
||||
.venv/bin/python .circleci/scripts/wait_integration_services.py
|
||||
curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \
|
||||
-d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json"
|
||||
if [ "$suite" = management ] || [ "$suite" = mcp ] || [ "$suite" = accounting ]; then
|
||||
export INTEGRATION_PEER_URL=http://127.0.0.1:4001
|
||||
start_proxy 4001 peer.log
|
||||
peer_pid="$launched_pid"
|
||||
.venv/bin/python .circleci/scripts/wait_integration_services.py
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$suite" = providers ]; then
|
||||
INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --tb=short --noconftest -o addopts= \
|
||||
--strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
|
||||
tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
|
||||
tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
|
||||
tests/e2e/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
|
||||
tests/e2e/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
|
||||
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
|
||||
tests/e2e_harness/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
|
||||
tests/e2e_harness/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
|
||||
tests/e2e_harness/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
|
||||
--junitxml="$results/replay-controls.xml"
|
||||
fi
|
||||
|
||||
|
|
@ -213,30 +232,42 @@ if [ "$suite" = browser ]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
node_files=()
|
||||
shard_arguments=()
|
||||
if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then
|
||||
split="$(.venv/bin/python tests/integration/run.py "$suite" --list \
|
||||
| circleci tests split --split-by=timings --timings-type=filename)"
|
||||
read -r -a node_files <<< "$(printf '%s' "$split" | tr '\n' ' ')"
|
||||
test "${#node_files[@]}" -gt 0
|
||||
printf '%s\n' "${node_files[@]}" > "$results/node-files.txt"
|
||||
.venv/bin/python tests/integration/run.py "$suite" --list \
|
||||
| circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt"
|
||||
shard_arguments=(--shard-files "$results/node-files.txt")
|
||||
fi
|
||||
|
||||
shared_services=()
|
||||
worker_coverage=()
|
||||
if [ "$isolated_workers" = false ]; then
|
||||
shared_services=(
|
||||
"REDIS_HOST=$REDIS_HOST" "REDIS_PORT=$REDIS_PORT"
|
||||
"INTEGRATION_PROXY_URL=$INTEGRATION_PROXY_URL" "INTEGRATION_PEER_URL=$INTEGRATION_PEER_URL"
|
||||
"INTEGRATION_UPSTREAM_URL=$INTEGRATION_UPSTREAM_URL" "INTEGRATION_MASTER_KEY=$INTEGRATION_MASTER_KEY"
|
||||
)
|
||||
elif [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
||||
worker_coverage=("INTEGRATION_COVERAGE_FILE=$coverage_data")
|
||||
fi
|
||||
|
||||
env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
|
||||
INTEGRATION_RUN_ID="$integration_identity" \
|
||||
DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
|
||||
INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
|
||||
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
|
||||
DATABASE_URL="$DATABASE_URL" \
|
||||
"${shared_services[@]}" "${worker_coverage[@]}" \
|
||||
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
|
||||
INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_MODE=PRODUCTION \
|
||||
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
|
||||
INTEGRATION_SEED="$INTEGRATION_SEED" \
|
||||
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
|
||||
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \
|
||||
LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
|
||||
INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \
|
||||
INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \
|
||||
INTEGRATION_ROUTING="$INTEGRATION_ROUTING" \
|
||||
.venv/bin/python tests/integration/run.py "$suite" --results "$results" "${node_files[@]}"
|
||||
MCP_TEST_PEER_PYTHON="${MCP_TEST_PEER_PYTHON:-}" \
|
||||
.venv/bin/python tests/integration/run.py "$suite" --results "$results" \
|
||||
"${shard_arguments[@]}"
|
||||
|
||||
if [ "${INTEGRATION_COVERAGE:-0}" = 1 ]; then
|
||||
for covered_pid in "$proxy_pid" "$peer_pid"; do
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from typing import Final
|
|||
from xml.etree import ElementTree
|
||||
|
||||
SUITES: Final = {
|
||||
"startup": (("test_startup.py",), 12),
|
||||
"startup": (("test_startup.py",), 13),
|
||||
"recovery": (("test_recovery.py",), 15),
|
||||
"legacy": (("test_legacy.py", "test_pooling.py"), 11),
|
||||
"upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5),
|
||||
|
|
|
|||
|
|
@ -1,42 +1,31 @@
|
|||
import os
|
||||
import time
|
||||
from contextlib import ExitStack
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.workers import wait_for_services
|
||||
from redis import Redis
|
||||
|
||||
|
||||
def main() -> None:
|
||||
primary: Final = os.environ["INTEGRATION_PROXY_URL"]
|
||||
peer: Final = os.environ.get("INTEGRATION_PEER_URL")
|
||||
proxies: Final = (primary, peer) if peer else (primary,)
|
||||
deadline: Final = time.monotonic() + 90
|
||||
headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"}
|
||||
with httpx.Client(trust_env=False, timeout=2) as client, Redis(
|
||||
host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2
|
||||
) as cache:
|
||||
while True:
|
||||
try:
|
||||
ready: Final = (
|
||||
client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200
|
||||
and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies)
|
||||
)
|
||||
if ready:
|
||||
for url in proxies:
|
||||
response: Final = client.get(f"{url}/cache/ping", headers=headers)
|
||||
response.raise_for_status()
|
||||
result: Final = response.json()
|
||||
assert result["status"] == "healthy", result
|
||||
assert result["cache_type"] == "redis", result
|
||||
assert result["ping_response"] is True, result
|
||||
assert result["set_cache_response"] == "success", result
|
||||
if cache.pubsub_numsub("litellm_proxy.auth_cache_invalidation")[0][1] >= len(proxies):
|
||||
return
|
||||
except httpx.TransportError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise SystemExit("Integration services or auth-cache subscribers did not become ready")
|
||||
time.sleep(0.2)
|
||||
urls: Final = (primary, peer) if peer else (primary,)
|
||||
upstream: Final = os.environ["INTEGRATION_UPSTREAM_URL"]
|
||||
with ExitStack() as stack:
|
||||
proxies: Final = tuple(
|
||||
Gateway(
|
||||
stack.enter_context(httpx.Client(base_url=url, trust_env=False, timeout=2)),
|
||||
os.environ["INTEGRATION_MASTER_KEY"],
|
||||
upstream,
|
||||
)
|
||||
for url in urls
|
||||
)
|
||||
cache: Final = stack.enter_context(
|
||||
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2)
|
||||
)
|
||||
wait_for_services(upstream, proxies, cache)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# pre-push — enforce Conventional Branches
|
||||
# https://conventional-branch.github.io/
|
||||
#
|
||||
# Branch format: <type>/<description>
|
||||
# <type> must be one of: feature, bugfix, hotfix, release, chore
|
||||
#
|
||||
# Protected branches (always allowed):
|
||||
# - main
|
||||
# - dependabot/*
|
||||
# - gh-readonly-queue/*
|
||||
#
|
||||
# Tag pushes and branch deletions are skipped.
|
||||
# Bypass: git push --no-verify.
|
||||
|
||||
set -eu
|
||||
|
||||
ZERO_OID="0000000000000000000000000000000000000000"
|
||||
ZERO_OID_SHA256="0000000000000000000000000000000000000000000000000000000000000000"
|
||||
ALLOWED_TYPES="feature|bugfix|hotfix|release|chore"
|
||||
BRANCH_PATTERN="^(${ALLOWED_TYPES})/.+"
|
||||
|
||||
PROTECTED_NAMES="main"
|
||||
PROTECTED_PREFIXES="dependabot/ gh-readonly-queue/"
|
||||
|
||||
is_protected() {
|
||||
branch="$1"
|
||||
for name in $PROTECTED_NAMES; do
|
||||
if [ "$branch" = "$name" ]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
for prefix in $PROTECTED_PREFIXES; do
|
||||
case "$branch" in "$prefix"*) return 0 ;; esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
invalid=""
|
||||
|
||||
while read -r local_ref local_oid remote_ref remote_oid; do
|
||||
# Branch deletion (no local commit being pushed).
|
||||
if [ "$local_oid" = "$ZERO_OID" ] || [ "$local_oid" = "$ZERO_OID_SHA256" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Only validate branch pushes; ignore tags and other ref namespaces.
|
||||
case "$remote_ref" in
|
||||
refs/heads/*) ;;
|
||||
*) continue ;;
|
||||
esac
|
||||
|
||||
branch="${remote_ref#refs/heads/}"
|
||||
|
||||
if is_protected "$branch"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! printf '%s' "$branch" | grep -Eq "$BRANCH_PATTERN"; then
|
||||
invalid="$invalid $branch"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$invalid" ]; then
|
||||
cat >&2 <<EOF
|
||||
✗ Branch name does not follow Conventional Branches.
|
||||
|
||||
Invalid:$invalid
|
||||
|
||||
Expected: <type>/<description>
|
||||
|
||||
Allowed types: feature, bugfix, hotfix, release, chore
|
||||
Examples:
|
||||
feature/weighted-round-robin
|
||||
bugfix/streaming-empty-chunks
|
||||
chore/bump-deps
|
||||
hotfix/auth-bypass
|
||||
|
||||
Protected (always allowed): main, dependabot/*, gh-readonly-queue/*.
|
||||
|
||||
See https://conventional-branch.github.io/
|
||||
|
||||
Rename with: git branch -m <new-name>
|
||||
To bypass (use sparingly): git push --no-verify
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
6
.github/CODEOWNERS
vendored
|
|
@ -1,2 +1,4 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
|
|
|
|||
11
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -4,6 +4,13 @@
|
|||
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
|
||||
"## Affected release" with nothing under it must not appear in the final description -->
|
||||
|
||||
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
|
||||
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
|
||||
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
|
||||
|
||||
<details>
|
||||
<summary>Details</summary>
|
||||
|
||||
## TLDR
|
||||
|
||||
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
|
||||
|
|
@ -68,7 +75,7 @@ After: the same request comes back with real token counts, so the dashboard show
|
|||
- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/unit/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
|
||||
- [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
|
||||
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
|
||||
- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
|
||||
- [ ] Every automated review on my PR's latest commit has finished, and I have addressed or replied to each of its findings before requesting a maintainer review
|
||||
|
||||
## Delays in PR merge?
|
||||
|
||||
|
|
@ -164,3 +171,5 @@ Example checklists:
|
|||
## Final Attestation
|
||||
|
||||
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
|
||||
|
||||
</details>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 248 KiB |
BIN
.github/assets/lens-datasets/dataset-case-panel.png
vendored
|
Before Width: | Height: | Size: 128 KiB |
BIN
.github/assets/lens-datasets/dataset-detail.png
vendored
|
Before Width: | Height: | Size: 101 KiB |
BIN
.github/assets/lens-datasets/datasets-list.png
vendored
|
Before Width: | Height: | Size: 132 KiB |
BIN
.github/assets/lens-datasets/old-revision.png
vendored
|
Before Width: | Height: | Size: 184 KiB |
BIN
.github/assets/lens-datasets/trace-header.png
vendored
|
Before Width: | Height: | Size: 259 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 77 KiB |
|
Before Width: | Height: | Size: 79 KiB |
|
Before Width: | Height: | Size: 61 KiB |
|
Before Width: | Height: | Size: 86 KiB |
|
Before Width: | Height: | Size: 75 KiB |
|
Before Width: | Height: | Size: 40 KiB |
|
Before Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 95 KiB |
|
Before Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 94 KiB |
|
Before Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 82 KiB |
|
Before Width: | Height: | Size: 64 KiB |
|
Before Width: | Height: | Size: 90 KiB |
BIN
.github/assets/roi-calculator/00-original-setup.png
vendored
|
Before Width: | Height: | Size: 80 KiB |
BIN
.github/assets/roi-calculator/01-connect-github.png
vendored
|
Before Width: | Height: | Size: 58 KiB |
BIN
.github/assets/roi-calculator/02-repositories.png
vendored
|
Before Width: | Height: | Size: 63 KiB |
|
Before Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 47 KiB |
BIN
.github/assets/roi-calculator/06-overview.png
vendored
|
Before Width: | Height: | Size: 76 KiB |
|
Before Width: | Height: | Size: 75 KiB |
BIN
.github/assets/roi-calculator/08-match-email.png
vendored
|
Before Width: | Height: | Size: 39 KiB |
BIN
.github/assets/roi-calculator/09-people-matched.png
vendored
|
Before Width: | Height: | Size: 70 KiB |
BIN
.github/assets/roi-calculator/10-pr-reasoning.png
vendored
|
Before Width: | Height: | Size: 93 KiB |
BIN
.github/assets/roi-calculator/11-settings.png
vendored
|
Before Width: | Height: | Size: 73 KiB |
BIN
.github/assets/roi-calculator/12-restart-setup.png
vendored
|
Before Width: | Height: | Size: 39 KiB |
|
Before Width: | Height: | Size: 81 KiB |
BIN
.github/assets/roi-calculator/14-overview-pulls.png
vendored
|
Before Width: | Height: | Size: 72 KiB |
BIN
.github/assets/roi-calculator/15-sample-preview.png
vendored
|
Before Width: | Height: | Size: 76 KiB |
|
Before Width: | Height: | Size: 50 KiB |
|
Before Width: | Height: | Size: 59 KiB |
|
Before Width: | Height: | Size: 57 KiB |
|
Before Width: | Height: | Size: 56 KiB |
|
Before Width: | Height: | Size: 65 KiB |
|
Before Width: | Height: | Size: 55 KiB |
BIN
.github/assets/scaledown/provider-after.jpg
vendored
Normal file
|
After Width: | Height: | Size: 108 KiB |
BIN
.github/assets/scaledown/provider-before.jpg
vendored
Normal file
|
After Width: | Height: | Size: 109 KiB |
63
.github/ci-coverage-allowlist.yml
vendored
|
|
@ -5,7 +5,7 @@ description: >-
|
|||
|
||||
test_paths:
|
||||
- reason: >-
|
||||
litellm.agent() end-to-end suite. It drives the real claude, codex and opencode CLIs and
|
||||
litellm.agent() end-to-end suite. It drives the real claude, codex, opencode and pi CLIs and
|
||||
deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus
|
||||
LITELLM_PROXY_API_BASE / LITELLM_PROXY_API_KEY, and skips without them. Run manually
|
||||
before changing litellm/harness; the mocked coverage runs in tests/unit/harness and
|
||||
|
|
@ -18,22 +18,11 @@ test_paths:
|
|||
paths:
|
||||
- tests/rust-python-harness
|
||||
- reason: >-
|
||||
Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that
|
||||
glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and
|
||||
not caching and not cache"`) or keep only another keyword (langfuse, router, assistants).
|
||||
Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in
|
||||
test_caching.py by node ID. It does not run that file's other tests.
|
||||
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now
|
||||
run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider
|
||||
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
|
||||
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
|
||||
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
|
||||
split that porting tests/local_testing off CircleCI will force, not a job that is red by
|
||||
construction
|
||||
Owned by the open pull request that moves the Rust/Python parity checks (#45135). It ran in
|
||||
the CircleCI build_and_test job that globbed tests/test_*.py, which this migration deletes,
|
||||
and that pull request gives it a new home
|
||||
paths:
|
||||
- tests/local_testing/test_caching.py
|
||||
- tests/local_testing/test_disk_cache_unit_tests.py
|
||||
- tests/local_testing/test_gcs_cache_unit_tests.py
|
||||
- tests/test_rust_python_harness.py
|
||||
- reason: >-
|
||||
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
|
||||
from a pull request; it needs a live gateway and provider credentials no PR job holds
|
||||
|
|
@ -59,48 +48,6 @@ test_paths:
|
|||
choice it informed is settled
|
||||
paths:
|
||||
- tests/code_coverage_tests/test_aio_http_image_conversion.py
|
||||
- reason: >-
|
||||
No job invokes this suite and its files mix pure transformation tests with ones driving live
|
||||
vendor vector stores, so assigning them needs a per-file decision
|
||||
paths:
|
||||
- tests/vector_store_tests/rag/test_rag_bedrock.py
|
||||
- tests/vector_store_tests/rag/test_rag_openai.py
|
||||
- tests/vector_store_tests/rag/test_rag_s3_vectors.py
|
||||
- tests/vector_store_tests/rag/test_rag_vertex_ai.py
|
||||
- tests/vector_store_tests/test_azure_ai_vector_store.py
|
||||
- tests/vector_store_tests/test_azure_vector_store.py
|
||||
- tests/vector_store_tests/test_bedrock_vector_store.py
|
||||
- tests/vector_store_tests/test_gemini_vector_store.py
|
||||
- tests/vector_store_tests/test_milvus_vector_store.py
|
||||
- tests/vector_store_tests/test_openai_vector_store.py
|
||||
- tests/vector_store_tests/test_ragflow_vector_store.py
|
||||
- tests/vector_store_tests/test_s3_vectors_vector_store.py
|
||||
- tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py
|
||||
- tests/vector_store_tests/test_vertex_ai_vector_store.py
|
||||
- reason: >-
|
||||
Throughput and memory-growth measurements whose runtime and variance make them unsuitable for
|
||||
a per-pull-request job
|
||||
paths:
|
||||
- tests/load_tests/test_datadog_load_test.py
|
||||
- tests/load_tests/test_langsmith_load_test.py
|
||||
- tests/load_tests/test_linear_memory_growth.py
|
||||
- tests/load_tests/test_memory_usage.py
|
||||
- tests/load_tests/test_otel_load_test.py
|
||||
- tests/load_tests/test_vertex_embeddings_load_test.py
|
||||
- tests/load_tests/test_vertex_load_tests.py
|
||||
- reason: >-
|
||||
Env-gated saturation benchmark requires a live proxy and provider credentials, so it is run
|
||||
locally rather than in pull-request jobs
|
||||
paths:
|
||||
- tests/load_tests/test_granian_admission_saturation.py
|
||||
- reason: >-
|
||||
A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
|
||||
localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
|
||||
pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v
|
||||
that this census could not see; the glob now excludes them structurally and this entry
|
||||
is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture
|
||||
paths:
|
||||
- tests/agent_tests/local_only_agent_tests
|
||||
- reason: >-
|
||||
Third-party integration tests that skip themselves without OCI configuration or sandbox
|
||||
credentials, neither of which a pull request job holds
|
||||
|
|
|
|||
14
.github/codeql/codeql-config.yml
vendored
|
|
@ -1,8 +1,5 @@
|
|||
name: "LiteLLM CodeQL config"
|
||||
|
||||
queries:
|
||||
- uses: security-and-quality
|
||||
|
||||
# Known OOM queries on large Python codebases:
|
||||
# CodeQL builds a full data flow graph in memory. These two queries trace
|
||||
# sensitive data through every log call / regex pattern, causing combinatorial
|
||||
|
|
@ -14,17 +11,6 @@ query-filters:
|
|||
id: py/clear-text-logging-sensitive-data # CWE-312
|
||||
- exclude:
|
||||
id: py/polynomial-redos # CWE-730
|
||||
# Import resolution confuses stdlib types with management_endpoints/types.py.
|
||||
# The generic cycle query also reports intentional deferred imports.
|
||||
- exclude:
|
||||
id: py/cyclic-import
|
||||
- exclude:
|
||||
id: py/unsafe-cyclic-import
|
||||
# Known false positives on live settings and Protocol placeholders.
|
||||
- exclude:
|
||||
id: py/unused-global-variable
|
||||
- exclude:
|
||||
id: py/ineffectual-statement
|
||||
|
||||
paths-ignore:
|
||||
- tests
|
||||
|
|
|
|||
69
.github/e2e-stack/assert_tests_ran.py
vendored
|
|
@ -1,69 +0,0 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
|
||||
from coverage_registry.management_cases import MANAGEMENT_CASES
|
||||
|
||||
|
||||
def main() -> int:
|
||||
selected: Final = tuple(sys.argv[2:])
|
||||
try:
|
||||
report: Final = ET.parse(Path(sys.argv[1])).getroot()
|
||||
except (ET.ParseError, OSError):
|
||||
_ = sys.stdout.write("::error::could not read the test execution report\n")
|
||||
return 1
|
||||
cases: Final = tuple(report.iter("testcase"))
|
||||
expected_count: Final = os.environ.get("E2E_REQUIRED_TEST_COUNT")
|
||||
passed: Final = frozenset(
|
||||
case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
|
||||
)
|
||||
missing: Final = tuple(path for path in selected if path not in passed)
|
||||
required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
|
||||
passed_nodes: Final = frozenset(
|
||||
prop.get("value")
|
||||
for case in cases
|
||||
if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
|
||||
for prop in case.findall("./properties/property")
|
||||
if prop.get("name") == "management_node"
|
||||
)
|
||||
missing_nodes: Final = required_nodes - passed_nodes
|
||||
for node in sorted(missing_nodes):
|
||||
_ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
|
||||
for path in selected:
|
||||
collected: Final = sum(case.get("file") == path for case in cases)
|
||||
skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
|
||||
_ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
|
||||
for case in cases:
|
||||
if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error", "skipped")):
|
||||
continue
|
||||
outcome = "skipped" if case.find("skipped") is not None else "failed"
|
||||
_ = sys.stdout.write(f" {outcome}: {case.get('classname', '')}::{case.get('name', '')}\n")
|
||||
for prop in case.findall("./properties/property"):
|
||||
name = prop.get("name", "")
|
||||
value = prop.get("value", "")
|
||||
if name in ("oauth_failure_phase", "oauth_exception_type", "oauth_frame") and re.fullmatch(
|
||||
r"[A-Za-z0-9_.:<>-]{1,240}", value
|
||||
):
|
||||
_ = sys.stdout.write(f" {name}: {value}\n")
|
||||
if expected_count is not None and (
|
||||
len(cases) != int(expected_count) or any(case.find("skipped") is not None for case in cases)
|
||||
):
|
||||
_ = sys.stdout.write("::error::required test count was not met or a required case was skipped\n")
|
||||
return 1
|
||||
if (
|
||||
selected
|
||||
and not missing
|
||||
and not missing_nodes
|
||||
and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
|
||||
):
|
||||
return 0
|
||||
_ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
15
.github/merge-smoke-tests.json
vendored
|
|
@ -1,15 +0,0 @@
|
|||
{
|
||||
"cases": {
|
||||
"CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
|
||||
"CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
|
||||
"CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
|
||||
"MODEL-ALLOW": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_allows_listed_model_for_key",
|
||||
"MODEL-DENY": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
|
||||
"COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
|
||||
"COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
|
||||
"LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",
|
||||
"LOG-CONTENT-OFF": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_redacts_message_content_when_message_logging_is_off",
|
||||
"CALLBACK-SUCCESS": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_success_handler_delivers_standard_logging_payload_to_custom_logger",
|
||||
"CALLBACK-FAILURE": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_async_failure_handler_delivers_failure_payload_to_custom_logger"
|
||||
}
|
||||
}
|
||||
75
.github/scripts/assert_ci_coverage.py
vendored
|
|
@ -9,6 +9,8 @@ import sys
|
|||
import warnings
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import yaml
|
||||
|
|
@ -30,13 +32,21 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
|
|||
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
|
||||
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
|
||||
GLOB_CHARS = frozenset("*?")
|
||||
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
|
||||
|
||||
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
|
||||
# tests has to be named by some shard or it runs nowhere. A child listed here is
|
||||
# itself decomposed one level deeper and is checked through its own entry.
|
||||
SHARDED_ROOTS: tuple[str, ...] = (
|
||||
"tests/test_litellm",
|
||||
"tests/unit",
|
||||
"tests/unit/enterprise",
|
||||
"tests/unit/enterprise/enterprise_callbacks",
|
||||
"tests/unit/enterprise/proxy",
|
||||
"tests/unit/llms",
|
||||
"tests/unit/proxy",
|
||||
"tests/unit/proxy/_experimental",
|
||||
"tests/unit/responses",
|
||||
"tests/unit/skills",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -308,6 +318,7 @@ def _keyword_terms(
|
|||
def _slices() -> tuple[Slice, ...]:
|
||||
if not CIRCLECI_CONFIG.exists():
|
||||
return ()
|
||||
test_files: Final = frozenset(_test_files())
|
||||
jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {})
|
||||
return tuple(
|
||||
Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood)
|
||||
|
|
@ -315,7 +326,7 @@ def _slices() -> tuple[Slice, ...]:
|
|||
for text in ("\n".join(_strings(body)),)
|
||||
if "pytest" in text
|
||||
for globs in (tuple(GLOB_CALL_RE.findall(text)),)
|
||||
for named in (frozenset(TEST_TOKEN_RE.findall(text)) & frozenset(_test_files()),)
|
||||
for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,)
|
||||
for required, excluded, understood in (
|
||||
_keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2),
|
||||
)
|
||||
|
|
@ -355,6 +366,7 @@ def _workflow_named_tokens() -> frozenset[str]:
|
|||
|
||||
|
||||
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
|
||||
matchable_names: Final = lru_cache(maxsize=None)(_matchable_names)
|
||||
slices: Final = _slices()
|
||||
named_by_workflow: Final = _workflow_named_tokens()
|
||||
globbed: Final = tuple(
|
||||
|
|
@ -368,7 +380,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
|
|||
for path in globbed
|
||||
if not allowlist.covers_test(path)
|
||||
and not any(_token_covers(token, path) for token in named_by_workflow)
|
||||
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
|
||||
and not any(slice_.claims(path, matchable_names(path)) for slice_ in slices)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -549,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
|
|||
return frozenset(ast.literal_eval(values))
|
||||
|
||||
|
||||
def _owns(workflow: str, relative_path: str) -> bool:
|
||||
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
|
||||
return owned_root is not None and relative_path.startswith(owned_root)
|
||||
|
||||
|
||||
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
|
||||
return frozenset[str]().union(
|
||||
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
|
||||
)
|
||||
|
||||
|
||||
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
|
||||
runner: Final = repo_root / "tests/integration/run.py"
|
||||
if not runner.exists():
|
||||
|
|
@ -600,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
)
|
||||
!= 1
|
||||
)
|
||||
gha_tokens: Final = _invoked_test_tokens(
|
||||
scalar
|
||||
for path in (repo_root / ".github/workflows").glob("*.y*ml")
|
||||
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
|
||||
workflow_tokens: Final = MappingProxyType(
|
||||
{
|
||||
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
|
||||
for path in (repo_root / ".github/workflows").glob("*.y*ml")
|
||||
}
|
||||
)
|
||||
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
|
||||
circle_owned: Final = (
|
||||
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
|
||||
)
|
||||
findings: Final = (
|
||||
tuple(
|
||||
Finding(path, "integration contract is also selected by GitHub Actions")
|
||||
for path in paths
|
||||
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
|
||||
for path in sorted(circle_owned)
|
||||
if any(_token_covers(token, path) for token in gha_tokens)
|
||||
)
|
||||
+ tuple(
|
||||
|
|
@ -621,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
|
|||
for path in sorted(github_files)
|
||||
if not (repo_root / path).is_file()
|
||||
)
|
||||
+ tuple(
|
||||
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
|
||||
for path in sorted(github_files)
|
||||
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
|
||||
)
|
||||
)
|
||||
browser_commands: Final = tuple(
|
||||
scalar.value
|
||||
|
|
@ -677,13 +710,9 @@ def main() -> int:
|
|||
scalars = _all_scalars()
|
||||
|
||||
integration_paths, ownership_findings = _integration_ownership()
|
||||
test_findings = (
|
||||
_uncovered_tests(
|
||||
allowlist,
|
||||
_invoked_selections(scalars)
|
||||
+ (Selection(included=integration_paths, ignored=frozenset()),),
|
||||
)
|
||||
+ ownership_findings
|
||||
test_findings = _uncovered_tests(
|
||||
allowlist,
|
||||
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
|
||||
)
|
||||
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
|
||||
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
|
||||
|
|
@ -700,13 +729,25 @@ def main() -> int:
|
|||
test_findings,
|
||||
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
|
||||
)
|
||||
if ownership_findings:
|
||||
browser_remedy = (
|
||||
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
|
||||
"select them and the default Playwright config must exclude them."
|
||||
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
|
||||
else ""
|
||||
)
|
||||
_report(
|
||||
"integration contracts with the wrong CI owner",
|
||||
ownership_findings,
|
||||
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
|
||||
)
|
||||
if dockerfile_findings:
|
||||
_report(
|
||||
"Dockerfiles that no CI job builds",
|
||||
dockerfile_findings,
|
||||
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
|
||||
)
|
||||
if stale_findings or test_findings or dockerfile_findings:
|
||||
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
|
||||
return 1
|
||||
|
||||
_write(
|
||||
|
|
|
|||
|
|
@ -1,319 +0,0 @@
|
|||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
# Asynchronously fetch data from a given URL
|
||||
async def fetch_data(url):
|
||||
try:
|
||||
# Create an asynchronous session
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Send a GET request to the URL
|
||||
async with session.get(url) as resp:
|
||||
# Raise an error if the response status is not OK
|
||||
resp.raise_for_status()
|
||||
# Parse the response JSON
|
||||
resp_json = await resp.json()
|
||||
print("Fetch the data from URL.")
|
||||
# Return the 'data' field from the JSON response
|
||||
return resp_json['data']
|
||||
except Exception as e:
|
||||
# Print an error message if fetching data fails
|
||||
print("Error fetching data from URL:", e)
|
||||
return None
|
||||
|
||||
|
||||
FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
|
||||
FRIENDLI_PROVIDER = "friendliai"
|
||||
|
||||
INHERITABLE_BASE_KEYS = (
|
||||
"supports_pdf_input",
|
||||
"supports_assistant_prefill",
|
||||
"supports_adaptive_thinking",
|
||||
"supports_output_config",
|
||||
)
|
||||
|
||||
REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
|
||||
|
||||
|
||||
def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
|
||||
if not base_model:
|
||||
return None
|
||||
bm_tail = base_model.split("/")[-1].lower()
|
||||
if base_model in local_data:
|
||||
return base_model
|
||||
for key in local_data:
|
||||
if key.startswith("sample_spec") or key == "fallback_generalizations":
|
||||
continue
|
||||
if key.split("/")[-1].lower() == bm_tail:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _reasoning_effort_levels(reasoning_options: list) -> list:
|
||||
offered = {
|
||||
val
|
||||
for opt in reasoning_options or []
|
||||
if opt.get("type") == "effort"
|
||||
for val in opt.get("values", [])
|
||||
}
|
||||
return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
|
||||
|
||||
|
||||
def _valid_token_price(value: object) -> bool:
|
||||
try:
|
||||
price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return math.isfinite(price) and price >= 0
|
||||
|
||||
|
||||
def _has_valid_token_prices(pricing: dict | None) -> bool:
|
||||
prices = pricing or {}
|
||||
return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
|
||||
|
||||
|
||||
def _pricing(pricing: dict) -> dict:
|
||||
out: dict[str, Any] = {}
|
||||
if not pricing:
|
||||
return out
|
||||
if "input" in pricing:
|
||||
out["input_cost_per_token"] = float(pricing["input"])
|
||||
if "output" in pricing:
|
||||
out["output_cost_per_token"] = float(pricing["output"])
|
||||
if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
|
||||
out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
|
||||
return out
|
||||
|
||||
|
||||
def _modality_flags(input_mods: list) -> dict:
|
||||
mods = input_mods or []
|
||||
has_image = "image" in mods
|
||||
return {
|
||||
"supports_vision": has_image,
|
||||
"supports_image_input": has_image,
|
||||
"supports_video_input": "video" in mods,
|
||||
}
|
||||
|
||||
|
||||
def transform_friendli_data(data: list, local_data: dict) -> dict:
|
||||
transformed: dict[str, dict] = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for model in data:
|
||||
# An unpriced row must never wholesale-replace an already priced local entry:
|
||||
# missing prices cost-calculate as zero, silently zeroing tracked spend
|
||||
if not _has_valid_token_prices(model.get("pricing")):
|
||||
continue
|
||||
model_id = model["id"]
|
||||
base_model = model.get("base_model") or ""
|
||||
entry: dict[str, Any] = {
|
||||
"litellm_provider": FRIENDLI_PROVIDER,
|
||||
}
|
||||
|
||||
base_key = _find_base_model_entry(base_model, local_data)
|
||||
if base_key:
|
||||
base_entry = local_data[base_key]
|
||||
for k in INHERITABLE_BASE_KEYS:
|
||||
if k in base_entry:
|
||||
entry[k] = base_entry[k]
|
||||
|
||||
ctx = model.get("context_length")
|
||||
if ctx is not None:
|
||||
entry["max_input_tokens"] = int(ctx)
|
||||
max_out = model.get("max_completion_tokens")
|
||||
if max_out is not None:
|
||||
entry["max_output_tokens"] = int(max_out)
|
||||
entry["max_tokens"] = int(max_out)
|
||||
|
||||
pricing = _pricing(model.get("pricing", {}))
|
||||
entry.update(pricing)
|
||||
entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
|
||||
|
||||
reasoning = model.get("reasoning") is True
|
||||
entry["supports_reasoning"] = reasoning
|
||||
if reasoning:
|
||||
entry["reasoning_effort_levels"] = _reasoning_effort_levels(
|
||||
model.get("reasoning_options", [])
|
||||
)
|
||||
|
||||
func = model.get("functionality", {})
|
||||
entry["supports_function_calling"] = func.get("tool_call") is True
|
||||
entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
|
||||
is_struct = func.get("structured_output") is True
|
||||
entry["supports_response_schema"] = is_struct
|
||||
entry["supports_native_structured_output"] = is_struct
|
||||
entry["supports_system_messages"] = func.get("system_messages") is True
|
||||
entry["supports_tool_choice"] = func.get("tool_choice") is True
|
||||
|
||||
entry.update(_modality_flags(model.get("input_modalities", [])))
|
||||
|
||||
entry["mode"] = model.get("mode", "chat")
|
||||
|
||||
desc = model.get("description")
|
||||
if desc:
|
||||
entry["comment"] = desc
|
||||
|
||||
dep = model.get("deprecation_date")
|
||||
if dep:
|
||||
entry["deprecation_date"] = dep.split("T")[0]
|
||||
|
||||
entry["source"] = FRIENDLI_API_URL
|
||||
|
||||
transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
|
||||
return transformed
|
||||
|
||||
# Synchronize local data with remote data
|
||||
def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
|
||||
# Update existing keys in local_data with values from remote_data
|
||||
# (replace_keys entries are swapped wholesale so a field the remote catalog
|
||||
# dropped, e.g. cache pricing, cannot survive as a stale value)
|
||||
for key in (set(local_data) & set(remote_data)):
|
||||
if key in replace_keys:
|
||||
local_data[key] = remote_data[key]
|
||||
else:
|
||||
local_data[key].update(remote_data[key])
|
||||
|
||||
# Add new keys from remote_data to local_data
|
||||
for key in (set(remote_data) - set(local_data)):
|
||||
local_data[key] = remote_data[key]
|
||||
|
||||
# Write data to the json file
|
||||
def write_to_file(file_path, data):
|
||||
try:
|
||||
# Open the file in write mode
|
||||
with open(file_path, "w") as file:
|
||||
# Dump the data as JSON into the file
|
||||
json.dump(data, file, indent=4)
|
||||
print("Values updated successfully.")
|
||||
except Exception as e:
|
||||
# Print an error message if writing to file fails
|
||||
print("Error updating JSON file:", e)
|
||||
|
||||
# Update the existing models and add the missing models for OpenRouter
|
||||
def transform_openrouter_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Add the fields 'max_tokens' and 'input_cost_per_token'
|
||||
obj = {
|
||||
"max_tokens": row["context_length"],
|
||||
"input_cost_per_token": float(row["pricing"]["prompt"]),
|
||||
}
|
||||
|
||||
# Add 'max_output_tokens' as a field if it is not None
|
||||
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
|
||||
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
|
||||
|
||||
# Add the field 'output_cost_per_token'
|
||||
obj.update({
|
||||
"output_cost_per_token": float(row["pricing"]["completion"]),
|
||||
})
|
||||
|
||||
# Add field 'input_cost_per_image' if it exists and is non-zero
|
||||
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
|
||||
obj['input_cost_per_image'] = float(row["pricing"]["image"])
|
||||
|
||||
# Add the fields 'litellm_provider' and 'mode'
|
||||
obj.update({
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat"
|
||||
})
|
||||
|
||||
# Add the 'supports_vision' field if the modality is 'multimodal'
|
||||
if row.get('architecture', {}).get('modality') == 'multimodal':
|
||||
obj['supports_vision'] = True
|
||||
|
||||
# Use a composite key to store the transformed object
|
||||
transformed[f'openrouter/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
# Update the existing models and add the missing models for Vercel AI Gateway
|
||||
def transform_vercel_ai_gateway_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
|
||||
if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
|
||||
row.get("pricing", {}).get(k) is None for k in ("input", "output")
|
||||
):
|
||||
continue
|
||||
obj = {
|
||||
"max_tokens": row["context_window"],
|
||||
"input_cost_per_token": float(row["pricing"]["input"]),
|
||||
"output_cost_per_token": float(row["pricing"]["output"]),
|
||||
'max_output_tokens': row['max_tokens'],
|
||||
'max_input_tokens': row["context_window"],
|
||||
}
|
||||
|
||||
# Handle cache pricing if available
|
||||
if "pricing" in row:
|
||||
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
|
||||
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
|
||||
|
||||
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
|
||||
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
|
||||
|
||||
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
|
||||
|
||||
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
|
||||
|
||||
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
# Load local data from a specified file
|
||||
def load_local_data(file_path):
|
||||
try:
|
||||
# Open the file in read mode
|
||||
with open(file_path, "r") as file:
|
||||
# Load and return the JSON data
|
||||
return json.load(file)
|
||||
except FileNotFoundError:
|
||||
# Print an error message if the file is not found
|
||||
print("File not found:", file_path)
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
# Print an error message if JSON decoding fails
|
||||
print("Error decoding JSON:", e)
|
||||
return None
|
||||
|
||||
def main():
|
||||
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
|
||||
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
|
||||
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
|
||||
|
||||
# Load local data from file
|
||||
local_data = load_local_data(local_file_path)
|
||||
|
||||
# Fetch OpenRouter data
|
||||
openrouter_data = asyncio.run(fetch_data(openrouter_url))
|
||||
# Transform the fetched OpenRouter data
|
||||
openrouter_data = transform_openrouter_data(openrouter_data)
|
||||
|
||||
# Fetch Vercel AI Gateway data
|
||||
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
|
||||
# Transform the fetched Vercel AI Gateway data
|
||||
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
|
||||
|
||||
friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
|
||||
friendli_data = transform_friendli_data(friendli_data, local_data)
|
||||
|
||||
# Combine both datasets
|
||||
all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
|
||||
|
||||
# If both local and openrouter data are available, synchronize and save
|
||||
if local_data and all_remote_data:
|
||||
sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
|
||||
write_to_file(local_file_path, local_data)
|
||||
else:
|
||||
print("Failed to fetch model data from either local file or URL.")
|
||||
|
||||
# Entry point of the script
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
140
.github/scripts/run_merge_smoke.py
vendored
|
|
@ -16,29 +16,11 @@ import socket
|
|||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NoReturn, TextIO, cast
|
||||
|
||||
import pytest
|
||||
|
||||
EXPECTED_CASES: Final = (
|
||||
"CHAT-JSON",
|
||||
"CHAT-TEXT-STREAM",
|
||||
"CHAT-TOOL-STREAM",
|
||||
"MODEL-ALLOW",
|
||||
"MODEL-DENY",
|
||||
"COST-EXPLICIT",
|
||||
"COST-ZERO",
|
||||
"LOG-CONTENT-ON",
|
||||
"LOG-CONTENT-OFF",
|
||||
"CALLBACK-SUCCESS",
|
||||
"CALLBACK-FAILURE",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CheckResult:
|
||||
|
|
@ -57,8 +39,6 @@ class _Args:
|
|||
ready_deadline: float = 120.0
|
||||
shutdown_deadline: float = 20.0
|
||||
poll_interval: float = 0.5
|
||||
manifest: str = ""
|
||||
rootdir: str | None = None
|
||||
|
||||
|
||||
def fail(reason: str) -> NoReturn:
|
||||
|
|
@ -345,120 +325,6 @@ def _terminate(proc: subprocess.Popen[bytes], log_file: TextIO) -> None:
|
|||
log_file.close()
|
||||
|
||||
|
||||
def _load_manifest(path: Path) -> MappingProxyType[str, str]:
|
||||
def no_duplicates(pairs: list[tuple[object, object]]) -> dict[object, object]:
|
||||
seen: dict[object, object] = {}
|
||||
for key, value in pairs:
|
||||
if key in seen:
|
||||
raise ValueError(f"duplicate key in manifest: {key}")
|
||||
seen[key] = value
|
||||
return seen
|
||||
|
||||
raw_value: object = cast(object, json.loads(path.read_text(), object_pairs_hook=no_duplicates))
|
||||
if not isinstance(raw_value, dict):
|
||||
raise ValueError("manifest must be an object")
|
||||
loaded: Final = cast(dict[object, object], raw_value)
|
||||
cases_value: object = loaded.get("cases")
|
||||
if not isinstance(cases_value, dict):
|
||||
raise ValueError("manifest must be an object with a 'cases' object")
|
||||
cases_any: Final = cast(dict[object, object], cases_value)
|
||||
cases: Final = {k: v for k, v in cases_any.items() if isinstance(k, str) and isinstance(v, str)}
|
||||
if len(cases) != len(cases_any):
|
||||
raise ValueError("manifest 'cases' must map string ids to string node ids")
|
||||
return MappingProxyType(cases)
|
||||
|
||||
|
||||
@dataclass(slots=True, eq=False)
|
||||
class _Recorder:
|
||||
collect_failed: list[str] = field(default_factory=list)
|
||||
collected: tuple[str, ...] = ()
|
||||
reports: dict[str, list[tuple[str, str, bool]]] = field(default_factory=dict)
|
||||
|
||||
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
|
||||
if report.failed:
|
||||
self.collect_failed.append(report.nodeid)
|
||||
|
||||
def pytest_collection_finish(self, session: pytest.Session) -> None:
|
||||
self.collected = tuple(item.nodeid for item in session.items)
|
||||
|
||||
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
|
||||
self.reports.setdefault(report.nodeid, []).append((report.when, report.outcome, hasattr(report, "wasxfail")))
|
||||
|
||||
|
||||
def cmd_pytest(args: _Args) -> int:
|
||||
try:
|
||||
cases: Final = _load_manifest(Path(args.manifest))
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
fail(f"manifest invalid: {exc}")
|
||||
if tuple(cases) != EXPECTED_CASES:
|
||||
fail(f"manifest case ids must be exactly {list(EXPECTED_CASES)} in order, got {list(cases)}")
|
||||
node_ids: Final = tuple(cases.values())
|
||||
if len(set(node_ids)) != len(node_ids):
|
||||
fail("manifest node ids are not unique")
|
||||
argv: Final = [
|
||||
*node_ids,
|
||||
"-p",
|
||||
"no:cacheprovider",
|
||||
"-p",
|
||||
"no:xdist",
|
||||
"-p",
|
||||
"no:rerunfailures",
|
||||
"-p",
|
||||
"no:randomly",
|
||||
"-rA",
|
||||
"-q",
|
||||
*(["--rootdir", args.rootdir] if args.rootdir else []),
|
||||
]
|
||||
|
||||
recorder: Final = _Recorder()
|
||||
code: Final = pytest.main(argv, plugins=[recorder])
|
||||
name_of: Final = MappingProxyType({node_id: case_id for case_id, node_id in cases.items()})
|
||||
problems: Final[list[str]] = []
|
||||
if code != 0:
|
||||
problems.append(f"pytest exit code {code}")
|
||||
for failed_id in recorder.collect_failed:
|
||||
problems.append(f"collection failed: {name_of.get(failed_id, failed_id)}")
|
||||
expected: Final = Counter(node_ids)
|
||||
collected: Final = Counter(recorder.collected)
|
||||
for node_id in expected - collected:
|
||||
problems.append(f"missing case {name_of[node_id]} ({node_id})")
|
||||
for node_id in collected - expected:
|
||||
problems.append(f"unexpected test collected: {node_id}")
|
||||
for node_id, count in collected.items():
|
||||
if count > 1:
|
||||
problems.append(f"duplicated test id: {node_id}")
|
||||
if len(recorder.collected) != len(EXPECTED_CASES):
|
||||
problems.append(f"collected {len(recorder.collected)} tests, expected {len(EXPECTED_CASES)}")
|
||||
rows: Final[list[tuple[str, bool]]] = []
|
||||
for case_id, node_id in cases.items():
|
||||
reports = recorder.reports.get(node_id, [])
|
||||
case_ok = (
|
||||
bool(reports)
|
||||
and all(outcome == "passed" and not wasxfail for _, outcome, wasxfail in reports)
|
||||
and {when for when, _, _ in reports} >= {"setup", "call", "teardown"}
|
||||
)
|
||||
rows.append((case_id, case_ok))
|
||||
if not reports:
|
||||
problems.append(f"{case_id} ({node_id}) produced no runtest reports")
|
||||
continue
|
||||
for when, outcome, wasxfail in reports:
|
||||
if outcome != "passed":
|
||||
problems.append(f"{case_id} ({node_id}) {when} outcome={outcome}")
|
||||
if wasxfail:
|
||||
problems.append(f"{case_id} ({node_id}) {when} was xfail/xpass")
|
||||
missing_phases = {"setup", "call", "teardown"} - {when for when, _, _ in reports}
|
||||
for phase in sorted(missing_phases):
|
||||
problems.append(f"{case_id} ({node_id}) missing {phase} report")
|
||||
for case_id, passed in rows:
|
||||
print(f"{case_id} {'PASS' if passed else 'FAIL'} {cases[case_id]}")
|
||||
if problems:
|
||||
for problem in problems:
|
||||
print(f"merge-smoke: {problem}", file=sys.stderr)
|
||||
fail("pytest verdict failed")
|
||||
ok("pytest 11 cases")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser: Final = argparse.ArgumentParser(description=__doc__)
|
||||
subs: Final = parser.add_subparsers(dest="command", required=True)
|
||||
|
|
@ -475,16 +341,12 @@ def main() -> int:
|
|||
p_proxy.add_argument("--ready-deadline", type=float, default=120)
|
||||
p_proxy.add_argument("--shutdown-deadline", type=float, default=20)
|
||||
p_proxy.add_argument("--poll-interval", type=float, default=0.5)
|
||||
p_test: Final = subs.add_parser("pytest")
|
||||
p_test.add_argument("--manifest", required=True)
|
||||
p_test.add_argument("--rootdir", default=None)
|
||||
args: Final = parser.parse_args(namespace=_Args())
|
||||
handlers: Final = {
|
||||
"verify-isolation": cmd_verify_isolation,
|
||||
"interpreter": cmd_interpreter,
|
||||
"cli": cmd_cli,
|
||||
"proxy-startup": cmd_proxy_startup,
|
||||
"pytest": cmd_pytest,
|
||||
}
|
||||
return handlers[args.command](args)
|
||||
|
||||
|
|
|
|||
283
.github/workflows/_test-unit-base.yml
vendored
|
|
@ -1,283 +0,0 @@
|
|||
name: _Unit Test Base (Reusable)
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
python-version:
|
||||
description: "Python version used to install dependencies and run tests"
|
||||
required: false
|
||||
type: string
|
||||
default: "3.12"
|
||||
test-path:
|
||||
description: >-
|
||||
Space-separated pytest paths to run. A path that no longer exists is
|
||||
dropped with a warning instead of being passed to pytest, because one
|
||||
missing path makes pytest-xdist collect nothing and report exit 5, which
|
||||
the step treats as a drained shard. Options are passed through as
|
||||
written, so use the `--flag=value` form: a bare `--ignore path` would
|
||||
have its path existence-checked like any other token.
|
||||
required: true
|
||||
type: string
|
||||
workers:
|
||||
description: "Number of pytest-xdist workers"
|
||||
required: false
|
||||
type: number
|
||||
default: 2
|
||||
reruns:
|
||||
description: "Number of reruns for flaky tests"
|
||||
required: false
|
||||
type: number
|
||||
default: 2
|
||||
timeout-minutes:
|
||||
description: >-
|
||||
Timeout for the test step alone. Setup (checkout, dependency install,
|
||||
Prisma client generation) gets its own allowance on top, so a slow
|
||||
runner or a cold binary download can never cancel passing tests.
|
||||
required: false
|
||||
type: number
|
||||
default: 20
|
||||
job-timeout-minutes:
|
||||
description: >-
|
||||
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
|
||||
the per-step ceilings on the setup steps below, and 5 for the runner
|
||||
overhead the job clock charges but no step owns (job init, step
|
||||
transitions, post-job cleanup). That headroom is what makes the test
|
||||
budget a floor rather than a hope, since setup cannot overrun into it
|
||||
without failing its own step first. GitHub expressions have no
|
||||
arithmetic, so the sum is passed in rather than computed.
|
||||
required: false
|
||||
type: number
|
||||
default: 60
|
||||
test-timeout-seconds:
|
||||
description: >-
|
||||
Per-test ceiling enforced by pytest-timeout, covering fixture setup and
|
||||
teardown as well as the test body. A test that hangs fails with a
|
||||
traceback of where it was stuck instead of idling the shard until
|
||||
`timeout-minutes` cancels it. Timed-out tests are excluded from reruns
|
||||
because pytest-timeout arms its timer once per test and
|
||||
pytest-rerunfailures reruns inside that same window, so a rerun of a
|
||||
timed-out test would run with no timer at all.
|
||||
required: false
|
||||
type: number
|
||||
default: 120
|
||||
max-failures:
|
||||
description: "Stop after this many failures"
|
||||
required: false
|
||||
type: number
|
||||
default: 10
|
||||
dist:
|
||||
description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
|
||||
required: false
|
||||
type: string
|
||||
default: "loadscope"
|
||||
artifact-name:
|
||||
description: "Unique name for the coverage artifact (must be unique per run)"
|
||||
required: true
|
||||
type: string
|
||||
rust-bridge-artifact:
|
||||
description: "Prebuilt editable Rust bridge artifact"
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
UV_PYTHON: ${{ inputs.python-version }}
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: Run tests
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: ${{ inputs.job-timeout-minutes }}
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
decision: ${{ steps.changes.outputs.decision }}
|
||||
has-coverage: ${{ steps.tests.outputs.has-coverage }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
timeout-minutes: 3
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
timeout-minutes: 2
|
||||
uses: ./.github/actions/detect-changes
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ env.UV_PYTHON }}
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/cache-uv-downloads
|
||||
|
||||
- name: Set up the Rust build
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 5
|
||||
uses: ./.github/actions/rust-bridge
|
||||
with:
|
||||
artifact: ${{ inputs.rust-bridge-artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 8
|
||||
env:
|
||||
RUST_BRIDGE_ARTIFACT: ${{ inputs.rust-bridge-artifact }}
|
||||
run: |
|
||||
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
||||
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime
|
||||
else
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project
|
||||
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
|
||||
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
|
||||
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
|
||||
fi
|
||||
uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: 3
|
||||
run: |
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Run tests
|
||||
id: tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
timeout-minutes: ${{ inputs.timeout-minutes }}
|
||||
env:
|
||||
TEST_PATH: ${{ inputs.test-path }}
|
||||
MAX_FAILURES: ${{ inputs.max-failures }}
|
||||
WORKERS: ${{ inputs.workers }}
|
||||
RERUNS: ${{ inputs.reruns }}
|
||||
TEST_TIMEOUT_SECONDS: ${{ inputs.test-timeout-seconds }}
|
||||
DIST: ${{ inputs.dist }}
|
||||
COVERAGE_CORE: sysmon
|
||||
run: |
|
||||
echo "has-coverage=false" >> "$GITHUB_OUTPUT"
|
||||
selection="${TEST_PATH}"
|
||||
if [ -z "${selection// /}" ]; then
|
||||
echo "shard selection is empty; nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
pytest_args=()
|
||||
existing_paths=0
|
||||
for token in ${selection}; do
|
||||
case "${token}" in
|
||||
-*) pytest_args+=("${token}") ;;
|
||||
*)
|
||||
if [ -e "${token%%::*}" ]; then
|
||||
pytest_args+=("${token}")
|
||||
existing_paths=$((existing_paths + 1))
|
||||
else
|
||||
echo "::warning::${token} does not exist; drop it from this shard's test-path"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [ "${existing_paths}" -eq 0 ]; then
|
||||
echo "No path in the selection exists (${selection}); nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
xdist_args=()
|
||||
if [ "${WORKERS}" != "0" ]; then
|
||||
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
|
||||
fi
|
||||
set +e
|
||||
uv run --no-sync pytest "${pytest_args[@]}" \
|
||||
--tb=short -vv \
|
||||
--maxfail="${MAX_FAILURES}" \
|
||||
"${xdist_args[@]}" \
|
||||
--reruns "${RERUNS}" \
|
||||
--reruns-delay 1 \
|
||||
--timeout="${TEST_TIMEOUT_SECONDS}" \
|
||||
--rerun-except "from pytest-timeout" \
|
||||
--durations=20 \
|
||||
--cov=./litellm --cov=./enterprise/litellm_enterprise \
|
||||
--cov-report=xml:coverage.xml \
|
||||
--cov-config=pyproject.toml
|
||||
status=$?
|
||||
set -e
|
||||
if [ -f coverage.xml ]; then
|
||||
echo "has-coverage=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ "$status" -eq 5 ]; then
|
||||
echo "pytest collected no tests from ${selection}; passing"
|
||||
exit 0
|
||||
fi
|
||||
exit "$status"
|
||||
|
||||
- name: Save coverage report
|
||||
if: always() && steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: coverage.xml
|
||||
retention-days: 1
|
||||
|
||||
upload-coverage:
|
||||
name: Upload coverage to Codecov
|
||||
needs: run
|
||||
if: always() && needs.run.outputs.decision != 'skip' && needs.run.outputs.has-coverage == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download coverage report
|
||||
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: coverage-reports
|
||||
merge-multiple: true
|
||||
|
||||
- name: Upload to Codecov
|
||||
id: codecov-upload
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
directory: coverage-reports
|
||||
root_dir: ${{ github.workspace }}
|
||||
flags: ${{ inputs.artifact-name }}
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Upload to Codecov (retry)
|
||||
if: steps.codecov-upload.outcome == 'failure'
|
||||
continue-on-error: true
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
directory: coverage-reports
|
||||
root_dir: ${{ github.workspace }}
|
||||
flags: ${{ inputs.artifact-name }}
|
||||
fail_ci_if_error: false
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
name: Updates model_prices_and_context_window.json and Create Pull Request
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Run every Sundays at midnight
|
||||
#- cron: "0 0 * * *" # Run daily at midnight
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
auto_update_price_and_context_window:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
|
||||
- name: Regenerate JSON Schema
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
|
||||
--base main
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
134
.github/workflows/check-ui-api-types.yml
vendored
|
|
@ -1,134 +0,0 @@
|
|||
name: Check UI API Types Sync
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check-sync:
|
||||
name: Verify schema.d.ts matches the proxy OpenAPI spec
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Detect changes that can affect the generated types
|
||||
id: changes
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
|
||||
echo "Not a pull request merge commit, running the full check."
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
files="$(git diff --name-only "$base" HEAD)"
|
||||
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No proxy, types or generator changes in this pull request, nothing to verify."
|
||||
echo "relevant=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install backend dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Regenerate the lazy OpenAPI snapshot
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
|
||||
|
||||
- name: Fail if the lazy OpenAPI snapshot is stale
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: |
|
||||
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
|
||||
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
|
||||
echo ""
|
||||
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
|
||||
echo "To fix, run from the repo root:"
|
||||
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
|
||||
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
|
||||
exit 1
|
||||
fi
|
||||
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dashboard dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
run: npm ci
|
||||
|
||||
- name: Regenerate types from the live spec
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
env:
|
||||
LITELLM_PYTHON: "uv run --no-sync python"
|
||||
run: npm run gen:api
|
||||
|
||||
- name: Fail if types are stale
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: |
|
||||
if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
|
||||
echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."
|
||||
echo ""
|
||||
echo "A backend route or model changed without regenerating the dashboard types."
|
||||
echo "To fix, run from ui/litellm-dashboard:"
|
||||
echo " npm run gen:api"
|
||||
echo "then commit the updated src/lib/http/schema.d.ts."
|
||||
exit 1
|
||||
fi
|
||||
echo "schema.d.ts is in sync with the proxy OpenAPI spec."
|
||||
48
.github/workflows/ci-coverage.yml
vendored
|
|
@ -1,48 +0,0 @@
|
|||
name: "CI Coverage"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
assert-ci-coverage:
|
||||
name: assert-ci-coverage
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Assert every test file and Dockerfile is invoked by a job
|
||||
run: |
|
||||
python -m pip install "pyyaml==6.0.3"
|
||||
python .github/scripts/assert_ci_coverage.py
|
||||
|
||||
# The census asks whether a job names a file; this asks whether that job's -k
|
||||
# then throws it back out. A file both globbed and deselected everywhere runs
|
||||
# nowhere while counting as covered, which is how the caching suite went unrun.
|
||||
- name: Assert no -k expression deselects a file from every job that globs it
|
||||
run: python .github/scripts/assert_ci_coverage.py --slices
|
||||
|
||||
- name: Assert .github/workflows/ holds only workflows, correctly named
|
||||
run: python .github/scripts/assert_workflow_dir_hygiene.py
|
||||
9
.github/workflows/codeql.yml
vendored
|
|
@ -1,8 +1,6 @@
|
|||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
|
|
@ -43,14 +41,15 @@ jobs:
|
|||
persist-credentials: false
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
|
||||
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
config-file: ./.github/codeql/codeql-config.yml
|
||||
queries: ${{ github.event_name == 'pull_request' && '+security-extended' || '' }}
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
output: sarif-results
|
||||
|
|
@ -83,7 +82,7 @@ jobs:
|
|||
output: sarif-results/python.sarif
|
||||
|
||||
- name: Upload SARIF
|
||||
uses: github/codeql-action/upload-sarif@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3.34.1
|
||||
uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
with:
|
||||
sarif_file: sarif-results
|
||||
category: "/language:${{ matrix.language }}"
|
||||
|
|
|
|||
2
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -40,4 +40,4 @@ jobs:
|
|||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
uv run --frozen python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
|
||||
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
|
||||
|
|
|
|||
30
.github/workflows/deploy-dev-on-main.yml
vendored
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
name: Deploy main to gateway-dev
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: deploy-dev-dispatch
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
dispatch:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2.2.2
|
||||
id: token
|
||||
with:
|
||||
app-id: ${{ secrets.LITELLM_OPS_DISPATCH_APP_ID }}
|
||||
private-key: ${{ secrets.LITELLM_OPS_DISPATCH_APP_PRIVATE_KEY }}
|
||||
owner: BerriAI
|
||||
repositories: litellm-ops
|
||||
permission-actions: write
|
||||
- name: Dispatch the litellm-ops dev deploy
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.token.outputs.token }}
|
||||
run: gh api --method POST repos/BerriAI/litellm-ops/actions/workflows/deploy-dev.yml/dispatches -f ref=main
|
||||
54
.github/workflows/helm_unit_test.yml
vendored
|
|
@ -1,54 +0,0 @@
|
|||
name: Helm unit test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
unit-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Helm 3.11.1
|
||||
uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
|
||||
with:
|
||||
version: "3.11.1"
|
||||
|
||||
- name: Download and verify Helm Unit Test Plugin
|
||||
run: |
|
||||
curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
|
||||
echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94 $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
|
||||
|
||||
- name: Install Helm Unit Test Plugin
|
||||
run: |
|
||||
PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
|
||||
mkdir -p "$PLUGIN_DIR"
|
||||
tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
|
||||
helm plugin list
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
for chart in helm/litellm-helm helm/litellm; do
|
||||
declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
|
||||
output="$(mktemp)"
|
||||
helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
|
||||
executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
|
||||
if [ "$declared" != "$executed" ]; then
|
||||
echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
|
||||
exit 1
|
||||
fi
|
||||
echo "$chart: all $declared declared test suites ran"
|
||||
done
|
||||
96
.github/workflows/image-scan.yml
vendored
|
|
@ -17,9 +17,8 @@ on:
|
|||
- gateway/routes/allowlist.py
|
||||
- backend/Dockerfile
|
||||
- backend/main.py
|
||||
- deploy/lens/**
|
||||
- litellm-rust/**
|
||||
- litellm/proxy/lens/**
|
||||
- tests/e2e/migrations/lens_compose_smoke.sh
|
||||
- docker/component_entrypoint.sh
|
||||
- docker/entrypoint.sh
|
||||
- litellm/proxy/prisma_migration.py
|
||||
|
|
@ -28,7 +27,7 @@ on:
|
|||
- backend/routes/allowlist.py
|
||||
- pyproject.toml
|
||||
- litellm-proxy-extras/**
|
||||
- tests/proxy_migration_tests/**
|
||||
- tests/integration/database/image/**
|
||||
- uv.lock
|
||||
- ui/litellm-dashboard/package-lock.json
|
||||
- ui/Dockerfile
|
||||
|
|
@ -46,80 +45,6 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lens-worker-image:
|
||||
name: lens-worker-image (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
if: >-
|
||||
github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: ubuntu-latest
|
||||
grype_sha256: edda0968d8827daab01d32b3cd7de192ae0915005e7bbfcfef9e68e79bc43343
|
||||
- arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
grype_sha256: 553e4c36d9d61349830ba6034d43b8700a7f10576d3e2f4981c0fd2b96086465
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Build the release worker
|
||||
env:
|
||||
RELEASE_TAG: sha-${{ github.sha }}
|
||||
run: docker build --build-arg LITELLM_RELEASE_TAG="${RELEASE_TAG}" -f deploy/lens/Dockerfile -t lens-worker-scan .
|
||||
- name: Verify the standalone worker on a read-only filesystem
|
||||
env:
|
||||
RELEASE_TAG: sha-${{ github.sha }}
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
|
||||
-e EXPECTED_RELEASE_TAG="${RELEASE_TAG}" --entrypoint python lens-worker-scan -c '
|
||||
import os
|
||||
import lens.worker
|
||||
from lens.release import release_tag
|
||||
from lens.trace_store import trace_store
|
||||
assert os.getuid() == 65532
|
||||
assert release_tag() == os.environ["EXPECTED_RELEASE_TAG"]
|
||||
with trace_store() as store:
|
||||
assert store.count() == 0
|
||||
'
|
||||
- name: Reject a dependency whose hash has changed
|
||||
run: |
|
||||
docker build --target builder -f deploy/lens/Dockerfile -t lens-worker-deps .
|
||||
sed -E 's/sha256:[0-9a-f]{64}/sha256:0000000000000000000000000000000000000000000000000000000000000000/g' \
|
||||
deploy/lens/requirements.lock > "$RUNNER_TEMP/tampered.lock"
|
||||
if docker run --rm -v "$RUNNER_TEMP/tampered.lock:/tmp/tampered.lock:ro" \
|
||||
--entrypoint uv lens-worker-deps pip sync --python /app/.venv/bin/python \
|
||||
--require-hashes --only-binary :all: --reinstall --no-cache /tmp/tampered.lock \
|
||||
> "$RUNNER_TEMP/hash-check.log" 2>&1; then
|
||||
echo "::error::Dependency hash mismatch was accepted"
|
||||
exit 1
|
||||
fi
|
||||
cat "$RUNNER_TEMP/hash-check.log"
|
||||
grep -qi 'hash mismatch' "$RUNNER_TEMP/hash-check.log"
|
||||
- name: Download Grype v0.114.0
|
||||
env:
|
||||
ARCH: ${{ matrix.arch }}
|
||||
GRYPE_SHA256: ${{ matrix.grype_sha256 }}
|
||||
run: |
|
||||
curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
|
||||
"https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_${ARCH}.tar.gz"
|
||||
echo "${GRYPE_SHA256} $RUNNER_TEMP/grype.tar.gz" | sha256sum -c -
|
||||
tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
|
||||
chmod +x "$RUNNER_TEMP/grype"
|
||||
- name: Scan the worker for fixable HIGH/CRITICAL CVEs
|
||||
env:
|
||||
GRYPE_MATCH_PYTHON_USING_CPES: "true"
|
||||
run: |
|
||||
"$RUNNER_TEMP/grype" lens-worker-scan \
|
||||
--config .grype.yaml --only-fixed --fail-on high --output table
|
||||
|
||||
image-scan:
|
||||
name: image-scan
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -167,7 +92,7 @@ jobs:
|
|||
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
|
||||
|
||||
# Scans the whole shipped artifact: OS/apk plus every language package
|
||||
# baked into the image, including ones no lockfile declares (e.g. prisma's
|
||||
|
|
@ -220,13 +145,8 @@ jobs:
|
|||
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
|
||||
|
||||
- name: Verify the bundled Lens Compose installation and restart
|
||||
if: matrix.dockerfile == 'Dockerfile'
|
||||
env:
|
||||
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
|
||||
run: bash tests/e2e/migrations/lens_compose_smoke.sh
|
||||
|
||||
migrations-image:
|
||||
name: migrations-image
|
||||
|
|
@ -257,7 +177,7 @@ jobs:
|
|||
LITELLM_MIGRATION_SCRIPT: /app/run.py
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v
|
||||
|
||||
gateway-image:
|
||||
name: gateway-image
|
||||
|
|
@ -288,7 +208,7 @@ jobs:
|
|||
LITELLM_IMAGE_COMPONENT: gateway
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
|
||||
|
||||
ui-image:
|
||||
name: ui-image
|
||||
|
|
@ -317,7 +237,7 @@ jobs:
|
|||
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v
|
||||
|
||||
backend-image:
|
||||
name: backend-image
|
||||
|
|
@ -352,4 +272,4 @@ jobs:
|
|||
LITELLM_IMAGE_COMPONENT: backend
|
||||
run: |
|
||||
python -m pip install "pytest==9.0.3"
|
||||
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py tests/proxy_migration_tests/test_image_admin_mcp.py -v
|
||||
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v
|
||||
|
|
|
|||
94
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
name: Interactions spec canary
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 7 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/interactions-spec-canary.yml
|
||||
- tests/unit/interactions/**
|
||||
- litellm/types/interactions/**
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency: interactions-spec-canary
|
||||
|
||||
jobs:
|
||||
interactions-spec-canary:
|
||||
name: interactions-spec-canary
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
compliance: ${{ steps.compliance.outcome }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
uses: ./.github/actions/rust-bridge
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
|
||||
|
||||
- name: Run the compliance tests against Google's live spec
|
||||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
needs: interactions-spec-canary
|
||||
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: File the tracking issue unless the compliance step passed, close it when it did
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
with:
|
||||
script: |
|
||||
const title = "Interactions spec canary is red";
|
||||
const outcome = process.env.COMPLIANCE_OUTCOME;
|
||||
const runUrl = process.env.RUN_URL;
|
||||
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
|
||||
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
|
||||
const open = found.items.find((issue) => issue.title === title);
|
||||
if (outcome === "success") {
|
||||
if (open === undefined) return;
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
|
||||
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
|
||||
return;
|
||||
}
|
||||
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
|
||||
if (open !== undefined) {
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.create({
|
||||
...context.repo,
|
||||
title,
|
||||
assignees: ["mateo-berri"],
|
||||
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
|
||||
});
|
||||
152
.github/workflows/lens-install-smoke.yml
vendored
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
name: Lens installation smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
lens_image:
|
||||
description: Verified baseline Lens image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
lens_upgrade_image:
|
||||
description: Compatible Lens upgrade image at ghcr.io/berriai/lens@sha256
|
||||
required: true
|
||||
type: string
|
||||
gateway_baseline_ref:
|
||||
description: Previously qualified gateway source commit, full SHA
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: lens-install-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
- component: ui
|
||||
dockerfile: ui/Dockerfile
|
||||
- component: migrations
|
||||
dockerfile: migrations/Dockerfile
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
- component: gateway
|
||||
dockerfile: gateway/Dockerfile
|
||||
baseline: true
|
||||
- component: backend
|
||||
dockerfile: backend/Dockerfile
|
||||
baseline: true
|
||||
- component: monolith
|
||||
dockerfile: Dockerfile
|
||||
baseline: true
|
||||
env:
|
||||
COMPONENT: ${{ matrix.component }}
|
||||
DOCKERFILE: ${{ matrix.dockerfile }}
|
||||
SOURCE_REF: ${{ matrix.baseline && inputs.gateway_baseline_ref || github.sha }}
|
||||
IMAGE_TAG: ${{ matrix.baseline && 'v0.0.0-lens-ci-baseline' || 'v0.0.0-lens-ci' }}
|
||||
CURRENT_REF: ${{ github.sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ env.SOURCE_REF }}
|
||||
- name: Verify the selected source
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_REF"
|
||||
if [[ "$IMAGE_TAG" == v0.0.0-lens-ci-baseline ]]; then
|
||||
test "$SOURCE_REF" != "$CURRENT_REF"
|
||||
fi
|
||||
- name: Build the matching release image
|
||||
run: |
|
||||
docker build --build-arg "LITELLM_RELEASE_TAG=$IMAGE_TAG" \
|
||||
-f "$DOCKERFILE" -t "lens-ci-$COMPONENT:$IMAGE_TAG" .
|
||||
- name: Save the matching release image
|
||||
run: |
|
||||
docker save "lens-ci-$COMPONENT:$IMAGE_TAG" \
|
||||
| gzip -1 > "$RUNNER_TEMP/lens-install-$COMPONENT-$IMAGE_TAG.tar.gz"
|
||||
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/lens-install-${{ matrix.component }}-${{ env.IMAGE_TAG }}.tar.gz
|
||||
compression-level: 0
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
|
||||
helm-install:
|
||||
needs: build-images
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
|
||||
with:
|
||||
pattern: lens-install-*-${{ github.sha }}
|
||||
merge-multiple: true
|
||||
path: ${{ runner.temp }}/lens-install-images
|
||||
- name: Load the matching release images
|
||||
run: |
|
||||
for archive in "$RUNNER_TEMP"/lens-install-images/lens-install-*.tar.gz; do
|
||||
gzip -dc "$archive" | docker load
|
||||
rm "$archive"
|
||||
done
|
||||
- uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
|
||||
with:
|
||||
cosign-release: v2.4.3
|
||||
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ vars.LENS_READ_USER || github.actor }}
|
||||
password: ${{ secrets.LENS_READ_TOKEN }}
|
||||
- name: Verify independent Lens images before loading them
|
||||
env:
|
||||
LENS_IMAGE: ${{ inputs.lens_image }}
|
||||
LENS_UPGRADE_IMAGE: ${{ inputs.lens_upgrade_image }}
|
||||
LENS_PUBLIC_KEY: ${{ vars.LENS_COSIGN_PUBLIC_KEY }}
|
||||
LENS_PUBLIC_KEY_SHA256: ${{ vars.LENS_COSIGN_PUBLIC_KEY_SHA256 }}
|
||||
shell: bash -euo pipefail {0}
|
||||
run: |
|
||||
[[ "$LENS_PUBLIC_KEY_SHA256" =~ ^[0-9a-f]{64}$ ]]
|
||||
printf '%s' "$LENS_PUBLIC_KEY" > "$RUNNER_TEMP/lens.pub"
|
||||
echo "$LENS_PUBLIC_KEY_SHA256 $RUNNER_TEMP/lens.pub" | sha256sum --check
|
||||
for image in "$LENS_IMAGE" "$LENS_UPGRADE_IMAGE"; do
|
||||
[[ "$image" =~ ^ghcr.io/berriai/lens@sha256:[0-9a-f]{64}$ ]]
|
||||
cosign verify --key "$RUNNER_TEMP/lens.pub" "$image" > /dev/null
|
||||
cosign verify-attestation --key "$RUNNER_TEMP/lens.pub" --type spdxjson "$image" > /dev/null
|
||||
docker pull "$image"
|
||||
done
|
||||
docker tag "$LENS_IMAGE" lens-ci-worker:baseline
|
||||
docker tag "$LENS_UPGRADE_IMAGE" lens-ci-worker:upgrade
|
||||
- name: Install pinned Kubernetes test tools
|
||||
run: |
|
||||
curl --fail --location --output "$RUNNER_TEMP/kind" \
|
||||
https://kind.sigs.k8s.io/dl/v0.27.0/kind-linux-amd64
|
||||
echo "a6875aaea358acf0ac07786b1a6755d08fd640f4c79b7a2e46681cc13f49a04b $RUNNER_TEMP/kind" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kind"
|
||||
curl --fail --location --output "$RUNNER_TEMP/kubectl" \
|
||||
https://dl.k8s.io/release/v1.32.2/bin/linux/amd64/kubectl
|
||||
echo "4f6a959dcc5b702135f8354cc7109b542a2933c46b808b248a214c1f69f817ea $RUNNER_TEMP/kubectl" | sha256sum --check
|
||||
chmod +x "$RUNNER_TEMP/kubectl"
|
||||
curl --fail --location --output "$RUNNER_TEMP/helm.tar.gz" \
|
||||
https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz
|
||||
echo "a7f81ce08007091b86d8bd696eb4d86b8d0f2e1b9f6c714be62f82f96a594496 $RUNNER_TEMP/helm.tar.gz" | sha256sum --check
|
||||
tar -xzf "$RUNNER_TEMP/helm.tar.gz" -C "$RUNNER_TEMP"
|
||||
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
|
||||
echo "$RUNNER_TEMP/linux-amd64" >> "$GITHUB_PATH"
|
||||
- name: Install, ingest, upgrade, and restart both charts
|
||||
run: bash tests/e2e/migrations/lens_helm_smoke.sh
|
||||
130
.github/workflows/lens-worker.yml
vendored
|
|
@ -1,130 +0,0 @@
|
|||
name: Lens Worker Image
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main, litellm_oss_branch, "litellm_**"]
|
||||
paths:
|
||||
- deploy/lens/**
|
||||
- litellm/proxy/lens/**
|
||||
- tests/proxy_behavior/lens/**
|
||||
- .github/workflows/lens-worker.yml
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- deploy/lens/**
|
||||
- litellm/proxy/lens/**
|
||||
- tests/proxy_behavior/lens/**
|
||||
- .github/workflows/lens-worker.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
lens-worker-image:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Build Lens worker
|
||||
run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
|
||||
- name: Reject custom builds without a matching release tag
|
||||
run: |
|
||||
if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
|
||||
echo "::error::An unversioned worker build unexpectedly succeeded"
|
||||
exit 1
|
||||
fi
|
||||
grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
|
||||
- name: Verify standalone imports with a read-only filesystem
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
|
||||
--security-opt no-new-privileges --entrypoint python \
|
||||
lens-worker:${{ github.sha }} -c '
|
||||
import os
|
||||
import lens.worker
|
||||
from lens.trace_store import trace_store
|
||||
assert os.getuid() == 65532
|
||||
with trace_store() as store:
|
||||
assert store.count() == 0
|
||||
'
|
||||
- name: Prepare test-only coverage tool
|
||||
run: |
|
||||
coverage_directory=$(mktemp -d "$RUNNER_TEMP/lens-coverage.XXXXXX")
|
||||
curl --fail --silent --show-error --location \
|
||||
https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl \
|
||||
--output "$coverage_directory/coverage.whl"
|
||||
printf '%s %s\n' 8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1 \
|
||||
"$coverage_directory/coverage.whl" | sha256sum --check
|
||||
chmod 777 "$coverage_directory"
|
||||
echo "LENS_COVERAGE_DIRECTORY=$coverage_directory" >> "$GITHUB_ENV"
|
||||
- name: Verify confined Python execution
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
|
||||
-v "$PWD/tests/proxy_behavior/lens/worker_python_smoke.py:/app/python_smoke.py:ro" \
|
||||
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
|
||||
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
|
||||
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
|
||||
--entrypoint python lens-worker:${{ github.sha }} \
|
||||
-m coverage run --data-file=/coverage/.coverage.python /app/python_smoke.py
|
||||
- name: Verify workspace investigation and live review output
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=1g --security-opt no-new-privileges \
|
||||
-v "$PWD/tests/proxy_behavior/lens/worker_context_smoke.py:/app/context_smoke.py:ro" \
|
||||
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
|
||||
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
|
||||
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
|
||||
--entrypoint python lens-worker:${{ github.sha }} \
|
||||
-m coverage run --data-file=/coverage/.coverage.context /app/context_smoke.py
|
||||
- name: Verify default workspace recovery after Python scratch storage fills
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
|
||||
-v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
|
||||
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
|
||||
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
|
||||
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
|
||||
--entrypoint python lens-worker:${{ github.sha }} \
|
||||
-m coverage run --data-file=/coverage/.coverage.storage /app/storage_smoke.py
|
||||
- name: Map native worker coverage to repository sources
|
||||
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
|
||||
run: |
|
||||
docker run --rm --network none --read-only --cap-drop ALL \
|
||||
--security-opt no-new-privileges -w /workspace \
|
||||
-v "$PWD/litellm/proxy/lens:/workspace/litellm/proxy/lens:ro" \
|
||||
-v "$PWD/tests/proxy_behavior/lens/coverage.ini:/coverage.ini:ro" \
|
||||
-v "$LENS_COVERAGE_DIRECTORY:/coverage" \
|
||||
-e PYTHONPATH=/coverage/coverage.whl -e COVERAGE_RCFILE=/coverage.ini \
|
||||
--entrypoint /bin/sh lens-worker:${{ github.sha }} \
|
||||
-c 'python -m coverage combine && python -m coverage xml'
|
||||
- name: Upload native worker coverage
|
||||
if: always() && env.LENS_COVERAGE_DIRECTORY != ''
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
files: ${{ env.LENS_COVERAGE_DIRECTORY }}/lens-worker.xml
|
||||
root_dir: ${{ github.workspace }}
|
||||
flags: lens-worker
|
||||
fail_ci_if_error: false
|
||||
- name: Publish versioned Lens worker
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm' && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REGISTRY_USER: ${{ github.actor }}
|
||||
IMAGE: ghcr.io/berriai/litellm-lens-worker-dev:sha-${{ github.sha }}
|
||||
run: |
|
||||
printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
|
||||
docker tag lens-worker:${{ github.sha }} "$IMAGE"
|
||||
docker push "$IMAGE"
|
||||
printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
name: Publish basedpyright base counts
|
||||
|
||||
# Every commit on main can become a future merge-base.
|
||||
# Publishing its per-rule basedpyright counts as an artifact lets
|
||||
# scripts/type_check_gate.py download them in seconds instead of paying a
|
||||
# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
|
||||
# No concurrency group on purpose: runs must never cancel each other, because
|
||||
# every sha's artifact matters (any of them can become a merge-base).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
clean: true
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
# The gate provisions its own measurement env (.venv-typecheck: a frozen
|
||||
# uv sync of its canonical dependency groups plus a generated Prisma
|
||||
# client), so no install step here can drift from what local runs measure.
|
||||
- name: Emit basedpyright counts for HEAD
|
||||
run: |
|
||||
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
|
||||
counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
|
||||
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload counts artifact
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: ${{ env.COUNTS_ARTIFACT_NAME }}
|
||||
path: ${{ runner.temp }}/basedpyright-counts/
|
||||
if-no-files-found: error
|
||||
106
.github/workflows/publish-lint-base-counts.yml
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
name: Publish lint base counts
|
||||
|
||||
# Every commit on main can become a future merge-base.
|
||||
# Publishing its per-rule counts for each lint gate (strict ruff, type discipline,
|
||||
# test quality, basedpyright) as an artifact lets the gates download them through
|
||||
# scripts/lint_base_counts.py in seconds instead of scanning the merge-base tree in
|
||||
# a throwaway worktree on every fresh checkout or moved merge-base.
|
||||
# No concurrency group on purpose: runs must never cancel each other, because
|
||||
# every sha's artifact matters (any of them can become a merge-base).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)"
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- checker: ruff-strict
|
||||
script: scripts/ruff_strict_gate.py
|
||||
- checker: type-discipline
|
||||
script: scripts/type_discipline_gate.py
|
||||
- checker: test-quality
|
||||
script: scripts/test_quality_gate.py
|
||||
- checker: openapi-docs
|
||||
script: scripts/openapi_docs_gate.py
|
||||
- checker: basedpyright
|
||||
script: scripts/type_check_gate.py
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
clean: true
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: matrix.checker == 'basedpyright'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: matrix.checker == 'basedpyright' || matrix.checker == 'openapi-docs'
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
# The source scanners only need the pinned dev tools (ruff and the
|
||||
# stdlib checkers), the same versions test-linting.yml's python job runs.
|
||||
- name: Install the dev tools
|
||||
if: matrix.checker != 'basedpyright' && matrix.checker != 'openapi-docs'
|
||||
run: |
|
||||
uv sync --frozen --only-group dev --no-install-project
|
||||
|
||||
# The OpenAPI gate imports the proxy app, so it needs the same install the
|
||||
# test-linting.yml python job uses, Prisma client included.
|
||||
- name: Install the proxy
|
||||
if: matrix.checker == 'openapi-docs'
|
||||
run: |
|
||||
uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Emit ${{ matrix.checker }} counts for HEAD
|
||||
if: matrix.checker != 'basedpyright'
|
||||
run: |
|
||||
uv run --no-sync python ${{ matrix.script }} --emit-counts-dir "$RUNNER_TEMP/lint-counts"
|
||||
|
||||
# The basedpyright gate provisions its own measurement env (.venv-typecheck:
|
||||
# a frozen uv sync of its canonical dependency groups plus a generated Prisma
|
||||
# client), so no install step here can drift from what local runs measure.
|
||||
- name: Emit basedpyright counts for HEAD
|
||||
if: matrix.checker == 'basedpyright'
|
||||
run: |
|
||||
python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/lint-counts"
|
||||
|
||||
- name: Name the counts artifact
|
||||
run: |
|
||||
counts_file=$(ls "$RUNNER_TEMP"/lint-counts/${{ matrix.checker }}-counts-*.json)
|
||||
echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload counts artifact
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
with:
|
||||
name: ${{ env.COUNTS_ARTIFACT_NAME }}
|
||||
path: ${{ runner.temp }}/lint-counts/
|
||||
if-no-files-found: error
|
||||
49
.github/workflows/spend-logs-guard.yml
vendored
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
name: Spend logs guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
|
||||
pull_request_target:
|
||||
branches:
|
||||
- main
|
||||
pull_request_review:
|
||||
types:
|
||||
- submitted
|
||||
- dismissed
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
spend-logs-guard:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: |
|
||||
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
|
||||
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Run the guard
|
||||
env:
|
||||
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
uv run --no-project --with pydantic==2.14.1 python ci_cd/spend_logs_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --approver kerry-berri --reviews "$RUNNER_TEMP/reviews.json"
|
||||
70
.github/workflows/sync-together-ai-models.yml
vendored
|
|
@ -1,70 +0,0 @@
|
|||
name: Sync Together AI model registry
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 6 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
sync_together_ai_models:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
BASE_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ env.BASE_BRANCH }}
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Look for an already-open sync PR
|
||||
id: existing
|
||||
run: |
|
||||
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
|
||||
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
|
||||
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "$open_pr" ]; then
|
||||
echo "An open sync PR already exists on branch $open_pr; skipping this run."
|
||||
fi
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
- name: Run the sync
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
|
||||
env:
|
||||
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
||||
- name: Regenerate the JSON schema
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
uv run --frozen python ci_cd/generate_model_prices_schema.py
|
||||
- name: Create a pull request when the registry changed
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
if git diff --quiet; then
|
||||
echo "Registry already in sync; no PR needed."
|
||||
exit 0
|
||||
fi
|
||||
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$branch"
|
||||
git add model_prices_and_context_window.json \
|
||||
litellm/model_prices_and_context_window_backup.json \
|
||||
model_prices_and_context_window.schema.json
|
||||
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
|
||||
gh auth setup-git
|
||||
git push origin "$branch"
|
||||
gh pr create --title "feat(models): sync together_ai model registry" \
|
||||
--body-file "$RUNNER_TEMP/pr_body.md" \
|
||||
--head "$branch" \
|
||||
--base "$BASE_BRANCH"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
220
.github/workflows/test-code-quality.yml
vendored
|
|
@ -1,220 +0,0 @@
|
|||
name: Code Quality Checks
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
code-quality:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
repository: BerriAI/litellm-docs
|
||||
path: docs/my-website
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: github.ref == 'refs/heads/main'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: github.ref != 'refs/heads/main'
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --frozen --all-groups --all-extras
|
||||
|
||||
- name: check_licenses
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
|
||||
|
||||
- name: check_provider_folders_documented
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
|
||||
|
||||
- name: check_prisma_binary_cache
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
|
||||
|
||||
- name: check_workflow_startup_safety
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
|
||||
|
||||
- name: check_workflow_job_name_collisions
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
|
||||
|
||||
- name: test_workflow_job_name_collisions
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
|
||||
|
||||
- name: test_e2e_changed_gate
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
|
||||
|
||||
- name: test_e2e_metadata
|
||||
env:
|
||||
PYTHONPATH: tests/e2e
|
||||
run: |
|
||||
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
|
||||
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
|
||||
|
||||
- name: Check merge smoke harness
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
|
||||
|
||||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
- name: test_chat_completion_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
|
||||
- name: info_log_check
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
|
||||
|
||||
- name: check_guardrail_apply_decorator
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
|
||||
|
||||
- name: test_ban_set_verbose
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
|
||||
|
||||
- name: code_qa_check_tests
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
|
||||
|
||||
- name: check_get_model_cost_key_performance
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
||||
|
||||
- name: test_proxy_types_import
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
|
||||
|
||||
- name: callback_manager_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
|
||||
|
||||
- name: recursive_detector
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
|
||||
|
||||
- name: test_router_strategy_async
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
|
||||
|
||||
- name: litellm_logging_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
|
||||
|
||||
- name: ensure_async_clients_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
|
||||
|
||||
- name: enforce_llms_folder_style
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
|
||||
|
||||
- name: prevent_key_leaks_in_exceptions
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
|
||||
|
||||
- name: check_unsafe_enterprise_import
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
|
||||
|
||||
- name: ban_copy_deepcopy_kwargs
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
|
||||
|
||||
- name: check_fastuuid_usage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
|
||||
|
||||
- name: check_py310_typing_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py
|
||||
|
||||
- name: check_e2e_no_raw_requests
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
|
||||
|
||||
- name: check_migrations_no_data_rewrites
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
- name: test_check_no_publicly_known_master_key
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
|
||||
|
||||
- name: check_unbounded_in_lists (fails on findings not in the baseline)
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
|
||||
|
||||
- name: memory_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
|
||||
|
||||
- name: documentation_test_env_keys
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
|
||||
|
||||
- name: documentation_test_router_settings
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
|
||||
|
||||
- name: documentation_test_api_docs
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
|
||||
|
||||
python-310-import-smoke:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --frozen --extra proxy --extra cli --python 3.10
|
||||
|
||||
- run: uv run --no-sync python --version
|
||||
|
||||
- name: Import litellm
|
||||
run: uv run --no-sync python -c "import litellm"
|
||||
|
||||
- name: Check litellm CLI
|
||||
run: uv run --no-sync litellm --version
|
||||
|
||||
- name: Check lite CLI
|
||||
run: uv run --no-sync lite version
|
||||
4
.github/workflows/test-e2e-redis-chaos.yml
vendored
|
|
@ -91,12 +91,12 @@ jobs:
|
|||
|
||||
- name: Run the Redis chaos load test
|
||||
env:
|
||||
E2E_REDIS_CHAOS: "1"
|
||||
E2E_PERF_TESTS: "1"
|
||||
LITELLM_PROXY_URL: http://localhost:4000
|
||||
REDIS_HOST: 127.0.0.1
|
||||
REDIS_PORT: "6379"
|
||||
run: |
|
||||
uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s
|
||||
uv run --no-sync pytest tests/e2e/perf/test_redis_chaos_e2e.py -v --tb=short -rA -s
|
||||
|
||||
- name: Show proxy log on failure
|
||||
if: failure()
|
||||
|
|
|
|||
559
.github/workflows/test-linting.yml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: LiteLLM Linting
|
||||
name: lint
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
|
@ -14,22 +14,26 @@ concurrency:
|
|||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
# actions: read lets scripts/type_check_gate.py download the base-counts
|
||||
# artifact published by publish-basedpyright-base-counts.yml instead of
|
||||
# re-running basedpyright over the merge-base tree.
|
||||
python:
|
||||
name: python
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
actions: read
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
services:
|
||||
redis:
|
||||
image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
# Check out the PR head, not the default refs/pull/N/merge: the merge ref
|
||||
# folds in newer base commits, which the diff-based gates (ruff delta,
|
||||
# Any-discipline) would otherwise blame on this branch.
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 1
|
||||
|
|
@ -100,9 +104,6 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
# basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
|
||||
# only after `prisma generate` writes prisma/client.py et al. Without this the
|
||||
# DB wrappers typed against the generated client would degrade to Unknown.
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
|
|
@ -144,27 +145,41 @@ jobs:
|
|||
run: |
|
||||
uv run --no-sync ruff check --config ruff-tests.toml tests
|
||||
|
||||
- name: Check strict-rule budget (delta vs base)
|
||||
- name: Check strict ruff rules (delta vs merge-base counts)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
|
||||
|
||||
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
|
||||
- name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
|
||||
|
||||
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
|
||||
- name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
|
||||
|
||||
- name: Check OpenAPI docs (POST routes without a request schema, description or 200 schema, delta vs merge-base counts)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
uv sync --frozen --group proxy-dev --group e2e-dev --extra proxy
|
||||
uv run --no-sync python scripts/openapi_docs_gate.py --base "$GATE_BASE_SHA"
|
||||
|
||||
- name: Print OpenAI version
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
|
||||
|
||||
- name: Check basedpyright budget (delta vs base)
|
||||
- name: Check basedpyright (delta vs merge-base counts)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
|
@ -174,23 +189,29 @@ jobs:
|
|||
- name: Check tests/e2e basedpyright (zero errors)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' | grep -q .; then
|
||||
uv run --no-sync basedpyright tests/e2e
|
||||
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' ':(glob)tests/e2e_harness/**/*.py' pyrightconfig.json | grep -q .; then
|
||||
uv run --no-sync basedpyright tests/e2e tests/e2e_harness
|
||||
else
|
||||
echo "No changed tests/e2e Python files; skipping."
|
||||
fi
|
||||
|
||||
- name: Run the claude_code harness unit tests
|
||||
- name: Run the e2e harness tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
LITELLM_MASTER_KEY: sk-e2e-harness-tests-reach-no-proxy
|
||||
E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0
|
||||
run: |
|
||||
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/claude_code/**/*.py' ':(glob)tests/e2e/*.py' tests/e2e/claude_code/cron_vm/install_claude_code.sh pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
|
||||
echo "No changed claude_code harness files; skipping."
|
||||
if ! git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- tests/e2e tests/e2e_harness ':(exclude)tests/e2e/ui' tests/code_coverage_tests/test_provider_replay_harness.py tests/code_coverage_tests/test_provider_cache.py pyproject.toml uv.lock .github/workflows/test-linting.yml | grep -q .; then
|
||||
echo "No changed e2e harness files; skipping."
|
||||
exit 0
|
||||
fi
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
CLAUDE_VERSION="$(retry uv run --no-sync python tests/e2e/claude_code/pr_gate_version_resolver.py)"
|
||||
tests/e2e/claude_code/cron_vm/install_claude_code.sh "$CLAUDE_VERSION" "$RUNNER_TEMP/claude-cli"
|
||||
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures tests/e2e/claude_code/_*_unit_tests
|
||||
PATH="$RUNNER_TEMP/claude-cli:$PATH" uv run --no-sync pytest -q tests/e2e_harness
|
||||
uv run --no-sync pytest --tb=short -q --noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures \
|
||||
tests/code_coverage_tests/test_provider_replay_harness.py \
|
||||
tests/code_coverage_tests/test_provider_cache.py
|
||||
|
||||
- name: Check for circular imports
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
|
|
@ -203,47 +224,11 @@ jobs:
|
|||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
||||
|
||||
# Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is
|
||||
# raised (or a rule/budget is dropped) so a loosening is obvious in review, but it
|
||||
# must be kept OUT of the branch-protection required-checks list so a justified
|
||||
# bump can still be merged by a human who has seen and accepted the red.
|
||||
budget-ratchet:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch ratchet base
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
|
||||
retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Ratchet check (budgets may only decrease; non-gating)
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
run: |
|
||||
python scripts/budget_ratchet_check.py --base "$BASE_SHA"
|
||||
|
||||
secret-scan:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
|
|
@ -275,3 +260,453 @@ jobs:
|
|||
else
|
||||
echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
|
||||
fi
|
||||
ui:
|
||||
name: ui
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui/litellm-dashboard
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Collect changed files
|
||||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
: > "$RUNNER_TEMP/eslint_files.txt"
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
case "$f" in
|
||||
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
|
||||
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
|
||||
esac
|
||||
done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
|
||||
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
|
||||
echo "has_files=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_files=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No lintable UI files changed in this PR; nothing to check."
|
||||
fi
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Lint changed files (prettier + eslint)
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: |
|
||||
prettier_files=()
|
||||
eslint_files=()
|
||||
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
|
||||
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
|
||||
status=0
|
||||
if [ ${#prettier_files[@]} -gt 0 ]; then
|
||||
echo "::group::Prettier (${#prettier_files[@]} files)"
|
||||
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
if [ ${#eslint_files[@]} -gt 0 ]; then
|
||||
echo "::group::ESLint (${#eslint_files[@]} files)"
|
||||
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
exit $status
|
||||
|
||||
- name: Check lint budgets
|
||||
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
|
||||
run: |
|
||||
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
|
||||
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
|
||||
|
||||
- name: Check for dead code (knip)
|
||||
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
|
||||
run: npm run knip:ci
|
||||
ui-api-types:
|
||||
name: ui-api-types
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Detect changes that can affect the generated types
|
||||
id: changes
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
|
||||
echo "Not a pull request merge commit, running the full check."
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
files="$(git diff --name-only "$base" HEAD)"
|
||||
if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/test-linting\.yml$)' <<< "$files"; then
|
||||
echo "relevant=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No proxy, types or generator changes in this pull request, nothing to verify."
|
||||
echo "relevant=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install backend dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
- name: Regenerate the lazy OpenAPI snapshot
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
|
||||
|
||||
- name: Fail if the lazy OpenAPI snapshot is stale
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: |
|
||||
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
|
||||
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
|
||||
echo ""
|
||||
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
|
||||
echo "To fix, run from the repo root:"
|
||||
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
|
||||
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
|
||||
exit 1
|
||||
fi
|
||||
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dashboard dependencies
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
run: npm ci
|
||||
|
||||
- name: Regenerate types from the live spec
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
working-directory: ui/litellm-dashboard
|
||||
env:
|
||||
LITELLM_PYTHON: "uv run --no-sync python"
|
||||
run: npm run gen:api
|
||||
|
||||
- name: Fail if types are stale
|
||||
if: steps.changes.outputs.relevant == 'true'
|
||||
run: |
|
||||
if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
|
||||
echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."
|
||||
echo ""
|
||||
echo "A backend route or model changed without regenerating the dashboard types."
|
||||
echo "To fix, run from ui/litellm-dashboard:"
|
||||
echo " npm run gen:api"
|
||||
echo "then commit the updated src/lib/http/schema.d.ts."
|
||||
exit 1
|
||||
fi
|
||||
echo "schema.d.ts is in sync with the proxy OpenAPI spec."
|
||||
code-quality:
|
||||
name: code-quality
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
repository: BerriAI/litellm-docs
|
||||
path: docs/my-website
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --frozen --all-groups --all-extras
|
||||
|
||||
- name: check_licenses
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
|
||||
|
||||
- name: check_provider_folders_documented
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
|
||||
|
||||
- name: check_provider_endpoints_support_generated
|
||||
run: uv run --no-sync python scripts/generate_provider_endpoints_support.py --check
|
||||
|
||||
- name: test_generate_provider_endpoints_support
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_generate_provider_endpoints_support.py
|
||||
|
||||
- name: check_prisma_binary_cache
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
|
||||
|
||||
- name: check_workflow_startup_safety
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
|
||||
|
||||
- name: check_workflow_job_name_collisions
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
|
||||
|
||||
- name: test_workflow_job_name_collisions
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
|
||||
|
||||
- name: test_unit_passed_gate
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_unit_passed_gate.py
|
||||
|
||||
- name: test_e2e_idp_stack
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_idp_stack.py
|
||||
|
||||
- name: test_e2e_metadata
|
||||
env:
|
||||
PYTHONPATH: tests/e2e
|
||||
run: |
|
||||
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 16)"
|
||||
uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
|
||||
|
||||
- name: Check merge smoke harness
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
|
||||
|
||||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
- name: test_chat_completion_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
|
||||
- name: info_log_check
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
|
||||
|
||||
- name: check_guardrail_apply_decorator
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
|
||||
|
||||
- name: test_ban_set_verbose
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
|
||||
|
||||
- name: code_qa_check_tests
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
|
||||
|
||||
- name: check_get_model_cost_key_performance
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
|
||||
|
||||
- name: test_proxy_types_import
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
|
||||
|
||||
- name: callback_manager_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
|
||||
|
||||
- name: recursive_detector
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
|
||||
|
||||
- name: test_router_strategy_async
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
|
||||
|
||||
- name: litellm_logging_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
|
||||
|
||||
- name: ensure_async_clients_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
|
||||
|
||||
- name: enforce_llms_folder_style
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
|
||||
|
||||
- name: prevent_key_leaks_in_exceptions
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
|
||||
|
||||
- name: check_unsafe_enterprise_import
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
|
||||
|
||||
- name: check_telemetry_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_telemetry_imports.py
|
||||
|
||||
- name: ban_copy_deepcopy_kwargs
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
|
||||
|
||||
- name: check_fastuuid_usage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
|
||||
|
||||
- name: check_py310_typing_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_py310_typing_imports.py
|
||||
|
||||
- name: check_e2e_no_raw_requests
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
|
||||
|
||||
- name: check_migrations_no_data_rewrites
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
|
||||
|
||||
- name: check_migrations_lock_timeout
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
|
||||
|
||||
- name: check_no_publicly_known_master_key
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
|
||||
|
||||
- name: test_check_no_publicly_known_master_key
|
||||
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_publicly_known_master_key.py
|
||||
|
||||
- name: check_unbounded_in_lists (fails on findings not in the baseline)
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/check_unbounded_in_lists.py
|
||||
|
||||
- name: memory_test
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
|
||||
|
||||
- name: documentation_test_env_keys
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
|
||||
|
||||
- name: documentation_test_router_settings
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
|
||||
|
||||
- name: documentation_test_api_docs
|
||||
run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
|
||||
ci-coverage:
|
||||
name: ci-coverage
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Assert every test file and Dockerfile is invoked by a job
|
||||
run: |
|
||||
python -m pip install "pyyaml==6.0.3"
|
||||
python .github/scripts/assert_ci_coverage.py
|
||||
|
||||
- name: Assert no -k expression deselects a file from every job that globs it
|
||||
run: python .github/scripts/assert_ci_coverage.py --slices
|
||||
|
||||
- name: Assert .github/workflows/ holds only workflows, correctly named
|
||||
run: python .github/scripts/assert_workflow_dir_hygiene.py
|
||||
semgrep:
|
||||
name: semgrep
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Run Semgrep (custom rules)
|
||||
run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error
|
||||
lint-passed:
|
||||
name: lint passed
|
||||
needs: [python, secret-scan, ui, ui-api-types, code-quality, ci-coverage, semgrep]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Require every lint job to succeed
|
||||
env:
|
||||
NEEDS: ${{ toJSON(needs) }}
|
||||
run: |
|
||||
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
|
||||
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null
|
||||
|
|
|
|||
40
.github/workflows/test-litellm-ui-build.yml
vendored
|
|
@ -1,40 +0,0 @@
|
|||
name: UI Build Check
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
build-ui:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
uses: ./.github/actions/detect-changes
|
||||
with:
|
||||
category: ui
|
||||
|
||||
# Built through the image stage rather than the checkout, because the
|
||||
# stage copies ui/litellm-dashboard/ alone: an import reaching above the
|
||||
# dashboard root resolves in a checkout and fails in every image we ship.
|
||||
# Dockerfile, docker/Dockerfile.non_root and ui/Dockerfile share this
|
||||
# stage verbatim, so building one covers all three.
|
||||
- name: Build the dashboard as the shipped images build it
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: docker build --target ui-builder -f Dockerfile .
|
||||
105
.github/workflows/test-litellm-ui-lint.yml
vendored
|
|
@ -1,105 +0,0 @@
|
|||
name: UI Lint
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
frontend-lint:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui/litellm-dashboard
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Collect changed files
|
||||
id: changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
# base.sha is the base branch tip from when the PR was opened, while
|
||||
# actions/checkout leaves HEAD on a merge of the PR into the *current*
|
||||
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
|
||||
# landed since, so a PR that touches no UI file still gets linted
|
||||
# against hundreds of other people's files. Diff the PR head against its
|
||||
# own merge base instead, which is exactly what this PR changed.
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
: > "$RUNNER_TEMP/prettier_files.txt"
|
||||
: > "$RUNNER_TEMP/eslint_files.txt"
|
||||
while IFS= read -r f; do
|
||||
[ -f "$f" ] || continue
|
||||
case "$f" in
|
||||
*.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt"
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;;
|
||||
*.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html)
|
||||
printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;;
|
||||
esac
|
||||
done < <(git diff --name-only --diff-filter=ACMR --relative "$merge_base" "$HEAD_SHA" -- .)
|
||||
if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then
|
||||
echo "has_files=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_files=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No lintable UI files changed in this PR; nothing to check."
|
||||
fi
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Lint changed files (prettier + eslint)
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: |
|
||||
prettier_files=()
|
||||
eslint_files=()
|
||||
while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt"
|
||||
while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt"
|
||||
status=0
|
||||
if [ ${#prettier_files[@]} -gt 0 ]; then
|
||||
echo "::group::Prettier (${#prettier_files[@]} files)"
|
||||
npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; }
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
if [ ${#eslint_files[@]} -gt 0 ]; then
|
||||
echo "::group::ESLint (${#eslint_files[@]} files)"
|
||||
npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
exit $status
|
||||
|
||||
- name: Check lint budgets
|
||||
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
|
||||
run: |
|
||||
npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true
|
||||
node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json
|
||||
|
||||
- name: Check for dead code (knip)
|
||||
if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }}
|
||||
run: npm run knip:ci
|
||||
99
.github/workflows/test-litellm-ui-unit.yml
vendored
|
|
@ -1,99 +0,0 @@
|
|||
name: UI Unit Tests
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ui-unit-tests:
|
||||
runs-on: ubuntu-latest-16-cores
|
||||
timeout-minutes: 20
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui/litellm-dashboard
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
uses: ./.github/actions/detect-changes
|
||||
with:
|
||||
category: ui
|
||||
|
||||
- name: Setup Node.js
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
cache: "npm"
|
||||
cache-dependency-path: ui/litellm-dashboard/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: npm ci
|
||||
|
||||
- name: Check UI production source types
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Run UI type tests (Vitest)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
CI: "true"
|
||||
run: npm run test:types
|
||||
|
||||
- name: Run UI unit tests (Vitest)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
env:
|
||||
CI: "true"
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
|
||||
|
||||
if [ -z "$BASE_SHA" ]; then
|
||||
echo "Push to $GITHUB_REF_NAME: running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
|
||||
test -n "$merge_base"
|
||||
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
|
||||
changed_files=()
|
||||
while IFS= read -r f; do
|
||||
changed_files+=("$f")
|
||||
done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
|
||||
if [ ${#changed_files[@]} -eq 0 ]; then
|
||||
echo "No UI files changed in this PR; skipping unit tests."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
scope=$(printf '%s\n' "${changed_files[@]}" | bash "$GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh")
|
||||
if [ "$scope" != related ]; then
|
||||
echo "Pull request: ${#changed_files[@]} changed UI files reach outside src/, so related would miss their dependents; running the full suite"
|
||||
full_suite
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
|
||||
npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
|
||||
--pool forks --maxWorkers=14
|
||||
|
|
@ -13,6 +13,9 @@ on:
|
|||
- "rust-toolchain.toml"
|
||||
- "litellm-rust/**"
|
||||
- "litellm/__init__.py"
|
||||
- "litellm/_version.py"
|
||||
- "litellm/_lazy_imports_registry.py"
|
||||
- "scripts/build_core_distribution.py"
|
||||
- "litellm/proxy/proxy_server.py"
|
||||
- "litellm/**/*mcp*"
|
||||
- "litellm/**/*mcp*/**"
|
||||
|
|
@ -42,7 +45,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
@ -86,6 +89,7 @@ jobs:
|
|||
uv pip check --python ".venv-$extra"
|
||||
if [ "$extra" = core ]; then
|
||||
checker=("$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py")
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-$extra/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
|
||||
else
|
||||
checker=("$GITHUB_WORKSPACE/scripts/check_mcp_sdk_install.py" --extra "$extra")
|
||||
fi
|
||||
|
|
@ -115,3 +119,63 @@ jobs:
|
|||
fi
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-lowest-$extra/bin/python" "${checker[@]}")
|
||||
done
|
||||
|
||||
core-distribution:
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
UV_PYTHON: ${{ matrix.python-version }}
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- uses: ./.github/actions/cache-cargo-build
|
||||
- name: Build the core wheel and source distribution
|
||||
run: >-
|
||||
uv run --no-project --with coverage==7.14.0 python -m coverage run --rcfile=/dev/null
|
||||
--branch --include="*/scripts/build_core_distribution.py"
|
||||
scripts/build_core_distribution.py --out-dir dist/core
|
||||
- name: Verify metadata, resources, and independent source rebuild
|
||||
env:
|
||||
CORE_DISTRIBUTION_DIR: ${{ github.workspace }}/dist/core
|
||||
run: >-
|
||||
uv run --no-project --with pytest==9.0.3 --with pytest-cov==5.0.0 --with coverage==7.14.0 --with 'tomli==2.4.1; python_version < "3.11"'
|
||||
python -m pytest tests/base_sdk_tests/test_core_distribution.py -v
|
||||
--cov-config=/dev/null --cov=scripts.build_core_distribution --cov-branch --cov-append --cov-report=
|
||||
- name: Verify isolated core installations
|
||||
run: |
|
||||
wheel=$(realpath dist/core/litellm_core-*.whl)
|
||||
for resolution in highest lowest-direct; do
|
||||
uv venv --python ${{ matrix.python-version }} ".venv-core-$resolution"
|
||||
uv pip install --python ".venv-core-$resolution" --resolution "$resolution" "$wheel"
|
||||
uv pip check --python ".venv-core-$resolution"
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile core)
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-$resolution/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
|
||||
done
|
||||
uv pip install --python .venv-core-highest coverage==7.14.0
|
||||
.venv-core-highest/bin/python -I -m coverage run --rcfile=/dev/null --append --branch --include="*/tests/base_sdk_tests/check_base_sdk_install.py" tests/base_sdk_tests/check_base_sdk_install.py --profile core
|
||||
uv pip install --python .venv-core-highest boto3 tokenizers huggingface-hub
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_base_sdk_install.py" --profile dependencies)
|
||||
(cd "$RUNNER_TEMP" && "$GITHUB_WORKSPACE/.venv-core-highest/bin/python" -I "$GITHUB_WORKSPACE/tests/base_sdk_tests/check_sdk_http.py")
|
||||
uv run --no-project --with coverage==7.14.0 python -m coverage xml --rcfile=/dev/null -o coverage-core.xml
|
||||
- name: Upload core packaging coverage
|
||||
uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5
|
||||
with:
|
||||
use_oidc: true
|
||||
files: coverage-core.xml
|
||||
flags: core-packaging
|
||||
fail_ci_if_error: false
|
||||
|
|
|
|||
180
.github/workflows/test-mcp-oauth-e2e.yml
vendored
|
|
@ -1,180 +0,0 @@
|
|||
name: MCP OAuth happy path
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/test-mcp-oauth-e2e.yml'
|
||||
- '.github/e2e-stack/**'
|
||||
- 'tests/e2e/*.py'
|
||||
- 'tests/e2e/pytest.ini'
|
||||
- 'tests/e2e/idp_realm.json'
|
||||
- 'tests/e2e/mcp/**'
|
||||
- 'litellm/experimental_mcp_client/**'
|
||||
- 'litellm/proxy/_experimental/mcp_server/**'
|
||||
- 'litellm/proxy/auth/**'
|
||||
- 'litellm/proxy/management_endpoints/mcp_management_endpoints.py'
|
||||
- 'litellm/proxy/_types.py'
|
||||
- 'litellm/types/mcp_server/mcp_server_manager.py'
|
||||
- 'litellm/proxy/management_endpoints/*sso*.py'
|
||||
- 'litellm/proxy/management_endpoints/sso/**'
|
||||
- 'litellm/proxy/common_utils/encrypt_decrypt_utils.py'
|
||||
- 'litellm/proxy/proxy_server.py'
|
||||
- 'litellm/proxy/schema.prisma'
|
||||
- 'ui/litellm-dashboard/src/app/connect/**'
|
||||
- 'ui/litellm-dashboard/src/app/mcp/oauth/**'
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: mcp-oauth-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
oauth:
|
||||
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
environment: e2e-changed
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6
|
||||
env:
|
||||
POSTGRES_USER: litellm
|
||||
POSTGRES_PASSWORD: dbpassword9090
|
||||
POSTGRES_DB: litellm
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U litellm"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
env:
|
||||
DATABASE_HOST: 127.0.0.1
|
||||
DATABASE_PORT: '5432'
|
||||
DATABASE_USER: litellm
|
||||
DATABASE_PASSWORD: dbpassword9090
|
||||
DATABASE_NAME: litellm
|
||||
DATABASE_URL: postgresql://litellm:dbpassword9090@127.0.0.1:5432/litellm
|
||||
E2E_KEYCLOAK_URL: http://127.0.0.1:8081
|
||||
E2E_KEYCLOAK_ADMIN_USER: admin
|
||||
E2E_KEYCLOAK_ADMIN_PASSWORD: e2e-ephemeral-idp-not-a-secret
|
||||
E2E_FIXTURE_MODE: live
|
||||
E2E_PROVIDER_CACHE: '0'
|
||||
E2E_MCP_OAUTH_LIVE: '1'
|
||||
E2E_REQUIRED_TEST_COUNT: '4'
|
||||
steps:
|
||||
- name: Checkout the tested source
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Require and materialize the upstream login
|
||||
env:
|
||||
STORAGE_STATE: ${{ secrets.E2E_LINEAR_STORAGE_STATE_B64 }}
|
||||
run: |
|
||||
umask 077
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
encoded = os.environ.get("STORAGE_STATE", "")
|
||||
if not encoded:
|
||||
raise SystemExit("E2E_LINEAR_STORAGE_STATE_B64 is required; capture and provision a test-account login")
|
||||
state = json.loads(base64.b64decode(encoded, validate=True))
|
||||
if not isinstance(state, dict) or not state.get("cookies"):
|
||||
raise SystemExit("The captured login must contain browser cookies")
|
||||
directory = Path(os.environ["RUNNER_TEMP"]) / "mcp-oauth-private"
|
||||
directory.mkdir(mode=0o700)
|
||||
path = directory / "linear-state.json"
|
||||
path.write_text(json.dumps(state))
|
||||
with open(os.environ["GITHUB_ENV"], "a") as output:
|
||||
output.write(f"E2E_LINEAR_STORAGE_STATE={path}\n")
|
||||
for name in ("LITELLM_MASTER_KEY", "LITELLM_SALT_KEY"):
|
||||
value = "sk-e2e-" + secrets.token_hex(24)
|
||||
print(f"::add-mask::{value}")
|
||||
output.write(f"{name}={value}\n")
|
||||
PY
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: '0.10.9'
|
||||
- uses: ./.github/actions/cache-cargo-build
|
||||
- name: Install the frozen E2E environment
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --extra proxy --extra proxy-runtime --extra extra_proxy --group ci --group proxy-dev --group e2e-dev
|
||||
uv run --no-sync python scripts/prisma_generate_if_needed.py
|
||||
uv run --no-sync playwright install --with-deps chromium
|
||||
|
||||
- name: Configure license access
|
||||
id: aws
|
||||
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
|
||||
with:
|
||||
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
|
||||
aws-region: us-east-1
|
||||
role-session-name: mcp-oauth-${{ github.run_id }}
|
||||
role-duration-seconds: 900
|
||||
output-env-credentials: false
|
||||
output-credentials: true
|
||||
- name: Load the E2E license
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
|
||||
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
|
||||
AWS_DEFAULT_REGION: us-east-1
|
||||
run: |
|
||||
license="$(aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license --query SecretString --output text)"
|
||||
test -n "${license}"
|
||||
echo "::add-mask::${license}"
|
||||
echo "LITELLM_LICENSE=${license}" >> "${GITHUB_ENV}"
|
||||
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: ui/litellm-dashboard/.nvmrc
|
||||
- name: Build the gateway consent UI at the tested commit
|
||||
run: |
|
||||
cd ui/litellm-dashboard
|
||||
../../scripts/with_dashboard_node.sh npm ci
|
||||
../../scripts/with_dashboard_node.sh npm run build
|
||||
mkdir -p ../../litellm/proxy/_experimental/out
|
||||
cp -r out/. ../../litellm/proxy/_experimental/out/
|
||||
find ../../litellm/proxy/_experimental/out -name '*.html' ! -name index.html | while read -r page; do
|
||||
mkdir -p "${page%.html}"
|
||||
mv "${page}" "${page%.html}/index.html"
|
||||
done
|
||||
|
||||
- name: Prepare the isolated database and IdP
|
||||
run: |
|
||||
umask 077
|
||||
bash .github/e2e-stack/start-idp.sh
|
||||
uv run --no-sync python migrations/run.py > "${RUNNER_TEMP}/mcp-oauth-private/migrations.log" 2>&1
|
||||
|
||||
- name: Run every required OAuth variant without retries
|
||||
run: |
|
||||
umask 077
|
||||
uv run --no-sync pytest -c tests/e2e/pytest.ini tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
|
||||
--rootdir=. --reruns 0 --tb=short -o junit_family=xunit1 \
|
||||
--junitxml="${RUNNER_TEMP}/mcp-oauth-private/results.xml" \
|
||||
> "${RUNNER_TEMP}/mcp-oauth-private/pytest.log" 2>&1
|
||||
- name: Report JUnit results and reject skipped or missing cases
|
||||
if: always()
|
||||
run: |
|
||||
uv run --no-sync python .github/e2e-stack/assert_tests_ran.py \
|
||||
"${RUNNER_TEMP}/mcp-oauth-private/results.xml" tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py
|
||||
- name: Remove private login and logs
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f e2e-keycloak >/dev/null 2>&1 || true
|
||||
rm -rf "${RUNNER_TEMP}/mcp-oauth-private"
|
||||
207
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: Merge smoke checks
|
||||
name: smoke
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
|
@ -14,7 +14,7 @@ concurrency:
|
|||
|
||||
jobs:
|
||||
dashboard-build:
|
||||
name: Dashboard build
|
||||
name: dashboard-build
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
|
@ -27,13 +27,13 @@ jobs:
|
|||
run: docker build --target ui-builder -f Dockerfile .
|
||||
|
||||
core-checks:
|
||||
name: Core checks (Python ${{ matrix.python-version }})
|
||||
name: py${{ matrix.python-version }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
|
|
@ -76,12 +76,15 @@ jobs:
|
|||
- name: Import and CLI checks
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
|
||||
|
||||
- name: Routing and UUID compatibility
|
||||
run: >-
|
||||
$IN_NETNS .venv/bin/python -m pytest
|
||||
tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions
|
||||
tests/unit/test_uuid_helper.py -q
|
||||
|
||||
- name: Proxy startup check
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
|
||||
|
||||
- name: Run curated smoke cases
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py pytest --manifest .github/merge-smoke-tests.json
|
||||
|
||||
- name: Upload smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
|
||||
|
|
@ -93,3 +96,193 @@ jobs:
|
|||
- name: Remove the network namespace
|
||||
if: always()
|
||||
run: sudo ip netns delete smoke
|
||||
|
||||
all-extras:
|
||||
name: all-extras-py${{ matrix.python-version }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.12", "3.13"]
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install every extra and dependency group
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras --python ${{ matrix.python-version }}
|
||||
|
||||
- name: Import the SDK, the proxy server and the enterprise package
|
||||
run: .venv/bin/python -c "import litellm, litellm.proxy.proxy_server, litellm_enterprise"
|
||||
|
||||
windows-sdk:
|
||||
name: windows-sdk
|
||||
runs-on: windows-2025
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Set up the Rust bridge
|
||||
uses: ./.github/actions/rust-bridge
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
|
||||
|
||||
- name: Run the Windows tests
|
||||
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests
|
||||
|
||||
proxy-build-from-pip:
|
||||
name: proxy-build-from-pip
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: circle_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 2s
|
||||
--health-timeout 5s
|
||||
--health-retries 30
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install the test client dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12
|
||||
|
||||
- name: Build the image from the published package
|
||||
run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip .
|
||||
|
||||
- name: Start the fake OpenAI endpoint
|
||||
run: |
|
||||
nohup .venv/bin/python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 > "${RUNNER_TEMP}/fake-openai.log" 2>&1 &
|
||||
for _ in $(seq 1 30); do
|
||||
curl -sf http://localhost:8190/health > /dev/null && exit 0
|
||||
sleep 1
|
||||
done
|
||||
cat "${RUNNER_TEMP}/fake-openai.log"
|
||||
exit 1
|
||||
|
||||
- name: Generate a master key
|
||||
run: |
|
||||
key="sk-$(openssl rand -hex 16)"
|
||||
echo "::add-mask::${key}"
|
||||
echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Start the proxy container
|
||||
run: |
|
||||
docker run -d \
|
||||
-p 4000:4000 \
|
||||
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
|
||||
-e LITELLM_MASTER_KEY \
|
||||
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
|
||||
-e OTEL_EXPORTER=in_memory \
|
||||
-e USE_DDTRACE=True \
|
||||
-e GCS_FLUSH_INTERVAL=1 \
|
||||
-e LITELLM_LOG=ERROR \
|
||||
--add-host host.docker.internal:host-gateway \
|
||||
--name litellm-build-from-pip \
|
||||
-v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \
|
||||
litellm-build-from-pip:smoke \
|
||||
--config /app/config.yaml \
|
||||
--port 4000
|
||||
for _ in $(seq 1 150); do
|
||||
curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0
|
||||
sleep 2
|
||||
done
|
||||
exit 1
|
||||
|
||||
- name: Run the startup tests
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
base_url=http://localhost:4000
|
||||
readiness_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' "${base_url}/health/readiness")"
|
||||
readiness_status="${readiness_response##*$'\n'}"
|
||||
readiness_body="${readiness_response%$'\n'*}"
|
||||
[[ "${readiness_status}" == "200" ]]
|
||||
jq -e '.status == "healthy" or .status == "connected"' <<< "${readiness_body}" > /dev/null
|
||||
liveness_status="$(curl -sS --fail-with-body --output /dev/null --write-out '%{http_code}' "${base_url}/health/liveness")"
|
||||
[[ "${liveness_status}" == "200" ]]
|
||||
completion_response="$(curl -sS --fail-with-body --write-out $'\n%{http_code}' \
|
||||
--header "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data '{"model":"gpt-4","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello!"}]}' \
|
||||
"${base_url}/chat/completions")"
|
||||
completion_status="${completion_response##*$'\n'}"
|
||||
completion_body="${completion_response%$'\n'*}"
|
||||
[[ "${completion_status}" == "200" ]]
|
||||
jq -e '
|
||||
(.choices | length) == 1 and
|
||||
.choices[0].message.role == "assistant" and
|
||||
.choices[0].message.content == "Hello! This is a mock response from the fake OpenAI endpoint."
|
||||
' <<< "${completion_body}" > /dev/null
|
||||
|
||||
- name: Show the proxy logs
|
||||
if: always()
|
||||
run: docker logs litellm-build-from-pip || true
|
||||
|
||||
smoke-passed:
|
||||
name: smoke passed
|
||||
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Require every smoke job to succeed
|
||||
env:
|
||||
NEEDS: ${{ toJSON(needs) }}
|
||||
run: |
|
||||
jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS"
|
||||
jq -e 'all(.[]; .result == "success")' <<< "$NEEDS" > /dev/null
|
||||
|
|
|
|||
106
.github/workflows/test-rust-db.yml
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
name: LiteLLM Rust DB
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "litellm-rust/crates/db/**"
|
||||
- "litellm-rust/Cargo.toml"
|
||||
- "litellm-rust/Cargo.lock"
|
||||
- "litellm-rust/clippy.toml"
|
||||
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
|
||||
- "**/schema.prisma"
|
||||
- ".cargo/**"
|
||||
- "rust-toolchain.toml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/workflows/test-rust-db.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- "litellm-rust/crates/db/**"
|
||||
- "litellm-rust/Cargo.toml"
|
||||
- "litellm-rust/Cargo.lock"
|
||||
- "litellm-rust/clippy.toml"
|
||||
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
|
||||
- "**/schema.prisma"
|
||||
- ".cargo/**"
|
||||
- "rust-toolchain.toml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/workflows/test-rust-db.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
db-queries:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
|
||||
env:
|
||||
POSTGRES_DB: litellm
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_USER: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d litellm"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm
|
||||
SQLX_OFFLINE: "false"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: litellm-rust
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: litellm-rust
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Install Python dependencies
|
||||
working-directory: .
|
||||
run: uv sync --frozen --extra extra-proxy
|
||||
|
||||
- name: Deploy Prisma migrations
|
||||
working-directory: .
|
||||
run: uv run --no-sync prisma migrate deploy --schema litellm-proxy-extras/litellm_proxy_extras/schema.prisma
|
||||
|
||||
- name: Install SQLx CLI
|
||||
run: cargo install sqlx-cli --version 0.9.0 --locked --no-default-features -F postgres
|
||||
|
||||
- name: Check SQLx query metadata
|
||||
working-directory: litellm-rust/crates/db
|
||||
run: cargo sqlx prepare --check
|
||||
|
||||
- name: Test database queries
|
||||
run: cargo test -p litellm-db --locked -- --include-ignored
|
||||
21
.github/workflows/test-rust.yml
vendored
|
|
@ -7,6 +7,8 @@ on:
|
|||
- "litellm/rust_bridge/**"
|
||||
- "scripts/generate_trace_types.py"
|
||||
- "scripts/trace_codegen/**"
|
||||
- "scripts/lens_assets/**"
|
||||
- "litellm/tracing/**"
|
||||
- "tests/test_litellm_rust/**"
|
||||
- "litellm/integrations/custom_logger.py"
|
||||
- "litellm/litellm_core_utils/litellm_logging.py"
|
||||
|
|
@ -14,9 +16,12 @@ on:
|
|||
- "litellm/proxy/guardrails/**"
|
||||
- "litellm/utils.py"
|
||||
- "litellm/ocr/**"
|
||||
- "litellm/chat_completions/dispatch.py"
|
||||
- "litellm/embeddings/dispatch.py"
|
||||
- "litellm/messages/dispatch.py"
|
||||
- "litellm/responses/dispatch.py"
|
||||
- "litellm/llms/base_llm/ocr/**"
|
||||
- "litellm/llms/custom_httpx/llm_http_handler.py"
|
||||
- "tests/test_litellm/conftest.py"
|
||||
- "Makefile"
|
||||
- ".cargo/**"
|
||||
- "pyproject.toml"
|
||||
|
|
@ -36,6 +41,8 @@ on:
|
|||
- "litellm/rust_bridge/**"
|
||||
- "scripts/generate_trace_types.py"
|
||||
- "scripts/trace_codegen/**"
|
||||
- "scripts/lens_assets/**"
|
||||
- "litellm/tracing/**"
|
||||
- "tests/test_litellm_rust/**"
|
||||
- "litellm/integrations/custom_logger.py"
|
||||
- "litellm/litellm_core_utils/litellm_logging.py"
|
||||
|
|
@ -43,9 +50,12 @@ on:
|
|||
- "litellm/proxy/guardrails/**"
|
||||
- "litellm/utils.py"
|
||||
- "litellm/ocr/**"
|
||||
- "litellm/chat_completions/dispatch.py"
|
||||
- "litellm/embeddings/dispatch.py"
|
||||
- "litellm/messages/dispatch.py"
|
||||
- "litellm/responses/dispatch.py"
|
||||
- "litellm/llms/base_llm/ocr/**"
|
||||
- "litellm/llms/custom_httpx/llm_http_handler.py"
|
||||
- "tests/test_litellm/conftest.py"
|
||||
- "Makefile"
|
||||
- ".cargo/**"
|
||||
- "pyproject.toml"
|
||||
|
|
@ -89,11 +99,14 @@ jobs:
|
|||
cache-on-failure: true
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- run: cargo clippy --workspace --all-targets --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema -- -D warnings
|
||||
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
|
||||
|
||||
rust-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
# Trim debuginfo to keep the job's target dir within the runner's disk
|
||||
env:
|
||||
CARGO_PROFILE_DEV_DEBUG: line-tables-only
|
||||
defaults:
|
||||
run:
|
||||
working-directory: litellm-rust
|
||||
|
|
@ -132,7 +145,7 @@ jobs:
|
|||
working-directory: .
|
||||
run: uv run scripts/generate_trace_types.py --check
|
||||
|
||||
- run: cargo nextest run --workspace --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema
|
||||
- run: cargo nextest run --workspace --locked
|
||||
|
||||
- run: cargo test --workspace --doc --locked
|
||||
|
||||
|
|
|
|||
37
.github/workflows/test-semgrep.yml
vendored
|
|
@ -1,37 +0,0 @@
|
|||
name: Semgrep
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
semgrep:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Run Semgrep (custom rules)
|
||||
run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error
|
||||
103
.github/workflows/test-unit-documentation.yml
vendored
|
|
@ -1,103 +0,0 @@
|
|||
name: "Unit Tests: Documentation Validation"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
documentation:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect relevant changes
|
||||
id: changes
|
||||
uses: ./.github/actions/detect-changes
|
||||
|
||||
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
repository: BerriAI/litellm-docs
|
||||
path: docs/my-website
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache uv dependencies
|
||||
if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main'
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-
|
||||
|
||||
- name: Cache the Rust build
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
|
||||
|
||||
- name: Cache Prisma binaries
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
uses: ./.github/actions/cache-prisma-binaries
|
||||
|
||||
- name: Generate Prisma client
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
|
||||
|
||||
# Run the same documentation tests that CircleCI ran (as direct Python scripts)
|
||||
- name: Run documentation validation tests
|
||||
if: steps.changes.outputs.decision != 'skip'
|
||||
run: |
|
||||
uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
|
||||
uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py
|
||||
223
.github/workflows/test-unit-proxy-db.yml
vendored
|
|
@ -1,223 +0,0 @@
|
|||
name: "Unit Tests: Proxy DB Operations"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
|
||||
# rather than alphabetical letter ranges. Adding a new test file means adding it
|
||||
# to whichever group it belongs to, not reshuffling slices.
|
||||
#
|
||||
# Design targets:
|
||||
# * Every shard runs in <= 7 minutes of wall-clock on the default runner.
|
||||
# Most of a shard's time is pytest plugin load + xdist worker imports +
|
||||
# pytest-cov instrumentation, not the tests themselves. Keeping per-shard
|
||||
# work low and matching worker count to runner cores is what controls it.
|
||||
# * `timeout` bounds the pytest step only. Checkout, dependency install, and
|
||||
# Prisma client generation draw on a separate allowance in the base
|
||||
# workflow, so slow setup shows up as a slow job rather than as a
|
||||
# cancelled shard whose tests were passing.
|
||||
# * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
|
||||
# oversubscribes 2x and workers fight for CPU during their cold-start
|
||||
# imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
|
||||
# * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
|
||||
# conflicts with the logging worker when run in parallel.
|
||||
# * test_proxy_utils.py runs as a single shard with --dist=worksteal so
|
||||
# xdist balances its 188 parametrized cases across workers instead of
|
||||
# pinning the whole file to one worker (the default --dist=loadscope
|
||||
# behavior for single-file targets).
|
||||
jobs:
|
||||
lens-python-310:
|
||||
name: Lens Python 3.10
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: write
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
python-version: "3.10"
|
||||
test-path: tests/unit/proxy/lens/test_inference.py
|
||||
workers: 0
|
||||
reruns: 0
|
||||
timeout-minutes: 5
|
||||
artifact-name: lens-python-310
|
||||
|
||||
# Fast guard — fails the workflow when a test directory or file inside a sharded
|
||||
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
|
||||
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
|
||||
# and reads the same test-path keys the coverage census does.
|
||||
assert-shard-coverage:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Assert every test directory and file is claimed by a shard
|
||||
run: python3 .github/scripts/assert_ci_coverage.py --shards
|
||||
|
||||
proxy-db:
|
||||
needs: assert-shard-coverage
|
||||
# Display only the semantic shard name in the checks UI instead of GHA's
|
||||
# default "proxy-db (key-generation, tests/unit/proxy/…, 0, loadscope, 20)"
|
||||
# which includes every matrix field and gets truncated past the test-path.
|
||||
name: ${{ matrix.test-group }}
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
pull-requests: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# Must run serially — event-loop conflict with the logging worker.
|
||||
- test-group: key-generation
|
||||
test-path: >-
|
||||
tests/unit/proxy/management_endpoints/test_key_generate_prisma.py
|
||||
workers: 0
|
||||
dist: loadscope
|
||||
timeout: 20
|
||||
|
||||
# ---- auth: split into 2 shards ----
|
||||
- test-group: auth-checks
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_auth_checks.py
|
||||
tests/unit/proxy/auth/test_user_api_key_auth.py
|
||||
tests/unit/proxy/test_credential_slot_registry.py
|
||||
tests/unit/proxy/test_deprecated_key_grace_period.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: jwt-and-keys
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_jwt.py
|
||||
tests/unit/proxy/management_endpoints/test_jwt_key_mapping.py
|
||||
tests/unit/proxy/test_proxy_custom_auth.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- test_proxy_utils.py, single shard, worksteal distribution ----
|
||||
- test-group: proxy-utils
|
||||
test-path: >-
|
||||
tests/unit/proxy/test_proxy_utils.py
|
||||
workers: 4
|
||||
dist: worksteal
|
||||
timeout: 15
|
||||
|
||||
# ---- proxy server: split into 2 shards ----
|
||||
- test-group: proxy-server-core
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_server_gemini_pass_through.py
|
||||
tests/unit/proxy/test__lazy_features.py
|
||||
tests/unit/proxy/test_aproxy_startup.py
|
||||
tests/unit/proxy/test_proxy_server.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: proxy-runtime
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_multipart_bypass_repro.py
|
||||
tests/unit/proxy/auth/test_proxy_routes.py
|
||||
tests/unit/proxy/middleware/test_request_size_limit_middleware.py
|
||||
tests/unit/proxy/test_proxy_config_unit_test.py
|
||||
tests/unit/proxy/test_proxy_token_counter.py
|
||||
tests/unit/proxy/test_server_root_path.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- logging: split into 2 shards ----
|
||||
- test-group: custom-logging
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_custom_logger.py
|
||||
tests/unit/proxy/test_custom_callback_input.py
|
||||
tests/unit/proxy/test_custom_logger_s3_gcs.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: logging-misc
|
||||
test-path: >-
|
||||
tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
|
||||
tests/unit/proxy/spend_tracking/test_search_api_logging.py
|
||||
tests/unit/proxy/test_proxy_reject_logging.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: db-and-spend
|
||||
test-path: >-
|
||||
tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
|
||||
tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
|
||||
tests/unit/proxy/db/test_update_daily_tag_spend.py
|
||||
tests/unit/proxy/test_db_schema_changes.py
|
||||
tests/unit/proxy/test_prisma_client_backoff_retry.py
|
||||
tests/unit/proxy/test_update_spend.py
|
||||
tests/unit/skills/test_skills_db.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
# ---- guardrails + budget + hooks: split into 2 ----
|
||||
- test-group: guardrails-hooks
|
||||
test-path: >-
|
||||
tests/unit/proxy/hooks/test_banned_keyword_list.py
|
||||
tests/unit/proxy/test_proxy_setting_guardrails.py
|
||||
tests/unit/proxy/test_unit_test_proxy_hooks.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
- test-group: budgets
|
||||
test-path: >-
|
||||
tests/unit/proxy/auth/test_default_end_user_budget_simple.py
|
||||
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
|
||||
tests/unit/proxy/test_zero_cost_model_budget_bypass.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
|
||||
- test-group: endpoints-and-responses
|
||||
test-path: >-
|
||||
tests/proxy_unit_tests/test_proxy_exception_mapping.py
|
||||
tests/unit/proxy/lens
|
||||
tests/unit/proxy/auth/test_models_fallback_endpoint.py
|
||||
tests/unit/proxy/common_utils/test_check_batch_cost.py
|
||||
tests/unit/proxy/common_utils/test_check_responses_cost.py
|
||||
tests/unit/proxy/common_utils/test_realtime_cache.py
|
||||
tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
|
||||
tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py
|
||||
tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py
|
||||
tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py
|
||||
tests/unit/proxy/response_polling
|
||||
tests/unit/proxy/test_custom_tokenizer_bug.py
|
||||
tests/unit/proxy/test_get_favicon.py
|
||||
tests/unit/proxy/test_get_image.py
|
||||
tests/unit/proxy/test_prompt_test_endpoint.py
|
||||
tests/unit/proxy/test_reducto_ocr_route.py
|
||||
tests/unit/proxy/test_response_polling_pre_call_checks.py
|
||||
tests/unit/proxy/test_ui_path_detection.py
|
||||
workers: 4
|
||||
dist: loadscope
|
||||
timeout: 15
|
||||
uses: ./.github/workflows/_test-unit-base.yml
|
||||
with:
|
||||
test-path: ${{ matrix.test-path }}
|
||||
workers: ${{ matrix.workers }}
|
||||
reruns: 2
|
||||
timeout-minutes: ${{ matrix.timeout }}
|
||||
dist: ${{ matrix.dist }}
|
||||
artifact-name: proxy-db-${{ matrix.test-group }}
|
||||
1282
.github/workflows/test-unit.yml
vendored
4
.github/workflows/weekly_load_anomaly.yml
vendored
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
|
||||
- name: Start the proxy
|
||||
run: |
|
||||
nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
|
||||
nohup uv run --no-sync litellm --config tests/e2e/perf/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
|
||||
for _ in $(seq 1 90); do
|
||||
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
|
||||
exit 0
|
||||
|
|
@ -78,7 +78,7 @@ jobs:
|
|||
env:
|
||||
E2E_WEEKLY_ANOMALY: "1"
|
||||
run: |
|
||||
uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
|
||||
uv run --no-sync pytest tests/e2e/perf/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
|
||||
|
||||
- name: Show proxy log on failure
|
||||
if: failure()
|
||||
|
|
|
|||
3
.gitignore
vendored
|
|
@ -105,7 +105,7 @@ litellm_config.yaml
|
|||
.cursor
|
||||
litellm/proxy/to_delete_loadtest_work/*
|
||||
update_model_cost_map.py
|
||||
tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py
|
||||
tests/unit/proxy/mcp_server/test_mcp_server_manager.py
|
||||
scripts/test_vertex_ai_search.py
|
||||
LAZY_LOADING_IMPROVEMENTS.md
|
||||
STABILIZATION_TODO.md
|
||||
|
|
@ -154,3 +154,4 @@ coverage-rust.xml
|
|||
|
||||
# make lens-dev worker token, generated config and logs
|
||||
.lens-dev/
|
||||
.mypy_cache/
|
||||
|
|
|
|||
45
AGENTS.md
|
|
@ -23,30 +23,30 @@ When adding new features, add meaningful tests. Don't add tests that don't check
|
|||
|
||||
Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
|
||||
|
||||
Never test structure of code only function of it
|
||||
Never test structure or implementation details of code, only function of it. A test should catch a change that breaks behavior by accident. If the only way it can fail is someone deliberately undoing the change, don't write it
|
||||
|
||||
A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
|
||||
A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent". Assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
|
||||
|
||||
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
|
||||
|
||||
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
|
||||
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
|
||||
|
||||
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
|
||||
|
||||
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
|
||||
When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
|
||||
|
||||
Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
|
||||
|
||||
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
|
||||
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
|
||||
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
|
||||
|
||||
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
|
||||
- don't use emojis
|
||||
- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
|
||||
- don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences
|
||||
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
|
||||
- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
|
||||
- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
|
||||
- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "."
|
||||
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
|
||||
- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
|
||||
|
||||
|
|
@ -54,25 +54,25 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
|
|||
|
||||
Python max line length is 120, not 88
|
||||
|
||||
Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR
|
||||
The five lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/openapi_docs_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch
|
||||
|
||||
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
|
||||
|
||||
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
|
||||
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy" followed by "queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
|
||||
|
||||
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
|
||||
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
|
||||
|
||||
If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
|
||||
If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used. Reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
|
||||
|
||||
Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType] # stubs lack async overload` or `# noqa: TID251 # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
|
||||
|
||||
Commit and push your work when you're done without asking
|
||||
|
||||
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
|
||||
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified. Treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
|
||||
|
||||
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
|
||||
|
||||
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
|
||||
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
|
||||
|
||||
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer de
|
|||
|
||||
Do not put names of customers or customer company names in code, PR descriptions, issue bodies, etc. This means never mention literally any company name. Especially if you're about to say a sentence mentioning that the reason the PR exists was a feature/model/bug fix/etc. requested by a company. That's the indication that you should replace that company name with "the customer". e.g. not "Model request from Acme (Pylon #1234)" but "Model request from a customer (Pylon #1234)". This is because the codebase is public. The only exception is for publicly known providers or vendors such as OpenAI, Anthropic, AWS Bedrock, etc. only IF we're adding support for that provider/vendor in general and NOT if that PR or whatnot was a request by one of them, and they're actually one of our customers.
|
||||
|
||||
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
|
||||
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
|
||||
|
||||
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
|
||||
|
||||
|
|
@ -92,18 +92,19 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
|
|||
|
||||
- Composition over inheritance
|
||||
- Never-nester: early returns over deep nesting
|
||||
- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
|
||||
- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
|
||||
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
|
||||
- Don't throw. Model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
|
||||
- No mutation. Don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
|
||||
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011). `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
|
||||
- Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
|
||||
- Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
|
||||
- Comprehensions take at most one `for` clause and one `if` clause (LIT014). Split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: <reason>` only when unavoidable
|
||||
- Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: <reason>` on the `class` line
|
||||
- Use dependency injection
|
||||
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
|
||||
- Fully typed. No `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
|
||||
- Use tagged unions + match
|
||||
- No monster files or god objects
|
||||
- No file sprawl: deliberate file and folder structure
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists; where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module; if none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
- Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
|
||||
- API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
|
||||
|
||||
Follow conventional commits for commit names and PR titles
|
||||
|
||||
|
|
@ -129,4 +130,4 @@ Before implementing:
|
|||
|
||||
Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify
|
||||
|
||||
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check
|
||||
Before requesting maintainer review, verify the current PR tip passes required CI and code coverage and has acceptable reviews from the review bots on the PR, such as Veria and Bugbot. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review. Never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check
|
||||
|
|
|
|||
|
|
@ -239,6 +239,8 @@ graph LR
|
|||
7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
|
||||
8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
|
||||
|
||||
Cost lookup falls back to the response model when the provider-reported name cannot be priced. Explicitly registered zero rates remain zero, and custom deployment pricing or an explicit `base_model` retain precedence
|
||||
|
||||
### Data Access Layer (Models & Repositories)
|
||||
|
||||
Database entities and the operations on them live in two packages at the root of `litellm/` so both the gateway (`proxy/`) and the SDK can use them without importing proxy internals:
|
||||
|
|
@ -332,13 +334,13 @@ Each translation is isolated in its own file, making it easy to test and modify
|
|||
|--------------|----------|------------------|
|
||||
| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
|
||||
| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
|
||||
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
|
||||
| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py` |
|
||||
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
|
||||
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
|
||||
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/messages/native/transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` |
|
||||
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py` |
|
||||
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
|
||||
|
||||
### Example: Debugging prompt caching
|
||||
|
|
@ -346,7 +348,7 @@ Each translation is isolated in its own file, making it easy to test and modify
|
|||
If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
|
||||
|
||||
1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
|
||||
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
|
||||
2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py`
|
||||
3. Compare how each handles `cache_control` in `transform_request()`
|
||||
|
||||
### How translations work
|
||||
|
|
|
|||
|
|
@ -39,12 +39,12 @@ git clone https://github.com/YOUR_USERNAME/litellm.git
|
|||
cd litellm
|
||||
|
||||
# Create a new branch for your feature (see "Commit and Branch Conventions" below)
|
||||
git checkout -b feature/your-feature
|
||||
git checkout -b litellm_your_feature
|
||||
|
||||
# Install development dependencies
|
||||
make install-dev
|
||||
|
||||
# Install git hooks that enforce commit + branch conventions (one-time, opt-in)
|
||||
# Install git hooks that enforce commit conventions (one-time, opt-in)
|
||||
make install-hooks
|
||||
|
||||
# Verify your setup works
|
||||
|
|
@ -55,7 +55,7 @@ That's it! Your local development environment is ready.
|
|||
|
||||
## Commit and Branch Conventions
|
||||
|
||||
Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and branches follow [Conventional Branches](https://conventional-branch.github.io/). Run `make install-hooks` once per clone to enable the local git hooks that enforce these — see the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for the full type list, examples, the protected-branch bypass list, and how to opt out.
|
||||
Commits follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). Internal contributors name branches with the `litellm_` prefix and no `/`, and fork branches can use any name. Run `make install-hooks` once per clone to enable the local hook that enforces the commit convention. See the [contributor docs](https://docs.litellm.ai/docs/extras/contributing_code#commit-and-branch-conventions) for details
|
||||
|
||||
### 2. Development Workflow
|
||||
|
||||
|
|
@ -78,8 +78,8 @@ uv run pytest tests/unit/<your_test_file>.py -v
|
|||
git add .
|
||||
git commit -m "feat(scope): your descriptive commit message"
|
||||
|
||||
# Push and create a PR (branch must follow Conventional Branches — see above)
|
||||
git push origin feature/your-feature
|
||||
# Push and create a PR
|
||||
git push origin litellm_your_feature
|
||||
```
|
||||
|
||||
## Adding Testing
|
||||
|
|
@ -94,6 +94,8 @@ Add your tests to the [`tests/unit/` directory](https://github.com/BerriAI/litel
|
|||
- **Only add mocked tests** - no real LLM API calls in this directory
|
||||
- For integration tests with real APIs, use the appropriate test directories
|
||||
|
||||
Tests that call `litellm.register_model` or change `litellm.model_cost` should use the `local_model_cost_map` fixture from `tests/unit/conftest.py`. It provides a fresh catalog and restores the previous map, so pricing and capability overrides cannot leak into later tests
|
||||
|
||||
### File Naming Convention
|
||||
|
||||
The `tests/unit/` directory follows the same structure as `litellm/`:
|
||||
|
|
|
|||
12
Dockerfile
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -39,6 +39,7 @@ ENV NEXT_TELEMETRY_DISABLED=1 \
|
|||
WORKDIR /ui
|
||||
|
||||
COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./
|
||||
COPY ui/litellm-dashboard/vendor/ ./vendor/
|
||||
RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline
|
||||
|
||||
COPY ui/litellm-dashboard/ ./
|
||||
|
|
@ -60,6 +61,7 @@ RUN apk add --no-cache \
|
|||
python-3.13-dev \
|
||||
rust \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -119,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
|
|
@ -140,7 +142,7 @@ USER root
|
|||
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||