mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-23 00:41:40 +00:00
d05e45893e
- Switch all per-cell tests from @pytest.mark.parametrize("model", ...)
(3 sequential invocations) to a single test that fans out to all 3
Claude tiers via run_claude_models_parallel. Per-cell wall time is now
bounded by the slowest model rather than the sum.
- Add 5 new v0 feature dirs (5 providers each, 25 new test files):
web_search, pdf_input, prompt_caching_1h,
tool_use_streaming, thinking_with_tool_use
Manifest expanded to match.
- Add cross-process token-bucket rate limiter (rate_limiter.py + tests)
so xdist workers stay under per-provider req/s limits during full-grid
runs. New env knobs: LITELLM_COMPAT_RATE_{ANTHROPIC,AZURE,VERTEX_AI,
BEDROCK_CONVERSE,BEDROCK_INVOKE}.
- conftest.py: write per-worker shards under <artifact>.shards/, merge
in the controller; preserve the "don't write empty artifact" guard so
unit-test runs don't clobber a real compat-results.json.
- Vertex test_config.yaml: route project/location through env so the
cron VM can target a different GCP project than the upstream default.
- Add run_compat.sh wrapper for binary-searching ideal req/s per
provider against compat-rate-limit-summary.json output.
106 lines
4.1 KiB
Bash
Executable file
106 lines
4.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Run the Claude Code compat matrix end-to-end against a live LiteLLM
|
|
# proxy, with per-provider rate limits applied via the cross-process
|
|
# token bucket in `tests/claude_code/rate_limiter.py`.
|
|
#
|
|
# Designed for binary-searching the ideal X / Y / Z req/s per provider:
|
|
# 1. Pick an initial rate (e.g. 5/s for everyone).
|
|
# 2. Run this script.
|
|
# 3. Read `compat-rate-limit-summary.json` to see whether any provider
|
|
# hit a 429-shaped error during the run.
|
|
# 4. If a provider has `rate_limited > 0`, halve its rate; else, double it.
|
|
# 5. Repeat until the highest no-429 rate is found.
|
|
#
|
|
# Required env (proxy connection):
|
|
# LITELLM_PROXY_BASE_URL e.g. http://localhost:4000
|
|
# LITELLM_PROXY_API_KEY e.g. sk-1234
|
|
#
|
|
# Optional env (rate limits, all default to 5 req/s; 0 disables a column):
|
|
# LITELLM_COMPAT_RATE_ANTHROPIC
|
|
# LITELLM_COMPAT_RATE_AZURE
|
|
# LITELLM_COMPAT_RATE_VERTEX_AI
|
|
# LITELLM_COMPAT_RATE_BEDROCK_CONVERSE
|
|
# LITELLM_COMPAT_RATE_BEDROCK_INVOKE
|
|
# LITELLM_COMPAT_RATE_BURST override per-bucket burst
|
|
#
|
|
# Optional env (parallelism):
|
|
# COMPAT_XDIST_WORKERS passed to `pytest -n` (default: auto)
|
|
#
|
|
# Optional env (artifacts):
|
|
# COMPAT_RESULTS_PATH default: compat-results.json
|
|
# COMPAT_RATE_LIMIT_SUMMARY_PATH default: compat-rate-limit-summary.json
|
|
|
|
set -euo pipefail
|
|
|
|
if [[ -z "${LITELLM_PROXY_BASE_URL:-}" || -z "${LITELLM_PROXY_API_KEY:-}" ]]; then
|
|
echo "error: LITELLM_PROXY_BASE_URL and LITELLM_PROXY_API_KEY must be set" >&2
|
|
exit 64
|
|
fi
|
|
|
|
# Reset the cross-process rate-limiter state from any prior run. Stale
|
|
# token-bucket files would let a previous run's accumulated budget bleed
|
|
# into the new one, which subtly biases the binary search.
|
|
state_dir="${LITELLM_COMPAT_RATE_STATE_DIR:-${TMPDIR:-/tmp}/litellm-claude-compat-ratelimit}"
|
|
if [[ -d "$state_dir" ]]; then
|
|
rm -rf "$state_dir"
|
|
fi
|
|
|
|
# Worker count. `auto` picks one worker per CPU; the rate limiter
|
|
# enforces aggregate provider rates regardless of worker count, so
|
|
# this is a "go as fast as the limiter allows" knob, not a tuning knob.
|
|
workers="${COMPAT_XDIST_WORKERS:-auto}"
|
|
|
|
# Where the artifacts land. We resolve them now so the summary file is
|
|
# always at a known path the caller can grep, even if they didn't set
|
|
# the env explicitly.
|
|
results_path="${COMPAT_RESULTS_PATH:-compat-results.json}"
|
|
summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}"
|
|
|
|
echo "[run_compat] rates:"
|
|
for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do
|
|
var="LITELLM_COMPAT_RATE_${provider}"
|
|
echo " ${provider}=${!var:-default(5/s)}"
|
|
done
|
|
echo " BURST=${LITELLM_COMPAT_RATE_BURST:-default(=rate)}"
|
|
echo "[run_compat] xdist workers: ${workers}"
|
|
echo "[run_compat] results: ${results_path}"
|
|
echo "[run_compat] summary: ${summary_path}"
|
|
|
|
# Run only the per-feature live tests; skip the unit-test directories
|
|
# (they're under directories starting with `_`). The dist=loadfile
|
|
# scheduler keeps each test file pinned to a single worker, which is
|
|
# what we want — every test in a file shares a single ThreadPoolExecutor
|
|
# fanout, and we don't gain anything by splitting it across workers.
|
|
start=$(date +%s)
|
|
COMPAT_RESULTS_PATH="${results_path}" \
|
|
COMPAT_RATE_LIMIT_SUMMARY_PATH="${summary_path}" \
|
|
PATH="$HOME/.local/bin:$PATH" \
|
|
uv run pytest \
|
|
tests/claude_code/basic_messaging_non_streaming \
|
|
tests/claude_code/basic_messaging_streaming \
|
|
tests/claude_code/extended_thinking \
|
|
tests/claude_code/tool_use \
|
|
tests/claude_code/vision \
|
|
tests/claude_code/prompt_caching_5m \
|
|
-n "${workers}" \
|
|
--dist=loadfile \
|
|
-q \
|
|
"$@"
|
|
|
|
exit_code=$?
|
|
end=$(date +%s)
|
|
echo "[run_compat] wall time: $((end - start))s"
|
|
|
|
# Surface the rate-limit summary inline so a human reader doesn't have
|
|
# to `cat` the JSON file. The full file is still on disk for the binary
|
|
# search loop.
|
|
if [[ -f "${summary_path}" ]]; then
|
|
echo "[run_compat] summary: ${summary_path}"
|
|
if command -v jq >/dev/null 2>&1; then
|
|
jq '.totals, .per_provider' "${summary_path}"
|
|
else
|
|
cat "${summary_path}"
|
|
fi
|
|
fi
|
|
|
|
exit "${exit_code}"
|