mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
17b88719a2
* ci(lint): grandfather any-discipline with a per-file ratchet budget (50% headroom) The any-discipline gate previously failed on any Any-typed value touched on a changed line, which tripped on merely editing a legacy `X | Any` line. Switch it to a per-file budget: `any-discipline-budget.json` records each file's current Any count and a changed file fails only when its count exceeds `baseline + slack` (50% headroom, rounded up). New/unbudgeted files have baseline 0, so they stay airtight, while editing legacy files no longer forces cleaning pre-existing debt. Only changed files are re-type-checked (per-PR cost unchanged); the whole-tree scan to recapture the budget runs under `--update` (`make lint-any-budget-update`). The budget is a one-way ratchet guarded by `budget_ratchet_check.py`, matching the ruff/mypy/basedpyright budgets, and folds into `make lint-budget-update`. Also fixes a RecursionError in `contains_any` (recursive type aliases yield fresh objects per unfold, defeating the id() cycle guard) by walking iteratively with a depth cap, exposed by the whole-tree scan. * chore: make CLAUDE.md more concise * chore: rearrange Makefile * ci(lint): make any-budget --update git-failure-safe; clarify over-budget message all_litellm_py_files now returns None when git is unavailable (mirroring changed_line_map) instead of letting CalledProcessError/FileNotFoundError escape as a raw traceback, and update_budget reports a clean setup error (exit 2) for that case. The list-files dependency is injected so the path is unit-testable without monkeypatching. The over-budget diagnostic now reads "N value(s) total, over budget" so the count isn't misread as the excess over the ceiling. * ci(lint): exempt the file-keyed any-discipline budget from the ratchet's dropped-entry rule budget_ratchet_check treats a vanished budget entry as a loosening (an untracked rule whose ceiling is now unbounded). That holds for the rule-keyed budgets, but the any-discipline budget is keyed by file and its gate treats an absent file as ceiling 0 (the file must be Any-free). Cleaning a file to zero drops its entry on the next --update, so the generic rule flagged that as a regression: a false- positive red on exactly the cleanup the ratchet exists to encourage. Exempt the file-keyed budget from the dropped-entry rule while still catching a raised ceiling.
230 lines
12 KiB
Makefile
230 lines
12 KiB
Makefile
# LiteLLM Makefile
|
|
# Simple Makefile for running tests and basic development tasks
|
|
|
|
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
|
|
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
|
|
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
|
|
info lint lint-dev format \
|
|
lint-mypy lint-mypy-budget-update lint-basedpyright lint-basedpyright-budget-update \
|
|
lint-ruff-budget lint-any lint-ruff-budget-update lint-budget-update lint-any-budget-update \
|
|
install-dev install-proxy-dev install-test-deps install-hooks \
|
|
install-helm-unittest check-circular-imports check-import-safety
|
|
|
|
# Default target
|
|
help:
|
|
@echo "Available commands:"
|
|
@echo " make install-dev - Install development dependencies"
|
|
@echo " make install-proxy-dev - Install proxy development dependencies"
|
|
@echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
|
|
@echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
|
|
@echo " make install-test-deps - Install the full local test environment"
|
|
@echo " make install-helm-unittest - Install helm unittest plugin"
|
|
@echo " make install-hooks - Install git hooks (Conventional Commits + Branches)"
|
|
@echo " make format - Apply Black code formatting"
|
|
@echo " make format-check - Check Black code formatting (matches CI)"
|
|
@echo " make lint - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
|
|
@echo " make lint-ruff - Run Ruff linting only"
|
|
@echo " make lint-mypy - Run MyPy (disallow_untyped_defs), gated by per-rule error counts"
|
|
@echo " make lint-mypy-budget-update - Re-capture the MyPy per-rule budget (ratchet)"
|
|
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
|
|
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
|
|
@echo " make lint-black - Check Black formatting (matches CI)"
|
|
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
|
|
@echo " make lint-any - Gate changed files under litellm/ against their per-file Any budget"
|
|
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
|
|
@echo " make lint-budget-update - Re-capture all four ratchet budgets (ruff + mypy + basedpyright + any)"
|
|
@echo " make lint-any-budget-update - Re-capture the per-file Any budget across the whole tree (ratchet)"
|
|
@echo " make check-circular-imports - Check for circular imports"
|
|
@echo " make check-import-safety - Check import safety"
|
|
@echo " make test - Run all tests"
|
|
@echo " make test-unit - Run unit tests (tests/test_litellm)"
|
|
@echo " make test-unit-llms - Run LLM provider tests (~225 files)"
|
|
@echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
|
|
@echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
|
|
@echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
|
|
@echo " make test-unit-integrations - Run integration tests (~60 files)"
|
|
@echo " make test-unit-core-utils - Run core utils tests (~32 files)"
|
|
@echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)"
|
|
@echo " make test-unit-root - Run root-level tests (~34 files)"
|
|
@echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)"
|
|
@echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)"
|
|
@echo " make test-integration - Run integration tests"
|
|
@echo " make test-unit-helm - Run helm unit tests"
|
|
|
|
UV := uv
|
|
UV_RUN := $(UV) run --no-sync
|
|
|
|
# Show info
|
|
info:
|
|
@echo "UV: $(UV)"
|
|
|
|
# Installation targets
|
|
install-dev:
|
|
$(UV) sync --frozen
|
|
|
|
install-proxy-dev:
|
|
$(UV) sync --frozen --group proxy-dev --extra proxy
|
|
|
|
# CI-compatible installations (matches GitHub workflows exactly)
|
|
install-dev-ci:
|
|
$(UV) sync --frozen
|
|
|
|
install-proxy-dev-ci:
|
|
$(UV) sync --frozen --group proxy-dev --extra proxy
|
|
|
|
install-test-deps: install-proxy-dev
|
|
$(UV) sync --frozen --all-groups --all-extras
|
|
$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
|
|
|
|
install-helm-unittest:
|
|
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
|
|
|
|
# Install git hooks that enforce Conventional Commits and Conventional Branches.
|
|
# Opt-in: not chained into install-dev.
|
|
install-hooks:
|
|
./scripts/install_git_hooks.sh
|
|
|
|
# Formatting
|
|
format: install-dev
|
|
cd litellm && $(UV_RUN) black . && cd ..
|
|
|
|
format-check: install-dev
|
|
cd litellm && $(UV_RUN) black --check . && cd ..
|
|
|
|
# Linting targets
|
|
lint-ruff: install-dev
|
|
cd litellm && $(UV_RUN) ruff check . && cd ..
|
|
|
|
# faster linter for developing ...
|
|
# inspiration from:
|
|
# https://github.com/astral-sh/ruff/discussions/10977
|
|
# https://github.com/astral-sh/ruff/discussions/4049
|
|
lint-format-changed: install-dev
|
|
@git diff origin/main --unified=0 --no-color -- '*.py' | \
|
|
perl -ne '\
|
|
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
|
|
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
|
|
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
|
|
print "$$file:$$start:1-$$end:999\n"; \
|
|
}' | \
|
|
while read range; do \
|
|
file="$${range%%:*}"; \
|
|
lines="$${range#*:}"; \
|
|
echo "Formatting $$file (lines $$lines)"; \
|
|
$(UV_RUN) ruff format --range "$$lines" "$$file"; \
|
|
done
|
|
|
|
lint-ruff-dev: install-dev
|
|
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
|
|
cd litellm && \
|
|
($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \
|
|
$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
|
|
cd .. ; \
|
|
rm -f "$$tmpfile"
|
|
|
|
lint-ruff-FULL-dev: install-dev
|
|
@files=$$(git diff --name-only origin/main -- '*.py'); \
|
|
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
|
|
else echo "No changed .py files to check."; fi
|
|
|
|
lint-mypy: install-dev
|
|
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy
|
|
|
|
lint-mypy-budget-update: install-dev
|
|
cd litellm && ($(UV_RUN) mypy . || true) | $(UV_RUN) python ../scripts/type_check_gate.py --tool mypy --update
|
|
|
|
lint-basedpyright: install-dev
|
|
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright
|
|
|
|
lint-basedpyright-budget-update: install-dev
|
|
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --tool basedpyright --update
|
|
|
|
lint-black: format-check
|
|
|
|
lint-ruff-budget: install-dev
|
|
$(UV_RUN) python scripts/ruff_strict_gate.py
|
|
|
|
lint-ruff-budget-update: install-dev
|
|
$(UV_RUN) python scripts/ruff_strict_gate.py --update
|
|
|
|
# Ratchet all four budgets in one shot (ruff strict + mypy + basedpyright + any)
|
|
lint-budget-update: lint-ruff-budget-update lint-mypy-budget-update lint-basedpyright-budget-update lint-any-budget-update
|
|
|
|
lint-any: install-dev
|
|
$(UV_RUN) python scripts/check_any_discipline.py --changed
|
|
|
|
lint-any-budget-update: install-dev
|
|
$(UV_RUN) python scripts/check_any_discipline.py --update
|
|
|
|
check-circular-imports: install-dev
|
|
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
|
|
|
|
check-import-safety: install-dev
|
|
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
|
|
|
# Combined linting (matches test-linting.yml workflow)
|
|
lint: format-check lint-ruff lint-mypy lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget lint-any
|
|
|
|
# Faster linting for local development (only checks changed code)
|
|
lint-dev: lint-format-changed lint-mypy lint-any check-circular-imports check-import-safety
|
|
|
|
# Testing targets
|
|
test: install-test-deps
|
|
$(UV_RUN) pytest tests/
|
|
|
|
test-unit: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
|
|
|
|
# Matrix test targets (matching CI workflow groups)
|
|
test-unit-llms: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-guardrails: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-core: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-misc: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-integrations: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-core-utils: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
|
|
|
|
test-unit-other: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-root: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
|
|
|
|
# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
|
|
test-proxy-unit-a: install-test-deps
|
|
$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
|
|
|
|
test-proxy-unit-b: install-test-deps
|
|
$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
|
|
|
|
test-integration: install-test-deps
|
|
$(UV_RUN) pytest tests/ -k "not test_litellm"
|
|
|
|
test-unit-helm: install-helm-unittest
|
|
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
|
|
|
# LLM Translation testing targets
|
|
test-llm-translation: install-test-deps
|
|
@echo "Running LLM translation tests..."
|
|
@python .github/workflows/run_llm_translation_tests.py
|
|
|
|
test-llm-translation-single: install-test-deps
|
|
@echo "Running single LLM translation test file..."
|
|
@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
|
|
@mkdir -p test-results
|
|
$(UV_RUN) pytest tests/llm_translation/$(FILE) \
|
|
--junitxml=test-results/junit.xml \
|
|
-v --tb=short --maxfail=100 --timeout=300
|
|
|
|
test-llm-translation-flush-vcr-cache:
|
|
$(UV_RUN) python tests/_flush_vcr_cache.py
|