mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
b8d79d1e0c
* ci: drop redundant mypy type-check gate, standardize on basedpyright Type checking ran both mypy (via the pydantic.mypy plugin) and basedpyright. pydantic v2 emits dataclass_transform, so basedpyright understands models natively with no plugin, and its gated rules already cover what the mypy pass caught (no-untyped-def, no-any-return, valid-type, import-not-found all map to basedpyright equivalents). Running both meant two checkers, two budgets, and a plugin only mypy could load. This removes the mypy type-check gate: the lint-mypy/lint-mypy-budget-update Makefile targets, the CI MyPy step, mypy-code-budget.json, the budget-ratchet entry, and the vestigial [tool.mypy] pydantic plugin block (the gating pass used litellm/mypy.ini, which never loaded the plugin). type_check_gate.py is specialized to basedpyright since the mypy parsing path is now unused. mypy stays a dev dependency because the Any-discipline gate (scripts/check_any_discipline.py) imports it as a library to detect Any-typed values; it is no longer run as a type checker. * ci: remove the Any-discipline gate, rely on basedpyright's reportAny The Any-discipline gate (scripts/check_any_discipline.py) was the last consumer of mypy: it imported mypy as a library to detect values whose inferred type contains Any, gated per-file against any-discipline-budget.json. basedpyright already reports the same class of finding through reportAny/reportExplicitAny, which are gated tree-wide in basedpyright-code-budget.json, so the separate gate (and the mypy dependency behind it) is redundant. Removes the gate end to end: check_any_discipline.py and its test, the any-discipline CI job, the lint-any/lint-any-budget-update Makefile targets, any-discipline-budget.json, litellm/mypy.ini, the .mypy_cache_any references, and mypy from the dev dependencies. budget_ratchet_check.py drops the any-discipline entry and the now-unused zero-floor mechanism (rewritten as a comprehension). check_type_discipline.py drops the any-ok suppression token, since # any-ok suppressed only the deleted gate; the 134 now-orphaned # any-ok comments across 14 files are stripped (they never affected basedpyright, which uses # pyright: ignore). uv.lock is intentionally left untouched: uv still considers it consistent with the mypy-removed pyproject (uv lock --check and uv sync --frozen both pass), and a relock bumps 30+ unrelated packages because of the moving exclude-newer window. A future intentional relock will prune the now-unreferenced mypy entry. * build: relock to drop mypy from uv.lock CI's uv 0.10.9 honors the repo's exclude-newer window and correctly flags the lockfile as out of sync once mypy leaves pyproject; my earlier local uv 0.8.17 could not parse exclude-newer and silently passed --check. Relocking with the pinned CI version removes only mypy and its transitive librt, with no other version changes.
214 lines
10 KiB
Makefile
214 lines
10 KiB
Makefile
# LiteLLM Makefile
|
|
# Simple Makefile for running tests and basic development tasks
|
|
|
|
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
|
|
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
|
|
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
|
|
info lint lint-dev format \
|
|
lint-basedpyright lint-basedpyright-budget-update \
|
|
lint-ruff-budget lint-ruff-budget-update lint-budget-update \
|
|
install-dev install-proxy-dev install-test-deps install-hooks \
|
|
install-helm-unittest check-circular-imports check-import-safety
|
|
|
|
# Default target
|
|
help:
|
|
@echo "Available commands:"
|
|
@echo " make install-dev - Install development dependencies"
|
|
@echo " make install-proxy-dev - Install proxy development dependencies"
|
|
@echo " make install-dev-ci - Install dev dependencies (CI-compatible, pins OpenAI)"
|
|
@echo " make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
|
|
@echo " make install-test-deps - Install the full local test environment"
|
|
@echo " make install-helm-unittest - Install helm unittest plugin"
|
|
@echo " make install-hooks - Install git hooks (Conventional Commits + Branches)"
|
|
@echo " make format - Apply Black code formatting"
|
|
@echo " make format-check - Check Black code formatting (matches CI)"
|
|
@echo " make lint - Run all linting (Ruff, basedpyright, Black check, circular imports, import safety)"
|
|
@echo " make lint-ruff - Run Ruff linting only"
|
|
@echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts"
|
|
@echo " make lint-basedpyright-budget-update - Re-capture the basedpyright per-rule budget (ratchet)"
|
|
@echo " make lint-black - Check Black formatting (matches CI)"
|
|
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its ceiling"
|
|
@echo " make lint-ruff-budget-update - Re-capture per-rule baselines in ruff-strict-budget.json (ratchet)"
|
|
@echo " make lint-budget-update - Re-capture all ratchet budgets (ruff + basedpyright)"
|
|
@echo " make check-circular-imports - Check for circular imports"
|
|
@echo " make check-import-safety - Check import safety"
|
|
@echo " make test - Run all tests"
|
|
@echo " make test-unit - Run unit tests (tests/test_litellm)"
|
|
@echo " make test-unit-llms - Run LLM provider tests (~225 files)"
|
|
@echo " make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
|
|
@echo " make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
|
|
@echo " make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
|
|
@echo " make test-unit-integrations - Run integration tests (~60 files)"
|
|
@echo " make test-unit-core-utils - Run core utils tests (~32 files)"
|
|
@echo " make test-unit-other - Run other tests (caching, responses, etc., ~69 files)"
|
|
@echo " make test-unit-root - Run root-level tests (~34 files)"
|
|
@echo " make test-proxy-unit-a - Run proxy_unit_tests (a-o, ~20 files)"
|
|
@echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)"
|
|
@echo " make test-integration - Run integration tests"
|
|
@echo " make test-unit-helm - Run helm unit tests"
|
|
|
|
UV := uv
|
|
UV_RUN := $(UV) run --no-sync
|
|
|
|
# Show info
|
|
info:
|
|
@echo "UV: $(UV)"
|
|
|
|
# Installation targets
|
|
install-dev:
|
|
$(UV) sync --frozen
|
|
|
|
install-proxy-dev:
|
|
$(UV) sync --frozen --group proxy-dev --extra proxy
|
|
|
|
# CI-compatible installations (matches GitHub workflows exactly)
|
|
install-dev-ci:
|
|
$(UV) sync --frozen
|
|
|
|
install-proxy-dev-ci:
|
|
$(UV) sync --frozen --group proxy-dev --extra proxy
|
|
|
|
install-test-deps: install-proxy-dev
|
|
$(UV) sync --frozen --all-groups --all-extras
|
|
$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
|
|
|
|
install-helm-unittest:
|
|
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
|
|
|
|
# Install git hooks that enforce Conventional Commits and Conventional Branches.
|
|
# Opt-in: not chained into install-dev.
|
|
install-hooks:
|
|
./scripts/install_git_hooks.sh
|
|
|
|
# Formatting
|
|
format: install-dev
|
|
cd litellm && $(UV_RUN) black . && cd ..
|
|
|
|
format-check: install-dev
|
|
cd litellm && $(UV_RUN) black --check . && cd ..
|
|
|
|
# Linting targets
|
|
lint-ruff: install-dev
|
|
cd litellm && $(UV_RUN) ruff check . && cd ..
|
|
|
|
# faster linter for developing ...
|
|
# inspiration from:
|
|
# https://github.com/astral-sh/ruff/discussions/10977
|
|
# https://github.com/astral-sh/ruff/discussions/4049
|
|
lint-format-changed: install-dev
|
|
@git diff origin/main --unified=0 --no-color -- '*.py' | \
|
|
perl -ne '\
|
|
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
|
|
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
|
|
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
|
|
print "$$file:$$start:1-$$end:999\n"; \
|
|
}' | \
|
|
while read range; do \
|
|
file="$${range%%:*}"; \
|
|
lines="$${range#*:}"; \
|
|
echo "Formatting $$file (lines $$lines)"; \
|
|
$(UV_RUN) ruff format --range "$$lines" "$$file"; \
|
|
done
|
|
|
|
lint-ruff-dev: install-dev
|
|
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
|
|
cd litellm && \
|
|
($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \
|
|
$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
|
|
cd .. ; \
|
|
rm -f "$$tmpfile"
|
|
|
|
lint-ruff-FULL-dev: install-dev
|
|
@files=$$(git diff --name-only origin/main -- '*.py'); \
|
|
if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
|
|
else echo "No changed .py files to check."; fi
|
|
|
|
lint-basedpyright: install-dev
|
|
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py
|
|
|
|
lint-basedpyright-budget-update: install-dev
|
|
($(UV_RUN) basedpyright --outputjson || true) | $(UV_RUN) python scripts/type_check_gate.py --update
|
|
|
|
lint-black: format-check
|
|
|
|
lint-ruff-budget: install-dev
|
|
$(UV_RUN) python scripts/ruff_strict_gate.py
|
|
|
|
lint-ruff-budget-update: install-dev
|
|
$(UV_RUN) python scripts/ruff_strict_gate.py --update
|
|
|
|
# Ratchet all budgets in one shot (ruff strict + basedpyright)
|
|
lint-budget-update: lint-ruff-budget-update lint-basedpyright-budget-update
|
|
|
|
check-circular-imports: install-dev
|
|
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
|
|
|
|
check-import-safety: install-dev
|
|
@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
|
|
|
# Combined linting (matches test-linting.yml workflow)
|
|
lint: format-check lint-ruff lint-basedpyright check-circular-imports check-import-safety lint-ruff-budget
|
|
|
|
# Faster linting for local development (only checks changed code)
|
|
lint-dev: lint-format-changed check-circular-imports check-import-safety
|
|
|
|
# Testing targets
|
|
test: install-test-deps
|
|
$(UV_RUN) pytest tests/
|
|
|
|
test-unit: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
|
|
|
|
# Matrix test targets (matching CI workflow groups)
|
|
test-unit-llms: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-guardrails: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-core: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-proxy-misc: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-integrations: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-core-utils: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
|
|
|
|
test-unit-other: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
|
|
|
|
test-unit-root: install-test-deps
|
|
$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
|
|
|
|
# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
|
|
test-proxy-unit-a: install-test-deps
|
|
$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
|
|
|
|
test-proxy-unit-b: install-test-deps
|
|
$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
|
|
|
|
test-integration: install-test-deps
|
|
$(UV_RUN) pytest tests/ -k "not test_litellm"
|
|
|
|
test-unit-helm: install-helm-unittest
|
|
helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
|
|
|
|
# LLM Translation testing targets
|
|
test-llm-translation: install-test-deps
|
|
@echo "Running LLM translation tests..."
|
|
@python .github/workflows/run_llm_translation_tests.py
|
|
|
|
test-llm-translation-single: install-test-deps
|
|
@echo "Running single LLM translation test file..."
|
|
@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
|
|
@mkdir -p test-results
|
|
$(UV_RUN) pytest tests/llm_translation/$(FILE) \
|
|
--junitxml=test-results/junit.xml \
|
|
-v --tb=short --maxfail=100 --timeout=300
|
|
|
|
test-llm-translation-flush-vcr-cache:
|
|
$(UV_RUN) python tests/_flush_vcr_cache.py
|