litellm/Makefile

# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks

.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
	test-rust-extension rust-sqlx-prepare \
	info lint lint-inner lint-dev lint-checks format \
	lint-basedpyright lint-e2e-basedpyright lint-type-discipline \
	lint-ruff-strict lint-gate lint-test-quality lint-openapi-docs \
	install-dev install-proxy-dev install-test-deps install-hooks \
	install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
	lint-install lint-fetch-base bootstrap

# Default target
help:
	@echo "Available commands:"
	@echo "  make bootstrap          - Provision a fresh clone/worktree"
	@echo "  make install-dev        - Install development dependencies"
	@echo "  make install-proxy-dev  - Install proxy development dependencies"
	@echo "  make install-dev-ci     - Install dev dependencies (CI-compatible, pins OpenAI)"
	@echo "  make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
	@echo "  make install-test-deps  - Install the full local test environment"
	@echo "  make install-helm-unittest - Install helm unittest plugin"
	@echo "  make install-hooks      - Install git hooks (Conventional Commits + Branches)"
	@echo "  make check              - Run CI-equivalent lint on staged files, or on the diff vs the base branch when nothing is staged"
	@echo "  make pre-commit         - Legacy alias for make check"
	@echo "  make format             - Apply ruff format code formatting"
	@echo "  make format-check       - Check ruff format code formatting (matches CI)"
	@echo "  make lint               - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
	@echo "  make lint-ruff          - Run Ruff linting only"
	@echo "  make lint-basedpyright  - Run basedpyright strict, gated by per-rule error counts"
	@echo "  make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)"
	@echo "  make lint-format        - Check ruff format formatting (matches CI)"
	@echo "  make lint-ruff-strict   - Gate each strict ruff rule's codebase total against its merge-base count"
	@echo "  make lint-gate        - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)"
	@echo "  make lint-test-quality  - Gate the test suite's TQ counts against their merge-base counts"
	@echo "  make lint-openapi-docs  - Gate undocumented POST routes in the proxy OpenAPI document against their merge-base counts"
	@echo "  make check-circular-imports - Check for circular imports"
	@echo "  make check-import-safety - Check import safety"
	@echo "  make test               - Run all tests"
	@echo "  make test-unit          - Run unit tests (tests/unit and tests/test_litellm)"
	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
	@echo "  make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
	@echo "  make test-unit-proxy-root - Run proxy root-file tests (tests/unit/proxy/test_*.py)"
	@echo "  make test-unit-integrations - Run integration tests (~60 files)"
	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
	@echo "  make test-proxy-unit-a  - Run tests/unit/proxy (a-o)"
	@echo "  make test-proxy-unit-b  - Run tests/unit/proxy (p-z)"
	@echo "  make test-integration   - Run integration tests"
	@echo "  make test-unit-helm     - Run helm unit tests"
	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
	@echo ""
	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."

UV := uv
UV_RUN := $(UV) run --no-sync
BASE_REF ?=
export BASE_REF
RESOLVE_BASE = python3 scripts/default_branch.py --base "$(BASE_REF)"

# Machine-wide slot queue for the heavy targets below; python3 + stdlib only, so
# it runs before any venv exists. See scripts/gate_slot_lock.py.
GATE_SLOT_LOCK := python3 scripts/gate_slot_lock.py

LINT_DEP_INSTALL ?= install-dev
LINT_E2E_DEP_INSTALL ?= lint-install
LINT_DEP_BASE ?=
LINT_JOBS := $(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)
LINT_OUTPUT_SYNC := $(if $(filter output-sync,$(.FEATURES)),--output-sync=target,)

# Show info
info:
	@echo "UV: $(UV)"

# Installation targets
# --inexact: sync the locked deps without pruning anything already installed, so running
# a lint/format target doesn't tear the proxy extras (prisma, websockets, ...) out from
# under a dev's venv (CI installs its own env per job, so it is unaffected by this).
install-dev:
	$(UV) sync --inexact --frozen

# Deliberately unqueued: provisioning is I/O bound, so it doesn't need one of the
# machine-wide slots the CPU-bound gates below share.
bootstrap:
	$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
	$(UV_RUN) python scripts/prisma_generate_if_needed.py
	cd ui/litellm-dashboard && ../../scripts/with_dashboard_node.sh npm install --no-audit --no-fund
	@main_root=$$(git worktree list --porcelain | head -1 | sed 's/^worktree //'); \
	if [ "$$main_root" != "$$(git rev-parse --show-toplevel)" ] && [ -f "$$main_root/.env" ] && [ ! -f .env ]; then \
		cp "$$main_root/.env" .env && echo "bootstrap: copied .env from $$main_root"; \
	else \
		echo "bootstrap: .env left untouched"; \
	fi
	@echo "bootstrap: done"

install-proxy-dev:
	$(UV) sync --frozen --group proxy-dev --extra proxy

# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
	$(UV) sync --frozen

install-proxy-dev-ci:
	$(UV) sync --frozen --group proxy-dev --extra proxy

install-test-deps: install-proxy-dev
	$(UV) sync --frozen --all-groups --all-extras
	$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma

install-helm-unittest:
	@helm plugin list | grep -qE '^unittest[[:space:]]+0\.8\.2([[:space:]]|$$)' || { \
		helm plugin uninstall unittest >/dev/null 2>&1 || true; \
		helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \
	}

# Install git hooks that enforce Conventional Commits and Conventional Branches.
# Opt-in: not chained into install-dev.
install-hooks:
	./scripts/install_git_hooks.sh

# Formatting
# Wrap width is ruff.toml's single source of truth (line-length = 120), shared by the
# formatter and the import sorter so there's no 88-vs-120 split to reconcile.
format: install-dev
	cd litellm && $(UV_RUN) ruff format --exclude '/enterprise/' . && cd ..

format-check: install-dev
	cd litellm && $(UV_RUN) ruff format --check --exclude '/enterprise/' . && cd ..

lint-fetch-base:
	@$(RESOLVE_BASE)

# Mirror test-linting.yml's python job environment: the proxy-dev group plus a generated
# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
# basedpyright gate itself no longer measures here (scripts/type_check_gate.py provisions its
# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
# gen:api and the running proxy need.
lint-install:
	$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
	$(UV_RUN) python scripts/prisma_generate_if_needed.py

# Diff-scoped format check, mirroring test-linting.yml's "Check ruff format" step:
# only the litellm Python files changed vs the base are checked, so a pre-existing
# format issue elsewhere doesn't block an unrelated commit. Git pathspecs match
# recursively, so 'litellm/*.py' covers top-level files and nested modules alike,
# the same set CI's ':(glob)litellm/**/*.py' selects.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	@base_ref=$$($(RESOLVE_BASE)) && \
	changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
	files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
	if [ -z "$$files" ]; then \
		echo "No changed litellm Python files to format-check."; \
	else \
		echo "$$files" | xargs $(UV_RUN) ruff format --check --exclude '/enterprise/'; \
	fi

# Linting targets
lint-ruff: $(LINT_DEP_INSTALL)
	$(UV_RUN) python scripts/check_mcp_operation_boundary.py
	cd litellm && $(UV_RUN) ruff check . && cd ..
	$(UV_RUN) ruff check --config ruff-tests.toml tests

# faster linter for developing ...
# inspiration from:
# https://github.com/astral-sh/ruff/discussions/10977
# https://github.com/astral-sh/ruff/discussions/4049
lint-format-changed: install-dev
	@base_ref=$$($(RESOLVE_BASE)) && \
	diff=$$(git diff "$$base_ref" --unified=0 --no-color -- '*.py') && \
	printf '%s\n' "$$diff" | \
	perl -ne '\
		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
			print "$$file:$$start:1-$$end:999\n"; \
		}' | \
		while read range; do \
			file="$${range%%:*}"; \
			lines="$${range#*:}"; \
			echo "Formatting $$file (lines $$lines)"; \
			$(UV_RUN) ruff format --range "$$lines" "$$file"; \
		done

lint-ruff-dev: install-dev
	@base_ref=$$($(RESOLVE_BASE)) || exit $$?; \
	tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
	cd litellm && \
	($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \
	$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch="$$base_ref" && \
	cd .. ; \
	rm -f "$$tmpfile"

lint-ruff-FULL-dev: install-dev
	@base_ref=$$($(RESOLVE_BASE)) && \
	files=$$(git diff --name-only "$$base_ref" -- '*.py') || exit $$?; \
	if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
	else echo "No changed .py files to check."; fi

lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	$(UV_RUN) python scripts/type_check_gate.py --base "$(BASE_REF)"

lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
	$(UV_RUN) basedpyright tests/e2e tests/e2e_harness

# Type-discipline gate (mutable collections / casts / type guards / kwargs /
# unexplained suppressions), the test-linting.yml step `make lint` used to omit.
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	$(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)"

# Test-quality gate (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation, credential-gated skips, conftest snapshot
# inventory), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	$(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)"

# OpenAPI docs gate (POST routes without a request body schema, a description or a
# 200 schema in app.openapi()), counted the same delta-vs-base way so a new endpoint
# has to show up documented in Swagger UI.
lint-openapi-docs: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	$(UV_RUN) python scripts/openapi_docs_gate.py --base "$(BASE_REF)"

lint-format: format-check

lint-ruff-strict: install-dev
	$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"

# Strict gate, invoked the same way CI does in test-linting.yml so a local pass
# means the CI check will pass too.
lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
	$(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)"

check-circular-imports: $(LINT_DEP_INSTALL)
	cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..

check-import-safety: $(LINT_DEP_INSTALL)
	@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)

# Combined linting, isomorphic to test-linting.yml's python job so a local pass means a
# green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then
# runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule /
# type-discipline / basedpyright gates as a delta vs the base, then the circular-import
# and import-safety checks. Steps that compare against the base resolve it the same way CI
# does (merge-base with origin's current default branch). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint:
	@$(GATE_SLOT_LOCK) $(MAKE) lint-inner

lint-inner: lint-install
	@base_ref=$$($(RESOLVE_BASE)) && \
	$(MAKE) BASE_REF="$$base_ref" -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks

lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-openapi-docs lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety

# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety

# Run the gating CI checks against your changes. Scopes to staged files when anything
# is staged (warning about changed files left unstaged); with nothing staged it falls
# back to the working tree's diff against the merge base with the base branch, so a
# fresh merge commit or an unstaged working tree still gets checked. Mirrors
# test-linting.yml (Python, UI, and API types), skipping any whose files aren't in scope.
# Not auto-installed as a git hook so it never slows an unrelated human commit.
check:
	@$(GATE_SLOT_LOCK) $(MAKE) check-inner

check-inner: bootstrap
	./scripts/pre_commit_lint.sh

pre-commit:
	@echo "make pre-commit is a legacy alias; use make check" >&2
	@$(MAKE) check

# Testing targets
test-rust-extension:
	@temporary=$$(mktemp -d) && \
	trap 'rm -rf "$$temporary"' EXIT HUP INT TERM && \
	$(UV) build --python 3.12 --wheel --out-dir "$$temporary/wheels" && \
	set -- "$$temporary"/wheels/*.whl && \
	[ "$$#" -eq 1 ] && \
	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
		--mypy-config-file tests/unit/rust_bridge/stubtest.ini \
		litellm.rust_bridge._native && \
	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust

rust-sqlx-prepare:
	cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare

test: install-test-deps
	$(UV_RUN) pytest tests/

test-unit: install-test-deps
	$(UV_RUN) pytest tests/unit tests/test_litellm -x -vv -n 4

# Matrix test targets (matching CI workflow groups)
test-unit-llms: install-test-deps
	$(UV_RUN) pytest tests/unit/llms --tb=short -vv -n 4 --durations=20

test-unit-proxy-guardrails: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy/guardrails tests/unit/proxy/management_endpoints tests/unit/proxy/management_helpers --tb=short -vv -n 4 --durations=20

test-unit-proxy-core: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/unit/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py --tb=short -vv -n 4 --durations=20

test-unit-proxy-misc: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy/agent_endpoints tests/unit/proxy/anthropic_endpoints tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py tests/unit/proxy/discovery_endpoints tests/unit/proxy/experimental tests/unit/proxy/google_endpoints tests/unit/proxy/health_endpoints tests/unit/proxy/image_endpoints tests/unit/proxy/middleware --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/openai_files_endpoint tests/unit/proxy/pass_through_endpoints tests/unit/proxy/prompts tests/unit/proxy/public_endpoints tests/unit/proxy/response_api_endpoints tests/unit/proxy/shutdown tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py tests/unit/proxy/ui_crud_endpoints tests/unit/proxy/vector_store_endpoints tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py --ignore=tests/unit/proxy/google_endpoints/test_google_endpoint_routing.py --ignore=tests/unit/proxy/google_endpoints/test_google_gemini_proxy_request.py --ignore=tests/unit/proxy/public_endpoints/test_blog_posts_endpoint.py --tb=short -vv -n 4 --durations=20

test-unit-proxy-root: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy/test_*.py --ignore=tests/unit/proxy/test_aproxy_startup.py --ignore=tests/unit/proxy/test_credential_slot_registry.py --ignore=tests/unit/proxy/test_custom_callback_input.py --ignore=tests/unit/proxy/test_custom_logger_s3_gcs.py --ignore=tests/unit/proxy/test_custom_tokenizer_bug.py --ignore=tests/unit/proxy/test_db_schema_changes.py --ignore=tests/unit/proxy/test_deprecated_key_grace_period.py --ignore=tests/unit/proxy/test_get_favicon.py --ignore=tests/unit/proxy/test_get_image.py --ignore=tests/unit/proxy/test_prisma_client_backoff_retry.py --ignore=tests/unit/proxy/test_prompt_test_endpoint.py --ignore=tests/unit/proxy/test_proxy_config_unit_test.py --ignore=tests/unit/proxy/test_proxy_custom_auth.py --ignore=tests/unit/proxy/test_proxy_reject_logging.py --ignore=tests/unit/proxy/test_proxy_server.py --ignore=tests/unit/proxy/test_proxy_setting_guardrails.py --ignore=tests/unit/proxy/test_proxy_token_counter.py --ignore=tests/unit/proxy/test_proxy_utils.py --ignore=tests/unit/proxy/test_reducto_ocr_route.py --ignore=tests/unit/proxy/test_response_polling_pre_call_checks.py --ignore=tests/unit/proxy/test_server_root_path.py --ignore=tests/unit/proxy/test_ui_path_detection.py --ignore=tests/unit/proxy/test_unit_test_proxy_hooks.py --ignore=tests/unit/proxy/test_update_spend.py --ignore=tests/unit/proxy/test_zero_cost_model_budget_bypass.py --tb=short -vv -n 4 --durations=20

test-unit-integrations: install-test-deps
	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20

test-unit-core-utils: install-test-deps
	$(UV_RUN) pytest tests/unit/litellm_core_utils --tb=short -vv -n 2 --durations=20

test-unit-other: install-test-deps
	$(UV_RUN) pytest tests/unit/caching tests/unit/responses tests/unit/secret_managers tests/unit/vector_stores tests/unit/a2a_protocol tests/unit/completion_extras tests/unit/containers tests/unit/enterprise tests/unit/experimental_mcp_client tests/unit/google_genai tests/unit/images tests/unit/interactions tests/unit/router_strategy tests/unit/router_utils tests/unit/types --tb=short -vv -n 4 --durations=20

test-unit-root: install-test-deps
	$(UV_RUN) pytest tests/unit/test_*.py tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20

# Proxy unit tests (tests/unit/proxy split alphabetically)
test-proxy-unit-a: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy --ignore-glob='tests/unit/proxy/test_[p-z]*.py' --tb=short -vv -n 2 --durations=20

test-proxy-unit-b: install-test-deps
	$(UV_RUN) pytest tests/unit/proxy/test_[p-z]*.py tests/unit/skills --tb=short -vv -n 2 --durations=20

test-integration: install-test-deps
	$(UV_RUN) pytest tests/ -k "not test_litellm"

test-unit-helm: install-helm-unittest
	helm unittest -f 'tests/*.yaml' helm/litellm-helm

# LLM Translation testing targets
test-llm-translation: install-test-deps
	@echo "Running LLM translation tests..."
	@python .github/scripts/run_llm_translation_tests.py

test-llm-translation-single: install-test-deps
	@echo "Running single LLM translation test file..."
	@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
	@mkdir -p test-results
	$(UV_RUN) pytest tests/llm_translation/$(FILE) \
		--junitxml=test-results/junit.xml \
		-v --tb=short --maxfail=100 --timeout=300

test-llm-translation-flush-vcr-cache:
	$(UV_RUN) python tests/_flush_vcr_cache.py