diff --git a/.circleci/config.yml b/.circleci/config.yml index 1798abe9de5..f2d51ba7a0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ commands: parameters: category: type: enum - enum: ["backend", "client", "provider-harness"] + enum: ["backend", "client", "provider-harness", "redis-compat"] default: "backend" steps: - run: @@ -229,6 +229,19 @@ commands: - wait_for_service: url: tcp://localhost:6379 timeout: "60" + install_codecov_cli: + steps: + - run: + name: Install Codecov CLI (pinned v11.3.1) + when: always + command: | + curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov + curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM + [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] + (cd /tmp && sha256sum -c codecov.SHA256SUM) + chmod +x /tmp/codecov + mkdir -p "$HOME/.local/bin" + mv /tmp/codecov "$HOME/.local/bin/codecov" start_openai_record_replay_proxy: description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." steps: @@ -3308,6 +3321,169 @@ jobs: - store_artifacts: path: test-results + postgres_suite: + parameters: + test_path: + type: string + seed: + type: boolean + coverage_flag: + type: string + default: "" + timeout_minutes: + type: integer + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + environment: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test + steps: + - checkout + - skip_if_unrelated_changes + - setup_litellm_test_deps + - start_postgres: + db_name: litellm_test + image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 + - run: + name: Generate Prisma client + command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + - when: + condition: << parameters.seed >> + steps: + - run: + name: Seed database schema + command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Run << parameters.test_path >> + command: | + mkdir -p test-results + coverage_args=() + if [ -n "<< parameters.coverage_flag >>" ]; then + coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) + fi + timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ + << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ + --junitxml=test-results/junit.xml "${coverage_args[@]}" + - when: + condition: << parameters.coverage_flag >> + steps: + - install_codecov_cli + - run: + name: Upload coverage + when: always + command: | + [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } + codecov upload-process --disable-search --fail-on-error -f coverage.xml \ + -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \ + -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github + - store_test_results: + path: test-results + - store_artifacts: + path: test-results + + mcp_integration: + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + environment: + COVERAGE_CORE: sysmon + LITELLM_LOCAL_MODEL_COST_MAP: "True" + steps: + - checkout + - skip_if_unrelated_changes + - setup_litellm_test_deps + - run: + name: Generate Prisma client + command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + - run: + name: Install MCP SDK1 peer + command: | + uv venv --python 3.12 .venv-mcp-peer + uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' + echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" + - run: + name: Run MCP integration tests + command: | + mkdir -p test-results + env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \ + timeout --signal=TERM 20m uv run --no-sync pytest \ + tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \ + tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \ + --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \ + --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml -o junit_family=xunit1 --junitxml=test-results/junit.xml + - install_codecov_cli + - run: + name: Upload MCP integration coverage + when: always + command: | + [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } + codecov upload-process --disable-search --fail-on-error -f coverage.xml \ + -F mcp-integration -C "$CIRCLE_SHA1" \ + -n "mcp-integration-${CIRCLE_BUILD_NUM}" --git-service github + - store_test_results: + path: test-results + - store_artifacts: + path: test-results + + redis_compat: + parameters: + redis_py: + type: string + machine: + image: ubuntu-2204:2024.04.1 + resource_class: large + working_directory: ~/project + steps: + - checkout + - skip_if_unrelated_changes: + category: redis-compat + - setup_litellm_test_deps + - run: + name: Pin redis-py version + command: | + uv pip install "redis==<< parameters.redis_py >>" + uv run --no-sync python -c "import redis; assert redis.__version__ == '<< parameters.redis_py >>', redis.__version__; print('redis-py', redis.__version__)" + - run: + name: Install redis-server 7.2.16 + command: | + mkdir -p "$HOME/.local/bin" + cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" + docker cp "${cid}":/usr/local/bin/redis-server "$HOME/.local/bin/redis-server" + docker rm "$cid" + redis-server --version | grep -q 'v=7.2.16' + - run: + name: Run Redis compatibility tests + command: | + mkdir -p test-results + env -u CASSETTE_REDIS_URL -u AZURE_CLIENT_ID -u AZURE_CLIENT_SECRET -u AZURE_TENANT_ID \ + timeout --signal=TERM 15m uv run --no-sync pytest \ + tests/unit/test_redis.py tests/unit/caching/test_redis_connection_pool.py \ + tests/unit/caching/test_redis_cluster_cache.py tests/unit/caching/test_evicted_client_closer.py \ + tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \ + tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \ + --tb=short -vv --reruns 2 --reruns-delay 1 --durations=20 --cov=./litellm \ + --cov-report=xml:coverage.xml -o junit_family=xunit1 --junitxml=test-results/junit.xml + - when: + condition: + equal: ["5.3.1", << parameters.redis_py >>] + steps: + - install_codecov_cli + - run: + name: Upload Redis compatibility coverage + when: always + command: | + [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } + codecov upload-process --disable-search --fail-on-error -f coverage.xml \ + -F redis-compat -C "$CIRCLE_SHA1" \ + -n "redis-compat-${CIRCLE_BUILD_NUM}" --git-service github + - store_test_results: + path: test-results + - store_artifacts: + path: test-results + workflows: migration_startup: when: << pipeline.parameters.run_migration_tests >> @@ -3376,6 +3552,39 @@ workflows: parameters: suite: [management, database] mode: [replica] + - postgres_suite: + name: proxy-behavior + test_path: tests/proxy_behavior + seed: true + coverage_flag: lens-postgres + timeout_minutes: 25 + - postgres_suite: + name: proxy-security + test_path: tests/proxy_security_tests + seed: true + timeout_minutes: 15 + - postgres_suite: + name: schema-migration + test_path: tests/proxy_migration_tests + seed: false + timeout_minutes: 20 + - postgres_suite: + name: roi-database + test_path: tests/integration/database/test_roi_observed.py + seed: false + coverage_flag: roi-postgres + timeout_minutes: 10 + - mcp_integration: + name: mcp-integration + - redis_compat: + name: redis-compat-<< matrix.redis_py >> + matrix: + parameters: + redis_py: + - "5.3.1" + - "6.4.0" + - "7.4.1" + - "8.0.1" build_and_test: unless: or: diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index 387197b65d7..273716025e5 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: classify_changes.sh }" +category="${1:?usage: classify_changes.sh }" has_client=false has_backend=false @@ -10,9 +10,14 @@ has_provider_harness=false has_cost_map=false has_mcp_dependencies=false has_windows_release=false +has_redis_compat=false outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue + case "$file" in + litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/local_testing/test_caching.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .circleci/config.yml | .circleci/scripts/classify_changes.sh | .circleci/scripts/path_filter.sh | pyproject.toml | uv.lock) + has_redis_compat=true ;; + esac case "$file" in *.md | *.mdx) : ;; pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) @@ -54,6 +59,9 @@ case "$category" in windows-release) [ "$has_windows_release" = true ] && echo run || echo skip ;; + redis-compat) + [ "$has_redis_compat" = true ] && echo run || echo skip + ;; backend) [ "$has_backend" = true ] && echo run || echo skip ;; diff --git a/.circleci/scripts/path_filter.sh b/.circleci/scripts/path_filter.sh index 3050674f562..ccc68a0750b 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.circleci/scripts/path_filter.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -uo pipefail -category="${1:?usage: path_filter.sh }" +category="${1:?usage: path_filter.sh }" here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index eea25e8e285..8ca45ba3f75 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -21,7 +21,7 @@ test_paths: Live-provider caching cases in tests/local_testing that remain outside CI. Jobs that glob that directory either deselect them (local_testing_part1 and part2 carry `-k "... and not caching and not cache"`) or keep only another keyword (langfuse, router, assistants). - Separately, test-redis-compat.yml selects two IAM cluster authentication tests in + Separately, the CircleCI redis-compat jobs select two IAM cluster authentication tests in test_caching.py by node ID. It does not run that file's other tests. The gap was eight files and 118 tests when measured 2026-08-20; the five keyless files now run in the caching-local shard, leaving live cases in these three. Measured 2026-08-21 with no provider diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh deleted file mode 100755 index 740d626beea..00000000000 --- a/.github/e2e-stack/down.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" - -for pid_file in "${STACK_DIR}"/pids/*.pid; do - [[ -f "${pid_file}" ]] || continue - pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null - kill -TERM "$(cat "${pid_file}")" 2>/dev/null - rm -f "${pid_file}" -done - -for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do - docker rm -f "${container}" >/dev/null 2>&1 -done - -exit 0 diff --git a/.github/e2e-stack/redact_output.py b/.github/e2e-stack/redact_output.py deleted file mode 100644 index 233a8be3e2d..00000000000 --- a/.github/e2e-stack/redact_output.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import os -import sys -from functools import reduce -from pathlib import Path -from typing import Final -from xml.sax.saxutils import escape - -from pydantic import JsonValue, TypeAdapter, ValidationError -from secrets_to_env import MIN_MASKED_LENGTH - -REDACTED: Final = "***" -json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) - - -def string_leaves(node: JsonValue) -> tuple[str, ...]: - match node: - case str(): - return (node,) - case list(): - return tuple(leaf for child in node for leaf in string_leaves(child)) - case dict(): - return tuple(leaf for child in node.values() for leaf in string_leaves(child)) - return () - - -def field_lines(value: str) -> tuple[str, ...]: - try: - return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines()) - except ValidationError: - return () - - -def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]: - values: Final = frozenset( - line.split("=", 1)[1].strip().strip("'") - for path in values_files - for line in path.read_text().splitlines() - if "=" in line - ) - texts: Final = frozenset(text for value in values for text in (value, *field_lines(value))) - renderings: Final = frozenset( - rendering - for text in texts - if len(text) >= MIN_MASKED_LENGTH - for rendering in (text, escape(text), escape(text, {'"': """})) - ) - return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering))) - - -def redact(text: str, values: tuple[str, ...]) -> str: - return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text) - - -def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None: - target: Final = out_dir / source.name - with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle: - _ = handle.write(redact(source.read_text(errors="replace"), values)) - - -def main() -> int: - parser: Final = argparse.ArgumentParser() - _ = parser.add_argument("--values", action="append", type=Path, required=True) - _ = parser.add_argument("--out", type=Path, required=True) - _ = parser.add_argument("files", nargs="*", type=Path) - args: Final = parser.parse_args() - values_files: Final = tuple(args.values) - out_dir: Final[Path] = args.out - sources: Final = tuple(args.files) - try: - values: Final = masked_values(values_files) - out_dir.mkdir(mode=0o700, exist_ok=True) - for source in sources: - write_redacted(source, out_dir, values) - except OSError as error: - _ = sys.stderr.write(f"could not redact {error.filename}\n") - return 1 - _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py deleted file mode 100644 index 691c10203bf..00000000000 --- a/.github/e2e-stack/secrets_to_env.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import re -import sys -from pathlib import Path -from typing import Final - -from pydantic import TypeAdapter, ValidationError - -secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) -ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -MIN_MASKED_LENGTH: Final = 8 -ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS" - - -def main() -> int: - env_path: Final = Path(sys.argv[1]) - try: - secrets: Final = { - key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items() - } - except (ValidationError, UnicodeError): - _ = sys.stderr.write("expected a JSON object containing string environment values\n") - return 1 - unusable: Final = tuple( - key - for key, value in secrets.items() - if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") - ) - if unusable: - _ = sys.stderr.write( - f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" - ) - return 1 - if os.environ.get(ACTIONS_RUNNER_FLAG) == "true": - _ = sys.stdout.write( - "".join( - f"::add-mask::{value.replace('%', '%25')}\n" - for value in secrets.values() - if len(value) >= MIN_MASKED_LENGTH - ) - ) - sys.stdout.flush() - lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) - try: - with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: - os.fchmod(handle.fileno(), 0o600) - _ = handle.write("\n".join(lines) + "\n") - except OSError: - _ = sys.stderr.write("could not write the environment file\n") - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py deleted file mode 100644 index 792da5ae09c..00000000000 --- a/.github/e2e-stack/select_tests.py +++ /dev/null @@ -1,52 +0,0 @@ -import re -import sys -from typing import Final - -SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$") -UNSUPPORTED: Final = re.compile( - r"^tests/e2e/(ui|claude_code|load|migrations)/" - r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$" - r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" - r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" - r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" - r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" - r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$" - r"|^tests/e2e/logging/test_s3_log_e2e\.py$" - r"|^tests/e2e/secret_manager/" -) -HARNESS: Final = re.compile( - r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" - r"|^tests/e2e/idp_realm\.json$" - r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$" - r"|^tests/e2e/coverage_registry/management_cases\.py$" - r"|^tests/e2e/gateway/" - r"|^\.github/e2e-stack/" - r"|^\.github/workflows/test-e2e-changed\.yml$" -) -UNEXPANDED: Final = re.compile(r"[*?\[]") - - -def is_selectable(path: str) -> bool: - return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None - - -def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]: - direct: Final = frozenset(path for path in changed if is_selectable(path)) - harness_changed: Final = any(HARNESS.match(path) for path in changed) - canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path)) - return tuple(sorted(direct | canary_tests)) - - -def main() -> int: - canary: Final = tuple(sys.argv[1:]) - unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path)) - if unexpanded: - _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n") - return 1 - changed: Final = tuple(line.strip() for line in sys.stdin if line.strip()) - _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh deleted file mode 100755 index 931b5eb2169..00000000000 --- a/.github/e2e-stack/up.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" -CERTS_DIR="${STACK_DIR}/certs" -LOGS_DIR="${STACK_DIR}/logs" -PIDS_DIR="${STACK_DIR}/pids" - -POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" -VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}" -JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}" -NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}" - -LB_PORT="${E2E_LB_PORT:-4000}" -GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}" -GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}" -BACKEND_PORT="${E2E_BACKEND_PORT:-4001}" -REDIS_PORT="${E2E_REDIS_PORT:-6379}" -DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}" -DATABASE_PORT="${E2E_DATABASE_PORT:-5432}" -DATABASE_USER="${E2E_DATABASE_USER:-litellm}" -DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" -DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" -JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" -JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}" -JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" -KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" - -MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" - -mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" -chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" -chmod 755 "${CERTS_DIR}" - -log() { printf 'e2e-stack: %s\n' "$*"; } - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } - -wait_for() { - local label="$1" check="$2" deadline=$((SECONDS + ${3:-120})) - until eval "${check}"; do - if ((SECONDS >= deadline)); then - log "timed out waiting for ${label}" - exit 1 - fi - sleep 2 - done - log "${label} is up" -} - -if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then - set -a - source "${REPO_ROOT}/tests/e2e/.env" - set +a -fi - -if [[ -z "${DD_API_KEY:-}" ]]; then - log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env" - exit 1 -fi -export DD_SITE="${DD_SITE:-datadoghq.com}" - -if ! port_open "${DATABASE_PORT}"; then - docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \ - -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \ - "${POSTGRES_IMAGE}" >/dev/null -fi -wait_for "postgres" "port_open ${DATABASE_PORT}" - -if ! port_open "${JAEGER_QUERY_PORT}"; then - docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \ - "${JAEGER_IMAGE}" >/dev/null -fi -wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null" - -openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null -openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \ - -subj "/CN=litellm-e2e-ca" \ - -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \ - -out "${CERTS_DIR}/ca.crt" 2>/dev/null -openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null -openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null -openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \ - -CAcreateserial -days 7 -sha256 \ - -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \ - -out "${CERTS_DIR}/server.crt" 2>/dev/null -chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt - -CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')" -cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem" - -docker rm -f e2e-valkey >/dev/null 2>&1 || true -docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \ - "${VALKEY_IMAGE}" valkey-server \ - --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \ - --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \ - --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null -VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}" -wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG" -${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null -wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok" - -CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml" -CONFIG_PATH="${CONFIG_SOURCE}" -if [[ "${REDIS_PORT}" != "6379" ]]; then - CONFIG_PATH="${STACK_DIR}/litellm-config.yml" - sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}" -fi - -SERVER_ENV=( - "LITELLM_MASTER_KEY=${MASTER_KEY}" - "DATABASE_HOST=${DATABASE_HOST}" - "DATABASE_PORT=${DATABASE_PORT}" - "DATABASE_USER=${DATABASE_USER}" - "DATABASE_PASSWORD=${DATABASE_PASSWORD}" - "DATABASE_NAME=${DATABASE_NAME}" - "DISABLE_SCHEMA_UPDATE=true" - "REDIS_HOST=127.0.0.1" - "REDIS_PORT=${REDIS_PORT}" - "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]" - "CONFIG_FILE_PATH=${CONFIG_PATH}" - "STORE_MODEL_IN_DB=True" - "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" - "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}" - "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" - "PYTHONPATH=${REPO_ROOT}" - "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" - "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" - "JWT_AUDIENCE=litellm-e2e" -) -if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then - printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" - SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json") -fi - -cd "${REPO_ROOT}" - -env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh - -log "running migrations" -env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 - -start_server() { - local name="$1"; shift - env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & - echo $! > "${PIDS_DIR}/${name}.pid" -} - -if [[ "$(uname)" == "Linux" ]]; then - NGINX_UPSTREAM_HOST=127.0.0.1 - NGINX_DOCKER_ARGS=(--network host) -else - NGINX_UPSTREAM_HOST=host.docker.internal - NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}") -fi - -cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true -docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ - -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \ - -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null - -wait_for "Jaeger OTLP TLS listener" \ - "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'" - -start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" -start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" -start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" - -wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 -wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 -wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300 -wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60 - -cat > "${STACK_DIR}/stack.env" < tuple[frozens browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes) circle_path: Final = repo_root / ".circleci/config.yml" circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {} + circle_test_path_tokens: Final = _invoked_test_tokens( + scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path" + ) steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ()) invoked: Final = any( ".circleci/scripts/run_integration.sh" in scalar.value @@ -609,9 +612,9 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens if any(_token_covers(token, path) for token in gha_tokens) ) + tuple( - Finding(path, "GitHub-owned integration contract has no invoking workflow") + Finding(path, "GitHub-owned integration contract has no invoking job") for path in sorted(github_files) - if not any(_token_covers(token, path) for token in gha_tokens) + if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens) ) + tuple( Finding(path, "GitHub-owned integration file is missing") diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 6d67bef44cb..a5a7787c569 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -79,12 +79,6 @@ on: description: "Unique name for the coverage artifact (must be unique per run)" required: true type: string - legacy-mcp-peer: - description: "Install the isolated SDK1 peer for MCP compatibility tests" - required: false - type: boolean - default: false - permissions: contents: read @@ -142,17 +136,10 @@ jobs: - name: Install dependencies if: steps.changes.outputs.decision != 'skip' timeout-minutes: 8 - env: - LEGACY_MCP_PEER: ${{ inputs.legacy-mcp-peer }} run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' - if [ "$LEGACY_MCP_PEER" = "true" ]; then - uv venv --python "${UV_PYTHON}" .venv-mcp-peer - uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "MCP_TEST_PEER_PYTHON=$GITHUB_WORKSPACE/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV" - fi - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml deleted file mode 100644 index 228e23f60d7..00000000000 --- a/.github/workflows/test-e2e-changed.yml +++ /dev/null @@ -1,265 +0,0 @@ -name: e2e-changed-tests - -on: - pull_request: - -concurrency: - group: e2e-changed-${{ github.event.pull_request.number }} - cancel-in-progress: true - -permissions: {} - -jobs: - detect: - name: Detect changed e2e tests - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read - outputs: - tests: ${{ steps.changed.outputs.tests }} - any: ${{ steps.changed.outputs.any }} - steps: - - name: Checkout the selector and the canary suite - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - sparse-checkout: | - .github/e2e-stack - tests/e2e/access_control - tests/e2e/management/test_jwt_management_e2e.py - tests/e2e/other/test_jwt_auth_e2e.py - persist-credentials: false - ref: ${{ github.sha }} - - - name: List the e2e test files this PR added or modified - id: changed - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - gh api "repos/${REPO}/pulls/${PR_NUMBER}" \ - --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \ - | grep -Fxq "${HEAD_SHA}" - files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \ - --jq '.[] | select(.status != "removed") | .filename')" - gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" - tests="$(printf '%s\n' "${files}" \ - | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \ - tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)" - echo "tests=${tests}" >> "${GITHUB_OUTPUT}" - if [ -n "${tests}" ]; then - echo "any=true" >> "${GITHUB_OUTPUT}" - echo "selected e2e tests: ${tests}" - else - echo "any=false" >> "${GITHUB_OUTPUT}" - echo "no changed e2e test files supported by this stack; nothing to run" - fi - - run: - name: Run changed e2e tests against the stage-mirror stack - needs: detect - if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - timeout-minutes: 90 - environment: e2e-changed - permissions: - contents: read - id-token: write - services: - postgres: - image: postgres:16.6 - env: - POSTGRES_USER: litellm - POSTGRES_PASSWORD: dbpassword9090 - POSTGRES_DB: litellm - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U litellm" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - jaeger: - image: jaegertracing/jaeger:2.10.0 - ports: - - 4318:4318 - - 16686:16686 - steps: - - name: Validate configuration - env: - ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} - run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; } - - - name: Checkout - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - ref: ${{ github.sha }} - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.13" - - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Cache the Rust build - uses: ./.github/actions/cache-cargo-build - - - name: Install dependencies - run: | - .github/scripts/uv_sync_with_retries.sh --frozen \ - --extra proxy --extra proxy-runtime --extra extra_proxy \ - --extra semantic-router --extra bedrock-realtime \ - --group ci --group proxy-dev --group e2e-dev - uv pip install "pipecat-ai[openai]==1.4.0" - - - name: Cache Prisma binaries - uses: ./.github/actions/cache-prisma-binaries - - - name: Generate Prisma client - run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Install Playwright chromium - run: uv run --no-sync playwright install --with-deps chromium - - - name: Configure AWS credentials - id: aws - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 - with: - role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} - aws-region: us-east-1 - role-session-name: litellm-e2e-changed-${{ github.run_id }} - role-duration-seconds: 900 - output-env-credentials: false - output-credentials: true - - - name: Fetch provider credentials from AWS Secrets Manager - env: - AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} - AWS_DEFAULT_REGION: us-east-1 - run: | - umask 077 - aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ - --query SecretString --output text \ - | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env - aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \ - --query SecretString --output text \ - | jq -R -s '{"LITELLM_LICENSE": .}' \ - | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env - - - name: Boot the stage-mirror stack - id: boot - run: | - umask 077 - if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then - echo "::error::stage-mirror stack failed to boot; raw logs are not published" - exit 1 - fi - - - name: Export stack environment - run: | - master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)" - echo "::add-mask::${master_key}" - cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}" - - - name: Run the selected tests three times - env: - TESTS: ${{ needs.detect.outputs.tests }} - E2E_FIXTURE_MODE: live - E2E_PROVIDER_EDGE_HOST_REACHABLE: '1' - E2E_OWNED_GATEWAY: '1' - COLUMNS: '400' - run: | - umask 077 - read -r -a test_files <<< "${TESTS}" - for pass in 1 2 3; do - report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" - log="${RUNNER_TEMP}/e2e-pass-${pass}.log" - echo "::group::pass ${pass} of 3" - set +e - uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \ - -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1 - status=$? - uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}" - verified=$? - set -e - grep -E '^(FAILED|ERROR) ' "${log}" || true - grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1 - echo "::endgroup::" - if [ "${status}" = "5" ]; then - echo "::error::the selected files collected no runnable tests, so nothing was verified" - exit 1 - fi - if [ "${status}" != "0" ]; then - echo "::error::pass ${pass} of 3 failed with exit code ${status}" - exit "${status}" - fi - if [ "${verified}" != "0" ]; then - echo "::error::pass ${pass} of 3 did not verify every selected file" - exit 1 - fi - echo "pass ${pass} of 3 passed" - done - - - name: Redact the pytest output - if: always() && steps.boot.outcome == 'success' - run: | - umask 077 - shopt -s nullglob - uv run --no-sync python .github/e2e-stack/redact_output.py \ - --values tests/e2e/.env --values "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" \ - --out "${RUNNER_TEMP}/e2e-redacted" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml - - - name: Keep the redacted pytest output - if: always() && steps.boot.outcome == 'success' - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: e2e-changed-pytest-output-${{ github.run_attempt }} - path: ${{ runner.temp }}/e2e-redacted - retention-days: 14 - if-no-files-found: ignore - - - name: Stop the stack - if: always() && steps.boot.outcome != 'skipped' - run: bash .github/e2e-stack/down.sh - - - name: Remove credentials and raw output - if: always() - run: | - rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml - rm -rf "${RUNNER_TEMP}/litellm-e2e-stack" "${RUNNER_TEMP}/e2e-redacted" - - gate: - name: e2e-changed-tests - needs: [detect, run] - if: always() - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Require three successful passes when tests changed - env: - DETECT_RESULT: ${{ needs.detect.result }} - ANY_TESTS: ${{ needs.detect.outputs.any }} - RUN_RESULT: ${{ needs.run.result }} - run: | - if [ "${DETECT_RESULT}" != "success" ]; then - echo "::error::changed-test detection did not succeed" - exit 1 - fi - if [ "${ANY_TESTS}" = "false" ]; then - echo "no changed e2e test files supported by this stack; nothing to run" - exit 0 - fi - if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then - echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch" - exit 1 - fi diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 8f70375a181..1772bdeabc6 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -5,6 +5,27 @@ on: branches: - main - "litellm_**" + paths: + - "**/pyproject.toml" + - "uv.lock" + - "uv.toml" + - ".python-version" + - "rust-toolchain.toml" + - "litellm-rust/**" + - "litellm/__init__.py" + - "litellm/proxy/proxy_server.py" + - "litellm/**/*mcp*" + - "litellm/**/*mcp*/**" + - "litellm/integrations/arize/**" + - "scripts/check_mcp_sdk_install.py" + - "tests/base_sdk_tests/**" + - ".github/workflows/test-mcp-dependency-resolution.yml" + - ".github/actions/detect-changes/**" + - ".github/actions/setup-uv-with-retries/**" + - ".github/actions/cache-cargo-build/**" + - ".github/scripts/detect_changes.sh" + - ".github/scripts/uv_sync_with_retries.sh" + - ".circleci/scripts/classify_changes.sh" permissions: contents: read diff --git a/.github/workflows/test-postgres.yml b/.github/workflows/test-postgres.yml deleted file mode 100644 index a1c639acbeb..00000000000 --- a/.github/workflows/test-postgres.yml +++ /dev/null @@ -1,185 +0,0 @@ -name: "Postgres Tests" - -on: - pull_request: - branches: - - main - - "litellm_**" - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - postgres: - name: ${{ matrix.shard }} - runs-on: ubuntu-latest - timeout-minutes: ${{ matrix.job-timeout-minutes }} - permissions: - contents: read - id-token: write - - services: - postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: litellm_test - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 10 - - strategy: - fail-fast: false - matrix: - include: - - shard: roi-database - test-path: "tests/integration/database/test_roi_observed.py" - seed: none - workers: 0 - timeout-minutes: 10 - job-timeout-minutes: 35 - - - shard: proxy-behavior - test-path: "tests/proxy_behavior" - seed: db-push - workers: 0 - timeout-minutes: 25 - job-timeout-minutes: 50 - - - shard: proxy-security - test-path: "tests/proxy_security_tests" - seed: db-push - workers: 0 - timeout-minutes: 15 - job-timeout-minutes: 40 - - - shard: schema-migration - test-path: "tests/proxy_migration_tests" - seed: none - workers: 0 - timeout-minutes: 20 - job-timeout-minutes: 45 - - env: - DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: 3 - with: - persist-credentials: false - - - name: Detect relevant changes - id: changes - timeout-minutes: 2 - uses: ./.github/actions/detect-changes - - - name: Set up Python - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 3 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 3 - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Cache uv dependencies - if: steps.changes.outputs.decision != 'skip' && github.ref == 'refs/heads/main' - timeout-minutes: 5 - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cache/uv - .venv - key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv-postgres- - - - name: Cache uv dependencies - if: steps.changes.outputs.decision != 'skip' && github.ref != 'refs/heads/main' - timeout-minutes: 5 - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cache/uv - .venv - key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv-postgres- - - - name: Install dependencies - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 12 - run: | - .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras - - - name: Cache Prisma binaries - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 3 - uses: ./.github/actions/cache-prisma-binaries - - - name: Generate Prisma client - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 5 - run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Seed database schema - if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none' - timeout-minutes: 10 - run: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - - name: Run tests - if: steps.changes.outputs.decision != 'skip' - timeout-minutes: ${{ matrix.timeout-minutes }} - env: - TEST_PATH: ${{ matrix.test-path }} - WORKERS: ${{ matrix.workers }} - PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || matrix.shard == 'roi-database' && '--cov=./litellm --cov-report=xml:coverage-roi-postgres.xml' || '' }} - run: | - if [ "${WORKERS}" = "0" ]; then - uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 - else - uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}" - fi - - - name: Upload Lens database coverage - if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled() - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 - with: - use_oidc: true - version: v11.3.1 - root_dir: ${{ github.workspace }} - files: coverage-lens-postgres.xml - flags: lens-postgres - fail_ci_if_error: true - - - name: Upload ROI database coverage - if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'roi-database' && !cancelled() - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 - with: - use_oidc: true - version: v11.3.1 - root_dir: ${{ github.workspace }} - files: coverage-roi-postgres.xml - flags: roi-postgres - fail_ci_if_error: true diff --git a/.github/workflows/test-redis-compat.yml b/.github/workflows/test-redis-compat.yml deleted file mode 100644 index d6cfacccace..00000000000 --- a/.github/workflows/test-redis-compat.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: "Unit Tests: Redis Client Version Compatibility" - -on: - pull_request: - branches: - - main - - "litellm_**" - paths: - - "litellm/_redis.py" - - "litellm/_redis_credential_provider.py" - - "litellm/caching/redis_cache.py" - - "litellm/caching/evicted_client_closer.py" - - "tests/unit/test_redis.py" - - "tests/local_testing/test_caching.py" - - "tests/unit/caching/test_redis_connection_pool.py" - - "tests/unit/caching/test_redis_cluster_cache.py" - - "tests/unit/caching/test_evicted_client_closer.py" - - ".github/workflows/test-redis-compat.yml" - - "pyproject.toml" - - "uv.lock" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - redis-compat: - name: "redis-py ${{ matrix.redis-version }}" - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - id-token: write - - strategy: - fail-fast: false - matrix: - # 5.3.1 is the version pinned in uv.lock (redisvl caps it below 6); the - # newer legs prove the inspect.signature introspection in litellm/_redis.py - # keeps extracting kwargs on the redis-py releases people actually run now. - # Only the exact release 6.0.0 is skipped: rq (pulled by the proxy extra) - # specifies `redis != 6`, which excludes 6.0.0 alone, so 6.4.0 stands in - # for the 6.x line. - redis-version: ["5.3.1", "6.4.0", "7.4.1", "8.0.1"] - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Install dependencies - run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra extra_proxy --extra semantic-router - - - name: Pin redis-py to the matrix version - env: - REDIS_VERSION: ${{ matrix.redis-version }} - run: | - uv pip install "redis==${REDIS_VERSION:?}" - uv run --no-sync python -c "import redis; assert redis.__version__ == '${REDIS_VERSION:?}', redis.__version__; print('redis-py', redis.__version__)" - - - name: Build Redis for cluster authentication tests - run: | - curl --fail --location --retry 3 https://download.redis.io/releases/redis-7.2.16.tar.gz -o "$RUNNER_TEMP/redis-7.2.16.tar.gz" - echo "960a8ec15e34ff40e57ff16837b26b33bd81f2da6d24497bb63de532a323a18e $RUNNER_TEMP/redis-7.2.16.tar.gz" | sha256sum --check - tar -xzf "$RUNNER_TEMP/redis-7.2.16.tar.gz" -C "$RUNNER_TEMP" - make -C "$RUNNER_TEMP/redis-7.2.16" -j2 MALLOC=libc OPTIMIZATION=-O1 redis-server - echo "$RUNNER_TEMP/redis-7.2.16/src" >> "$GITHUB_PATH" - - - name: Run redis unit tests - run: | - redis-server --version - uv run --no-sync pytest \ - tests/unit/test_redis.py \ - tests/unit/caching/test_redis_connection_pool.py \ - tests/unit/caching/test_redis_cluster_cache.py \ - tests/unit/caching/test_evicted_client_closer.py \ - tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_azure_credentials \ - tests/local_testing/test_caching.py::test_sync_cluster_authenticates_with_gcp_credentials \ - --tb=short -vv \ - --reruns 2 \ - --reruns-delay 1 \ - --durations=20 \ - --cov=./litellm --cov-report=xml:coverage-redis.xml - - - name: Upload Redis coverage - if: matrix.redis-version == '5.3.1' - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 - with: - use_oidc: true - files: coverage-redis.xml - flags: redis-compat - fail_ci_if_error: false diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index b769c8f6a3d..3f99cee1720 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -50,15 +50,6 @@ jobs: fail-fast: false matrix: include: - - shard: mcp-integration - artifact-name: mcp-integration - test-path: "tests/mcp_tests" - unit-flag: mcp-integration - workers: 2 - reruns: 0 - timeout-minutes: 20 - job-timeout-minutes: 60 - - shard: core-utils artifact-name: core-utils test-path: tests/unit/decisions @@ -302,4 +293,3 @@ jobs: timeout-minutes: ${{ matrix.timeout-minutes }} job-timeout-minutes: ${{ matrix.job-timeout-minutes }} artifact-name: ${{ matrix.artifact-name }} - legacy-mcp-peer: ${{ matrix.shard == 'mcp-integration' }} diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index 758d579f67d..c4d12d1282c 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -8,10 +8,6 @@ from typing import Final import pytest GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py" -SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py") -SELECT_TESTS: Final = GATE.with_name("select_tests.py") -REDACT_OUTPUT: Final = GATE.with_name("redact_output.py") -CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py") @@ -102,238 +98,6 @@ def test_one_passing_management_case_cannot_hide_a_missing_actor(tmp_path: Path, assert f"test_actor_subject_and_database_role[{omitted_role}]" in result.stdout -def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_path: Path) -> None: - env_path: Final = tmp_path / ".env" - - result: Final = subprocess.run( - [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], - input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', - capture_output=True, - text=True, - env={**os.environ, "GITHUB_ACTIONS": "true"}, - ) - - assert result.returncode == 0, result.stderr - assert result.stdout == "::add-mask::sk-0123456789abcdef\n" - assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" - - -def test_outside_actions_no_value_is_printed(tmp_path: Path) -> None: - env_path: Final = tmp_path / ".env" - local_env: Final = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} - - result: Final = subprocess.run( - [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], - input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', - capture_output=True, - text=True, - env=local_env, - ) - - assert result.returncode == 0, result.stderr - assert result.stdout == "" - assert "sk-0123456789abcdef" not in result.stderr - assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" - - -def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: - env_path: Final = tmp_path / ".env" - _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) - stack_env: Final = tmp_path / "stack.env" - _ = stack_env.write_text("LITELLM_MASTER_KEY=sk-e2e-master0123\nREDIS_PORT=6379\n") - log: Final = tmp_path / "e2e-pass-1.log" - _ = log.write_text(text) - out_dir: Final = tmp_path / "redacted" - result: Final = subprocess.run( # test-quality-ok: standalone script that imports its sibling by script directory - [ - sys.executable, - str(REDACT_OUTPUT), - "--values", - str(env_path), - "--values", - str(stack_env), - "--out", - str(out_dir), - str(log), - ], - capture_output=True, - text=True, - ) - return result, out_dir / log.name - - -def test_redacted_output_hides_every_masked_value_and_keeps_the_rest(tmp_path: Path) -> None: - text: Final = ( - "FAILED key=sk-0123456789abcdef master=sk-e2e-master0123 flag=1 port=6379 message=Missing credentials\n" - ) - - result, redacted = redact_output(tmp_path, ("sk-0123456789abcdef", "1"), text) - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "FAILED key=*** master=*** flag=1 port=6379 message=Missing credentials\n" - assert (redacted.stat().st_mode & 0o777) == 0o600 - assert (tmp_path / "e2e-pass-1.log").read_text() == text - assert "sk-" not in result.stdout + result.stderr - - -def test_a_masked_value_that_prefixes_a_longer_one_leaves_no_tail(tmp_path: Path) -> None: - result, redacted = redact_output(tmp_path, ("sk-0123456789", "sk-0123456789abcdef"), "token sk-0123456789abcdef\n") - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "token ***\n" - - -def test_a_json_secret_is_hidden_field_by_field_however_it_is_escaped(tmp_path: Path) -> None: - credentials: Final = ( - '{"type": "service_account", "signing_key": "MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\n' - 'c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n", "client_id": "104857600000000000001"}' - ) - text: Final = ( - "decoded MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\n" - "c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" - "escaped MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n\n" - "twice MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" - "client 104857600000000000001 status 403\n" - ) - - result, redacted = redact_output(tmp_path, (credentials,), text) - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "decoded ***\n***\nescaped ***\\n***\\n\ntwice ***\\\\n***\nclient *** status 403\n" - - -def test_a_secret_with_xml_special_characters_is_hidden_in_the_junit_file(tmp_path: Path) -> None: - text: Final = 'body p&ss<w"rd-1\n' - - result, redacted = redact_output(tmp_path, ('p&ssbody ***\n' - - -def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: - result: Final = subprocess.run( - [sys.executable, str(SELECT_TESTS), *CANARY], - input="".join(f"{path}\n" for path in changed), - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - return tuple(result.stdout.split()) - - -@pytest.mark.parametrize( - ("changed", "expected"), - ( - (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)), - (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()), - (("tests/e2e/migrations/test_startup.py", "tests/e2e/migrations/test_recovery.py"), ()), - (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), - (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), - (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), - (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), - ( - ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), - ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), - ), - ( - ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), - ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), - ), - ( - ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), - ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), - ), - (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/AGENTS.md"), ()), - ( - ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"), - ("tests/e2e/logging/test_datadog_e2e.py",), - ), - ), -) -def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( - changed: tuple[str, ...], expected: tuple[str, ...] -) -> None: - assert select_tests(changed) == expected - - -@pytest.mark.parametrize( - "harness_file", - ( - "tests/e2e/proxy_client.py", - "tests/e2e/conftest.py", - "tests/e2e/management/management_client.py", - "tests/e2e/management/jwt_actors.py", - "tests/e2e/management/conftest.py", - "tests/e2e/coverage_registry/management_cases.py", - "tests/e2e/pytest.ini", - "tests/e2e/gateway/stage_mirror_ci_config.yml", - ".github/e2e-stack/up.sh", - ".github/e2e-stack/start-idp.sh", - "tests/e2e/idp_realm.json", - ".github/workflows/test-e2e-changed.yml", - ), -) -def test_harness_changes_run_the_canary_suite(harness_file: str) -> None: - assert select_tests((harness_file, "litellm/router.py")) == CANARY - - -def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None: - assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY - - -def test_dedicated_migration_tests_do_not_suppress_shared_harness_canaries() -> None: - assert select_tests(("tests/e2e/migrations/test_startup.py", "tests/e2e/conftest.py")) == CANARY - - -def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None: - assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == ( - *CANARY, - "tests/e2e/logging/test_datadog_e2e.py", - ) - - -def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None: - assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py") - - -def test_a_canary_argument_the_shell_never_expanded_fails_the_selector() -> None: - result: Final = subprocess.run( - [sys.executable, str(SELECT_TESTS), "tests/e2e/access_control/test_*.py"], - input="tests/e2e/proxy_client.py\n", - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert "tests/e2e/access_control/test_*.py" in result.stderr - assert result.stdout == "" - - -@pytest.mark.parametrize( - ("secrets", "offender", "unprintable"), - ( - ('{"AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", "BAD-NAME": "shibboleth"}', "BAD-NAME", "shibboleth"), - ("""{"AWS_SECRET_ACCESS_KEY": "quote'shibboleth"}""", "AWS_SECRET_ACCESS_KEY", "shibboleth"), - ('{"DD_API_KEY": "line\\nshibboleth"}', "DD_API_KEY", "shibboleth"), - ), -) -def test_an_unusable_secret_is_named_without_printing_its_value( - tmp_path: Path, secrets: str, offender: str, unprintable: str -) -> None: - env_path: Final = tmp_path / ".env" - - result: Final = subprocess.run( - [sys.executable, str(SECRETS_TO_ENV), str(env_path)], input=secrets, capture_output=True, text=True - ) - - assert result.returncode == 1 - assert offender in result.stderr - assert unprintable not in result.stderr - assert result.stdout == "" - assert not env_path.exists() - - @pytest.mark.parametrize("phase", ("setup", "call", "teardown")) @pytest.mark.parametrize("required_count", ("1", "4")) def test_oauth_failure_diagnostics_do_not_publish_private_payloads( diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 920ca8b02a3..3fd13439fd8 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -125,7 +125,7 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py` -Current limits: Bedrock cannot be mounted in record or replay (SigV4 signs the Host header, so a rewritten api_base fails signature verification); a test that needs to observe the Converse body registers its own `LiveEdge` with `provider_edge_bedrock.bedrock_signer` re-signing the forwarded request, and carries the `provider_edge_host` opt-in marker because the gateway must reach the pytest host, which the Buildkite ephemeral stack cannot (the GitHub changed-e2e lane, whose gateways run on the runner, sets `E2E_PROVIDER_EDGE_HOST_REACHABLE`). Deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode +Current limits: Bedrock cannot be mounted in record or replay (SigV4 signs the Host header, so a rewritten api_base fails signature verification); a test that needs to observe the Converse body registers its own `LiveEdge` with `provider_edge_bedrock.bedrock_signer` re-signing the forwarded request, and carries the `provider_edge_host` opt-in marker because the gateway must reach the pytest host, which the Buildkite ephemeral stack cannot, so those tests are deselected unless `E2E_PROVIDER_EDGE_HOST_REACHABLE` is set, as on a local run whose gateway can reach the pytest host. Deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode ## Typing diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index fb2cf2dfa24..682bf41d7d9 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -73,7 +73,7 @@ The suites run against a live proxy, so bring one up first by running the litell uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v ``` - Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The GitHub Actions changed-test stack starts the same digest-pinned Keycloak through `.github/e2e-stack/start-idp.sh`, imports the checked-out realm, and exports the IdP URL and credentials in `stack.env`. Both runners configure issuer/audience validation and store the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. Both wait for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them + Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The runner configures issuer/audience validation and stores the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. It waits for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`). The suites' client dependencies (the provider SDKs, websockets) live in the `e2e-dev` dependency group; `make bootstrap` installs it, and naming the group on the run keeps the command working from any environment state: @@ -103,20 +103,6 @@ Alternatively, start the proxy with `STORE_PROMPTS_IN_SPEND_LOGS=true litellm -- A couple of logging destinations are configured on the proxy rather than by the test. The Weave tests scope their callback to the key they create, but litellm builds the `weave_otel` logger from `WANDB_API_KEY` and `WANDB_PROJECT_ID` before it applies the per-key vars, so the proxy needs both in its own environment or the key-scoped callback never initializes and nothing ships -### The pull request check - -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. The Buildkite PR stack exports its two gateway pods the same way and, because those pods sit behind one router base that also fronts the backend, names that base in `LITELLM_CONTROL_PLANE_REPLICA_URLS` so management read-backs poll the plane that serves them instead of the gateway pods, which trim management routes at startup and answer them 404. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, `load/`, and `secret_manager/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set. The `secret_manager/` lanes each need a proxy configured against their own secret manager (see Secret manager lanes below) - -Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch - -Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code - -Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need - -Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, pytest's summary line, failed test ids, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs - -To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use - ### Secret manager lanes `key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager__ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally: @@ -319,8 +305,7 @@ same-repository branch for their verification. The workflow's path-filtered check is not configured here as a globally required branch-protection check. Provision `E2E_LINEAR_STORAGE_STATE_B64` as a secret there and retain the existing E2E license/AWS role configuration. A missing or expired session fails the job; collection, deselection and skips are not passes. -The generic changed-test job excludes this file because it requires an owned -proxy and consent UI. No LLM call is needed +No LLM call is needed Coverage remains limited to authorization-code OAuth over HTTP. M2M, OBO, PKCE passthrough, static/BYOK, ID-JAG, forwarding, SigV4 and stdio are outside this diff --git a/tests/integration/README.md b/tests/integration/README.md index fe31e215f8a..c53220cc895 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The ROI database contracts in `database/test_roi_observed.py` run in the GitHub Actions `roi-database` Postgres shard and upload coverage on each PR. They own temporary databases and script only the external provider transport. `GITHUB_FILES` in `run.py` assigns these files to GitHub Actions and excludes them from the CircleCI selection +The ROI database contracts in `database/test_roi_observed.py` run as plain pytest outside `run_integration.sh` in CircleCI's `roi-database` Postgres job. The job uploads coverage with the `roi-postgres` flag. These contracts own temporary databases and script only the external provider transport The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, and add hand-computed expected values. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` @@ -14,7 +14,7 @@ The generated lifecycle models use 20 examples, eight steps, generation and shri Reuse the existing canned provider handlers through `_support/upstream.py`. It rejects internal request fields and exposes actual received requests for independent assertions. Register every created resource for cleanup immediately, keep expected values independent of production calculations, and assert readback plus the runtime effect of a change -The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. GitHub Actions runs only the explicit `GITHUB_FILES` set in `run.py` +The CircleCI workflow starts its own database and Redis, restricts test-phase egress to its owned services and writes JUnit plus an executed-node manifest. Missing setup, failed cleanup or a selected test with neither a passed call nor a skip fail qualification. Skipped nodes are listed under `skipped` in `execution.json`, so the skip reasons double as the open bug list. `GITHUB_FILES` in `run.py` lists the files excluded from the `run_integration.sh` selection There is no per-node manifest. A positional argument is a file of the group or a pytest node id inside one (`path::test[param]`), so one cell of a parametrized file can run alone. The runner fails only when pytest fails, when collection errors, or when a selected file collects zero tests. Older tests still carry `@pytest.mark.covers(...)` decorators; the marker stays registered so they collect, but the IDs are not checked against anything and new tests should not use it. The GitHub Actions coverage census reads the `GROUPS` literal in `run.py` and treats every `tests/integration//test_*.py` file in a scheduled group as owned by CircleCI diff --git a/tests/unit/test_assert_ci_coverage.py b/tests/unit/test_assert_ci_coverage.py index 5f3903a5feb..e4233dbe682 100644 --- a/tests/unit/test_assert_ci_coverage.py +++ b/tests/unit/test_assert_ci_coverage.py @@ -84,8 +84,52 @@ def test_integration_groups_require_exclusive_scheduled_circleci_owner(tmp_path: workflow.write_text(yaml.safe_dump({"jobs": {}})) _, missing_invocation = coverage._integration_ownership(tmp_path) assert [(finding.subject, finding.detail) for finding in missing_invocation] == [ - (github_path, "GitHub-owned integration contract has no invoking workflow") + (github_path, "GitHub-owned integration contract has no invoking job") ] + circle_config: Final = yaml.safe_load(circle.read_text()) + circle_config["jobs"]["postgres_suite"] = {"parameters": {"test_path": {"type": "string"}}} + circle_config["workflows"]["integration"]["jobs"].append({"postgres_suite": {"test_path": github_path}}) + circle.write_text(yaml.safe_dump(circle_config)) + _, circle_invocation = coverage._integration_ownership(tmp_path) + assert circle_invocation == () + + +def test_circleci_postgres_suites_count_their_test_path_as_invoked() -> None: + config: Final = { + "workflows": { + "integration": { + "jobs": [ + {"postgres_suite": {"name": "proxy-behavior", "test_path": "tests/proxy_behavior", "seed": True}}, + { + "postgres_suite": { + "name": "roi-database", + "test_path": "tests/integration/database/test_roi_observed.py", + "seed": False, + } + }, + ] + } + } + } + assert coverage._invoked_test_tokens(coverage._scalars(config, "config.yml")) == frozenset( + {"tests/proxy_behavior", "tests/integration/database/test_roi_observed.py"} + ) + + +def test_every_circleci_postgres_suite_is_credited_on_the_repo_as_it_stands() -> None: + circle: Final = yaml.safe_load(coverage.CIRCLECI_CONFIG.read_text()) + paths: Final = tuple( + job["postgres_suite"]["test_path"] + for job in circle["workflows"]["integration"]["jobs"] + if isinstance(job, dict) and "postgres_suite" in job + ) + assert paths == ( + "tests/proxy_behavior", + "tests/proxy_security_tests", + "tests/proxy_migration_tests", + "tests/integration/database/test_roi_observed.py", + ) + assert set(paths) <= coverage._invoked_test_tokens(coverage._all_scalars()) def test_an_ancestor_directory_covers_a_file_but_does_not_name_it(): diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index 3776aea28e4..281790ee13e 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -137,6 +137,21 @@ def test_classify_decisions(category: str, changed: list[str], expected: str) -> assert classify(category, changed) == expected +@pytest.mark.parametrize( + ("changed", "expected"), + ( + ("litellm/caching/redis_cache.py", "run"), + ("tests/unit/caching/test_redis_cluster_cache.py", "run"), + (".circleci/config.yml", "run"), + ("uv.lock", "run"), + ("litellm/router.py", "skip"), + ("docs/my-website/redis.md", "skip"), + ), +) +def test_redis_compat_path_filter(changed: str, expected: str) -> None: + assert classify("redis-compat", [changed]) == expected + + def test_markdown_under_ui_counts_as_client_not_docs() -> None: assert classify("client", ["ui/litellm-dashboard/README.md"]) == "run" assert classify("backend", ["ui/litellm-dashboard/README.md"]) == "skip"