From 8dc68bc32df35d5d87acf5f3792ca760d32c534e Mon Sep 17 00:00:00 2001 From: yuneng Date: Sun, 4 Oct 2026 07:14:59 +0000 Subject: [PATCH] ci: unset provider keys in the CircleCI MCP job and drop unused e2e-stack helpers --- .circleci/config.yml | 3 +- .github/e2e-stack/down.sh | 17 -- .github/e2e-stack/redact_output.py | 83 ------ .github/e2e-stack/secrets_to_env.py | 55 ---- .github/e2e-stack/select_tests.py | 52 ---- .github/e2e-stack/up.sh | 231 ----------------- .../test_e2e_changed_gate.py | 236 ------------------ 7 files changed, 2 insertions(+), 675 deletions(-) delete mode 100755 .github/e2e-stack/down.sh delete mode 100644 .github/e2e-stack/redact_output.py delete mode 100644 .github/e2e-stack/secrets_to_env.py delete mode 100644 .github/e2e-stack/select_tests.py delete mode 100755 .github/e2e-stack/up.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 34fbb39425b..f2d51ba7a0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3407,7 +3407,8 @@ jobs: name: Run MCP integration tests command: | mkdir -p test-results - timeout --signal=TERM 20m uv run --no-sync pytest \ + env -u OPENAI_API_KEY -u ANTHROPIC_API_KEY \ + timeout --signal=TERM 20m uv run --no-sync pytest \ tests/mcp_tests tests/unit/experimental_mcp_client tests/unit/proxy/_experimental/mcp_server \ tests/unit/responses/mcp --tb=short -vv --maxfail=10 -n 2 --dist=loadscope --reruns 0 \ --reruns-delay 1 --timeout=120 --rerun-except "from pytest-timeout" --durations=20 \ diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh deleted file mode 100755 index 740d626beea..00000000000 --- a/.github/e2e-stack/down.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" - -for pid_file in "${STACK_DIR}"/pids/*.pid; do - [[ -f "${pid_file}" ]] || continue - pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null - kill -TERM "$(cat "${pid_file}")" 2>/dev/null - rm -f "${pid_file}" -done - -for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do - docker rm -f "${container}" >/dev/null 2>&1 -done - -exit 0 diff --git a/.github/e2e-stack/redact_output.py b/.github/e2e-stack/redact_output.py deleted file mode 100644 index 233a8be3e2d..00000000000 --- a/.github/e2e-stack/redact_output.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import os -import sys -from functools import reduce -from pathlib import Path -from typing import Final -from xml.sax.saxutils import escape - -from pydantic import JsonValue, TypeAdapter, ValidationError -from secrets_to_env import MIN_MASKED_LENGTH - -REDACTED: Final = "***" -json_adapter: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) - - -def string_leaves(node: JsonValue) -> tuple[str, ...]: - match node: - case str(): - return (node,) - case list(): - return tuple(leaf for child in node for leaf in string_leaves(child)) - case dict(): - return tuple(leaf for child in node.values() for leaf in string_leaves(child)) - return () - - -def field_lines(value: str) -> tuple[str, ...]: - try: - return tuple(line for leaf in string_leaves(json_adapter.validate_json(value)) for line in leaf.splitlines()) - except ValidationError: - return () - - -def masked_values(values_files: tuple[Path, ...]) -> tuple[str, ...]: - values: Final = frozenset( - line.split("=", 1)[1].strip().strip("'") - for path in values_files - for line in path.read_text().splitlines() - if "=" in line - ) - texts: Final = frozenset(text for value in values for text in (value, *field_lines(value))) - renderings: Final = frozenset( - rendering - for text in texts - if len(text) >= MIN_MASKED_LENGTH - for rendering in (text, escape(text), escape(text, {'"': """})) - ) - return tuple(sorted(renderings, key=lambda rendering: (-len(rendering), rendering))) - - -def redact(text: str, values: tuple[str, ...]) -> str: - return reduce(lambda redacted, value: redacted.replace(value, REDACTED), values, text) - - -def write_redacted(source: Path, out_dir: Path, values: tuple[str, ...]) -> None: - target: Final = out_dir / source.name - with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600), "w") as handle: - _ = handle.write(redact(source.read_text(errors="replace"), values)) - - -def main() -> int: - parser: Final = argparse.ArgumentParser() - _ = parser.add_argument("--values", action="append", type=Path, required=True) - _ = parser.add_argument("--out", type=Path, required=True) - _ = parser.add_argument("files", nargs="*", type=Path) - args: Final = parser.parse_args() - values_files: Final = tuple(args.values) - out_dir: Final[Path] = args.out - sources: Final = tuple(args.files) - try: - values: Final = masked_values(values_files) - out_dir.mkdir(mode=0o700, exist_ok=True) - for source in sources: - write_redacted(source, out_dir, values) - except OSError as error: - _ = sys.stderr.write(f"could not redact {error.filename}\n") - return 1 - _ = sys.stdout.write(f"redacted {len(sources)} file(s) into {out_dir}\n") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py deleted file mode 100644 index 691c10203bf..00000000000 --- a/.github/e2e-stack/secrets_to_env.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import re -import sys -from pathlib import Path -from typing import Final - -from pydantic import TypeAdapter, ValidationError - -secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str]) -ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -MIN_MASKED_LENGTH: Final = 8 -ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS" - - -def main() -> int: - env_path: Final = Path(sys.argv[1]) - try: - secrets: Final = { - key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items() - } - except (ValidationError, UnicodeError): - _ = sys.stderr.write("expected a JSON object containing string environment values\n") - return 1 - unusable: Final = tuple( - key - for key, value in secrets.items() - if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") - ) - if unusable: - _ = sys.stderr.write( - f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n" - ) - return 1 - if os.environ.get(ACTIONS_RUNNER_FLAG) == "true": - _ = sys.stdout.write( - "".join( - f"::add-mask::{value.replace('%', '%25')}\n" - for value in secrets.values() - if len(value) >= MIN_MASKED_LENGTH - ) - ) - sys.stdout.flush() - lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) - try: - with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle: - os.fchmod(handle.fileno(), 0o600) - _ = handle.write("\n".join(lines) + "\n") - except OSError: - _ = sys.stderr.write("could not write the environment file\n") - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py deleted file mode 100644 index 792da5ae09c..00000000000 --- a/.github/e2e-stack/select_tests.py +++ /dev/null @@ -1,52 +0,0 @@ -import re -import sys -from typing import Final - -SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$") -UNSUPPORTED: Final = re.compile( - r"^tests/e2e/(ui|claude_code|load|migrations)/" - r"|^tests/e2e/mcp/test_mcp_oauth_happy_path_e2e\.py$" - r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$" - r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$" - r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$" - r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$" - r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$" - r"|^tests/e2e/logging/test_s3_log_e2e\.py$" - r"|^tests/e2e/secret_manager/" -) -HARNESS: Final = re.compile( - r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" - r"|^tests/e2e/idp_realm\.json$" - r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$" - r"|^tests/e2e/coverage_registry/management_cases\.py$" - r"|^tests/e2e/gateway/" - r"|^\.github/e2e-stack/" - r"|^\.github/workflows/test-e2e-changed\.yml$" -) -UNEXPANDED: Final = re.compile(r"[*?\[]") - - -def is_selectable(path: str) -> bool: - return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None - - -def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]: - direct: Final = frozenset(path for path in changed if is_selectable(path)) - harness_changed: Final = any(HARNESS.match(path) for path in changed) - canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path)) - return tuple(sorted(direct | canary_tests)) - - -def main() -> int: - canary: Final = tuple(sys.argv[1:]) - unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path)) - if unexpanded: - _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n") - return 1 - changed: Final = tuple(line.strip() for line in sys.stdin if line.strip()) - _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh deleted file mode 100755 index 931b5eb2169..00000000000 --- a/.github/e2e-stack/up.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" -CERTS_DIR="${STACK_DIR}/certs" -LOGS_DIR="${STACK_DIR}/logs" -PIDS_DIR="${STACK_DIR}/pids" - -POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" -VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}" -JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}" -NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}" - -LB_PORT="${E2E_LB_PORT:-4000}" -GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}" -GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}" -BACKEND_PORT="${E2E_BACKEND_PORT:-4001}" -REDIS_PORT="${E2E_REDIS_PORT:-6379}" -DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}" -DATABASE_PORT="${E2E_DATABASE_PORT:-5432}" -DATABASE_USER="${E2E_DATABASE_USER:-litellm}" -DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" -DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" -JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" -JAEGER_OTLP_TLS_PORT="${E2E_JAEGER_OTLP_TLS_PORT:-4319}" -JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" -KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" - -MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" - -mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" -chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" -chmod 755 "${CERTS_DIR}" - -log() { printf 'e2e-stack: %s\n' "$*"; } - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } - -wait_for() { - local label="$1" check="$2" deadline=$((SECONDS + ${3:-120})) - until eval "${check}"; do - if ((SECONDS >= deadline)); then - log "timed out waiting for ${label}" - exit 1 - fi - sleep 2 - done - log "${label} is up" -} - -if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then - set -a - source "${REPO_ROOT}/tests/e2e/.env" - set +a -fi - -if [[ -z "${DD_API_KEY:-}" ]]; then - log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env" - exit 1 -fi -export DD_SITE="${DD_SITE:-datadoghq.com}" - -if ! port_open "${DATABASE_PORT}"; then - docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \ - -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \ - "${POSTGRES_IMAGE}" >/dev/null -fi -wait_for "postgres" "port_open ${DATABASE_PORT}" - -if ! port_open "${JAEGER_QUERY_PORT}"; then - docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \ - "${JAEGER_IMAGE}" >/dev/null -fi -wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null" - -openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null -openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \ - -subj "/CN=litellm-e2e-ca" \ - -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \ - -out "${CERTS_DIR}/ca.crt" 2>/dev/null -openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null -openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null -openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \ - -CAcreateserial -days 7 -sha256 \ - -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \ - -out "${CERTS_DIR}/server.crt" 2>/dev/null -chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt - -CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')" -cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem" - -docker rm -f e2e-valkey >/dev/null 2>&1 || true -docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \ - "${VALKEY_IMAGE}" valkey-server \ - --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \ - --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \ - --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null -VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}" -wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG" -${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null -wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok" - -CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml" -CONFIG_PATH="${CONFIG_SOURCE}" -if [[ "${REDIS_PORT}" != "6379" ]]; then - CONFIG_PATH="${STACK_DIR}/litellm-config.yml" - sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}" -fi - -SERVER_ENV=( - "LITELLM_MASTER_KEY=${MASTER_KEY}" - "DATABASE_HOST=${DATABASE_HOST}" - "DATABASE_PORT=${DATABASE_PORT}" - "DATABASE_USER=${DATABASE_USER}" - "DATABASE_PASSWORD=${DATABASE_PASSWORD}" - "DATABASE_NAME=${DATABASE_NAME}" - "DISABLE_SCHEMA_UPDATE=true" - "REDIS_HOST=127.0.0.1" - "REDIS_PORT=${REDIS_PORT}" - "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]" - "CONFIG_FILE_PATH=${CONFIG_PATH}" - "STORE_MODEL_IN_DB=True" - "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" - "OTEL_EXPORTER_OTLP_ENDPOINT=https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}" - "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" - "PYTHONPATH=${REPO_ROOT}" - "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" - "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" - "JWT_AUDIENCE=litellm-e2e" -) -if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then - printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" - SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json") -fi - -cd "${REPO_ROOT}" - -env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh - -log "running migrations" -env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 - -start_server() { - local name="$1"; shift - env -u AWS_ROLE_NAME "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & - echo $! > "${PIDS_DIR}/${name}.pid" -} - -if [[ "$(uname)" == "Linux" ]]; then - NGINX_UPSTREAM_HOST=127.0.0.1 - NGINX_DOCKER_ARGS=(--network host) -else - NGINX_UPSTREAM_HOST=host.docker.internal - NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}" -p "${JAEGER_OTLP_TLS_PORT}:${JAEGER_OTLP_TLS_PORT}") -fi - -cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true -docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ - -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" \ - -v "${CERTS_DIR}:/certs:ro" "${NGINX_IMAGE}" >/dev/null - -wait_for "Jaeger OTLP TLS listener" \ - "curl -sS --cacert ${CERTS_DIR}/ca.crt https://127.0.0.1:${JAEGER_OTLP_TLS_PORT}/ -o /dev/null -w '%{http_code}' | grep -qE '^[2345]'" - -start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" -start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" -start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" - -wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 -wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 -wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300 -wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60 - -cat > "${STACK_DIR}/stack.env" < None: - env_path: Final = tmp_path / ".env" - - result: Final = subprocess.run( - [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], - input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', - capture_output=True, - text=True, - env={**os.environ, "GITHUB_ACTIONS": "true"}, - ) - - assert result.returncode == 0, result.stderr - assert result.stdout == "::add-mask::sk-0123456789abcdef\n" - assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" - - -def test_outside_actions_no_value_is_printed(tmp_path: Path) -> None: - env_path: Final = tmp_path / ".env" - local_env: Final = {key: value for key, value in os.environ.items() if key != "GITHUB_ACTIONS"} - - result: Final = subprocess.run( - [sys.executable, "-I", str(SECRETS_TO_ENV), str(env_path)], - input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}', - capture_output=True, - text=True, - env=local_env, - ) - - assert result.returncode == 0, result.stderr - assert result.stdout == "" - assert "sk-0123456789abcdef" not in result.stderr - assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n" - - -def redact_output(tmp_path: Path, values: tuple[str, ...], text: str) -> tuple[subprocess.CompletedProcess[str], Path]: - env_path: Final = tmp_path / ".env" - _ = env_path.write_text("".join(f"{name}='{value}'\n" for name, value in zip(("A", "B", "C"), values))) - stack_env: Final = tmp_path / "stack.env" - _ = stack_env.write_text("LITELLM_MASTER_KEY=sk-e2e-master0123\nREDIS_PORT=6379\n") - log: Final = tmp_path / "e2e-pass-1.log" - _ = log.write_text(text) - out_dir: Final = tmp_path / "redacted" - result: Final = subprocess.run( # test-quality-ok: standalone script that imports its sibling by script directory - [ - sys.executable, - str(REDACT_OUTPUT), - "--values", - str(env_path), - "--values", - str(stack_env), - "--out", - str(out_dir), - str(log), - ], - capture_output=True, - text=True, - ) - return result, out_dir / log.name - - -def test_redacted_output_hides_every_masked_value_and_keeps_the_rest(tmp_path: Path) -> None: - text: Final = ( - "FAILED key=sk-0123456789abcdef master=sk-e2e-master0123 flag=1 port=6379 message=Missing credentials\n" - ) - - result, redacted = redact_output(tmp_path, ("sk-0123456789abcdef", "1"), text) - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "FAILED key=*** master=*** flag=1 port=6379 message=Missing credentials\n" - assert (redacted.stat().st_mode & 0o777) == 0o600 - assert (tmp_path / "e2e-pass-1.log").read_text() == text - assert "sk-" not in result.stdout + result.stderr - - -def test_a_masked_value_that_prefixes_a_longer_one_leaves_no_tail(tmp_path: Path) -> None: - result, redacted = redact_output(tmp_path, ("sk-0123456789", "sk-0123456789abcdef"), "token sk-0123456789abcdef\n") - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "token ***\n" - - -def test_a_json_secret_is_hidden_field_by_field_however_it_is_escaped(tmp_path: Path) -> None: - credentials: Final = ( - '{"type": "service_account", "signing_key": "MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\n' - 'c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n", "client_id": "104857600000000000001"}' - ) - text: Final = ( - "decoded MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\n" - "c2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" - "escaped MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\\n\n" - "twice MIIEvAIBADANBgkqhkiG9w0BAQEFAASC\\\\nc2VjcmV0LWtleS1ib2R5LWxpbmUtdHdv\n" - "client 104857600000000000001 status 403\n" - ) - - result, redacted = redact_output(tmp_path, (credentials,), text) - - assert result.returncode == 0, result.stderr - assert redacted.read_text() == "decoded ***\n***\nescaped ***\\n***\\n\ntwice ***\\\\n***\nclient *** status 403\n" - - -def test_a_secret_with_xml_special_characters_is_hidden_in_the_junit_file(tmp_path: Path) -> None: - text: Final = 'body p&ss<w"rd-1\n' - - result, redacted = redact_output(tmp_path, ('p&ssbody ***\n' - - -def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]: - result: Final = subprocess.run( - [sys.executable, str(SELECT_TESTS), *CANARY], - input="".join(f"{path}\n" for path in changed), - capture_output=True, - text=True, - ) - assert result.returncode == 0, result.stderr - return tuple(result.stdout.split()) - - -@pytest.mark.parametrize( - ("changed", "expected"), - ( - (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)), - (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()), - (("tests/e2e/migrations/test_startup.py", "tests/e2e/migrations/test_recovery.py"), ()), - (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()), - (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()), - (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()), - (("tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e.py",), ()), - ( - ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), - ("tests/e2e/logging/test_team_langfuse_callback_e2e.py",), - ), - ( - ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), - ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",), - ), - ( - ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), - ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",), - ), - (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/AGENTS.md"), ()), - ( - ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"), - ("tests/e2e/logging/test_datadog_e2e.py",), - ), - ), -) -def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( - changed: tuple[str, ...], expected: tuple[str, ...] -) -> None: - assert select_tests(changed) == expected - - -@pytest.mark.parametrize( - "harness_file", - ( - "tests/e2e/proxy_client.py", - "tests/e2e/conftest.py", - "tests/e2e/management/management_client.py", - "tests/e2e/management/jwt_actors.py", - "tests/e2e/management/conftest.py", - "tests/e2e/coverage_registry/management_cases.py", - "tests/e2e/pytest.ini", - "tests/e2e/gateway/stage_mirror_ci_config.yml", - ".github/e2e-stack/up.sh", - ".github/e2e-stack/start-idp.sh", - "tests/e2e/idp_realm.json", - ".github/workflows/test-e2e-changed.yml", - ), -) -def test_harness_changes_run_the_canary_suite(harness_file: str) -> None: - assert select_tests((harness_file, "litellm/router.py")) == CANARY - - -def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None: - assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY - - -def test_dedicated_migration_tests_do_not_suppress_shared_harness_canaries() -> None: - assert select_tests(("tests/e2e/migrations/test_startup.py", "tests/e2e/conftest.py")) == CANARY - - -def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None: - assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == ( - *CANARY, - "tests/e2e/logging/test_datadog_e2e.py", - ) - - -def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None: - assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py") - - -def test_a_canary_argument_the_shell_never_expanded_fails_the_selector() -> None: - result: Final = subprocess.run( - [sys.executable, str(SELECT_TESTS), "tests/e2e/access_control/test_*.py"], - input="tests/e2e/proxy_client.py\n", - capture_output=True, - text=True, - ) - - assert result.returncode == 1 - assert "tests/e2e/access_control/test_*.py" in result.stderr - assert result.stdout == "" - - -@pytest.mark.parametrize( - ("secrets", "offender", "unprintable"), - ( - ('{"AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", "BAD-NAME": "shibboleth"}', "BAD-NAME", "shibboleth"), - ("""{"AWS_SECRET_ACCESS_KEY": "quote'shibboleth"}""", "AWS_SECRET_ACCESS_KEY", "shibboleth"), - ('{"DD_API_KEY": "line\\nshibboleth"}', "DD_API_KEY", "shibboleth"), - ), -) -def test_an_unusable_secret_is_named_without_printing_its_value( - tmp_path: Path, secrets: str, offender: str, unprintable: str -) -> None: - env_path: Final = tmp_path / ".env" - - result: Final = subprocess.run( - [sys.executable, str(SECRETS_TO_ENV), str(env_path)], input=secrets, capture_output=True, text=True - ) - - assert result.returncode == 1 - assert offender in result.stderr - assert unprintable not in result.stderr - assert result.stdout == "" - assert not env_path.exists() - - @pytest.mark.parametrize("phase", ("setup", "call", "teardown")) @pytest.mark.parametrize("required_count", ("1", "4")) def test_oauth_failure_diagnostics_do_not_publish_private_payloads(