diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index abdc1ecafbb..008a8f80cf0 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -46,7 +46,6 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, run by hand since its weekly workflow never had working Bedrock credentials), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The memory tests (`test_reliability_memory_e2e.py`, run by the perf lane on its own fresh stack: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches, the check for the v1.100.0 retry-breadcrumb leak). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares, or a `Ramp` that steps the rate up through the run, built with `ramped()`, whose plan carries each request's send second and whose run reports `sustained_rps`, the last step the pod held, scored in `perf/setup/ramp.py`, with every step's latency, completed rate, non-200s, CPU per request, Redis latency and Postgres connections cut from the samplers' time-stamped `Series` (`perf/setup/series.py`) to the step's window and emitted as `perf_ramp_step__` JUnit properties for the dashboard) and `Limit`s (one metric at one percentile under one ceiling, or over one floor for `sustained_rps`). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite -- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 4f0c7624e31..94fb0b3372f 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -92,31 +92,6 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. -### Secret manager lanes - -`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager__ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally: - -```bash -bash tests/e2e/secret_manager/backend.sh up cyberark -(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \ - LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000) -(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \ - uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v) -bash tests/e2e/secret_manager/backend.sh down cyberark -``` - -`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors - -A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane - -To add a backend, leave the tests and markers alone and add: - -1. `secret_manager/secret_store_.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E__*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports -2. its entry in `secret_backends.BACKENDS` -3. `gateway/secret_manager__ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed -4. an `up_` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env` -5. a CI step that runs `backend.sh up ` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env` - ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8fb96bef2f9..526651e88a1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -38,7 +38,6 @@ from e2e_config import ( PROXY_WIDE_CONFIG_OPT_IN_ENV, SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, SAGEMAKER_NOVA_OPT_IN_ENV, - SECRET_MANAGER_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV, unique_marker, ) @@ -74,7 +73,6 @@ OPT_IN_MARKERS: Final = MappingProxyType( "owned_gateway": OWNED_GATEWAY_OPT_IN_ENV, "otel_v2": OTEL_V2_OPT_IN_ENV, "otel_tls": OTEL_TLS_OPT_IN_ENV, - "secret_manager": SECRET_MANAGER_OPT_IN_ENV, "sagemaker_nova": SAGEMAKER_NOVA_OPT_IN_ENV, "sagemaker_nova2_lite": SAGEMAKER_NOVA2_LITE_OPT_IN_ENV, "proxy_wide_config": PROXY_WIDE_CONFIG_OPT_IN_ENV, @@ -181,11 +179,6 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set", ) - config.addinivalue_line( - "markers", - "secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live " - "secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)", - ) config.addinivalue_line( "markers", "sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; " diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index ff0f2263515..fc345786861 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -45,10 +45,6 @@ - {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"} - {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"} - {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"} -- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "A deployment whose api_key is os.environ/ gets its key from the configured secret manager when that name exists only in the manager"} -- {id: other.config.secret_resolution.manager_value_used, module: other, tier: P1, area: config, assertions: [manager_value_used], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "The value the manager holds is what reaches the provider: a bogus key in the manager is rejected by the provider with 401, so a passing resolution test cannot be an env fallback"} -- {id: other.config.secret_manager.virtual_key_stored, module: other, tier: P1, area: config, assertions: [virtual_key_stored], source: "key_management_event_hooks.py _store_virtual_key_in_secret_manager", rationale: "With store_virtual_keys, /key/generate writes the new key under prefix_for_stored_virtual_keys + key_alias in the manager"} -- {id: other.config.secret_manager.virtual_key_deleted, module: other, tier: P1, area: config, assertions: [virtual_key_deleted], source: "key_management_event_hooks.py _delete_virtual_keys_from_secret_manager", rationale: "/key/delete removes the stored key from the manager, so a revoked key does not linger there"} - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"} - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"} - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6316d22d3fa..bdd77d6fd6d 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -207,7 +207,6 @@ PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE" OWNED_GATEWAY_OPT_IN_ENV: Final = "E2E_OWNED_GATEWAY" OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2" OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT" -SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER" SAGEMAKER_NOVA_OPT_IN_ENV: Final = "SAGEMAKER_NOVA_ENDPOINT" SAGEMAKER_NOVA2_LITE_OPT_IN_ENV: Final = "SAGEMAKER_NOVA2_LITE_ENDPOINT" PROXY_WIDE_CONFIG_OPT_IN_ENV: Final = "E2E_PROXY_WIDE_CONFIG_STACK" diff --git a/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml b/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml deleted file mode 100644 index 32602783b02..00000000000 --- a/tests/e2e/gateway/secret_manager_cyberark_ci_config.yml +++ /dev/null @@ -1,8 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: true - key_management_system: cyberark - key_management_settings: - access_mode: read_and_write - store_virtual_keys: true - prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml b/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml deleted file mode 100644 index 8e6b7e74f8e..00000000000 --- a/tests/e2e/gateway/secret_manager_hashicorp_vault_ci_config.yml +++ /dev/null @@ -1,8 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: true - key_management_system: hashicorp_vault - key_management_settings: - access_mode: read_and_write - store_virtual_keys: true - prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/ diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 891c5fc282c..d0707d02d6c 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -17,7 +17,6 @@ markers = owned_gateway: boots its own proxy from source against the stack's Postgres, so it needs DATABASE_URL on the pytest host; deselected unless E2E_OWNED_GATEWAY is set otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set - secret_manager: needs a proxy booted from gateway/secret_manager__ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py) sagemaker_nova: needs a deployed SageMaker Nova endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA_ENDPOINT names it sagemaker_nova2_lite: needs a deployed SageMaker Nova 2 Lite endpoint the proxy's AWS credentials can invoke; deselected unless SAGEMAKER_NOVA2_LITE_ENDPOINT names it proxy_wide_config: turns on a proxy-wide setting through /config/update, so it runs on a stack no other suite is driving; deselected unless E2E_PROXY_WIDE_CONFIG_STACK is set diff --git a/tests/e2e/secret_manager/backend.sh b/tests/e2e/secret_manager/backend.sh deleted file mode 100755 index 70272c237c5..00000000000 --- a/tests/e2e/secret_manager/backend.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 - -usage() { - local systems - systems=$(declare -F | sed -n 's/^declare -f up_//p' | paste -sd '|' -) - echo "usage: $0 up|down $systems" >&2 - exit 2 -} - -action=${1:-} -system=${2:-} -dir=${E2E_SECRET_MANAGER_DIR:-$HOME/.cache/litellm-e2e-secret-manager}/$system -name=litellm-e2e-$system - -wait_for() { - local url=$1 - for _ in $(seq 1 90); do - if curl -sf -o /dev/null "$url"; then - return 0 - fi - sleep 2 - done - echo "$system did not answer at $url" >&2 - return 1 -} - -down() { - docker rm -f "$name" "$name-db" >/dev/null 2>&1 || true - docker network rm "$name" >/dev/null 2>&1 || true - rm -rf "$dir" -} - -up_hashicorp_vault() { - local port=${E2E_SECRET_MANAGER_PORT:-8200} - local token - token=e2e-$(openssl rand -hex 16) - docker run -d --name "$name" -p "127.0.0.1:$port:8200" --cap-add IPC_LOCK \ - -e VAULT_DEV_ROOT_TOKEN_ID="$token" hashicorp/vault:1.20 >/dev/null - wait_for "http://127.0.0.1:$port/v1/sys/health" - printf 'HCP_VAULT_ADDR=http://127.0.0.1:%s\nHCP_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/proxy.env" - printf 'E2E_VAULT_ADDR=http://127.0.0.1:%s\nE2E_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/tests.env" -} - -up_cyberark() { - local port=${E2E_SECRET_MANAGER_PORT:-8080} - local data_key api_key - docker network create "$name" >/dev/null - docker run -d --name "$name-db" --network "$name" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:15 >/dev/null - data_key=$(docker run --rm cyberark/conjur:1.24 data-key generate) - docker run -d --name "$name" --network "$name" -p "127.0.0.1:$port:80" \ - -e DATABASE_URL="postgres://postgres@$name-db/postgres" -e CONJUR_DATA_KEY="$data_key" \ - -e CONJUR_AUTHENTICATORS=authn cyberark/conjur:1.24 server >/dev/null - wait_for "http://127.0.0.1:$port/" - docker exec "$name" conjurctl account create --name default >/dev/null - api_key=$(docker exec "$name" conjurctl role retrieve-key default:user:admin | tr -d '\r\n') - printf 'CYBERARK_API_BASE=http://127.0.0.1:%s\nCYBERARK_ACCOUNT=default\nCYBERARK_USERNAME=admin\nCYBERARK_API_KEY=%s\n' \ - "$port" "$api_key" >"$dir/proxy.env" - printf 'E2E_CYBERARK_API_BASE=http://127.0.0.1:%s\nE2E_CYBERARK_ACCOUNT=default\nE2E_CYBERARK_USERNAME=admin\nE2E_CYBERARK_API_KEY=%s\n' \ - "$port" "$api_key" >"$dir/tests.env" -} - -[[ $# -eq 2 && -n $system ]] && declare -F "up_$system" >/dev/null || usage - -case $action in - up) - down - mkdir -p "$dir" - "up_$system" - echo "E2E_SECRET_MANAGER=$system" >>"$dir/tests.env" - echo "$system is up; env in $dir/proxy.env (proxy) and $dir/tests.env (pytest)" - ;; - down) down ;; - *) usage ;; -esac diff --git a/tests/e2e/secret_manager/conftest.py b/tests/e2e/secret_manager/conftest.py deleted file mode 100644 index 46ef598d711..00000000000 --- a/tests/e2e/secret_manager/conftest.py +++ /dev/null @@ -1,57 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass -from typing import Final - -import pytest - -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from proxy_client import ProxyClient -from secret_backends import BACKENDS, selected_backend -from secret_store import SecretBackend, SecretStore - -REQUIRES_CAPABILITY: Final = "requires_capability" - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line( - "markers", - f"{REQUIRES_CAPABILITY}(capability): secret_manager test deselected when the backend " - f"{SECRET_MANAGER_OPT_IN_ENV} names lacks the capability (secret_store.Capability)", - ) - - -def _lacks_capability(item: pytest.Item, backend: SecretBackend) -> bool: - marker: Final = item.get_closest_marker(REQUIRES_CAPABILITY) - return marker is not None and marker.args[0] not in backend.capabilities - - -def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: - backend: Final = BACKENDS.get(os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip()) - if backend is None: - return - deselected: Final = [item for item in items if _lacks_capability(item, backend)] - if deselected: - config.hook.pytest_deselected(items=deselected) - items[:] = [item for item in items if not _lacks_capability(item, backend)] - - -@dataclass(frozen=True, slots=True) -class SecretManagerClient: - proxy: ProxyClient - - -@pytest.fixture(scope="session") -def client(proxy: ProxyClient) -> SecretManagerClient: - return SecretManagerClient(proxy) - - -@pytest.fixture(scope="session") -def backend() -> SecretBackend: - return selected_backend() - - -@pytest.fixture(scope="session") -def store(backend: SecretBackend) -> SecretStore: - return backend.from_env() diff --git a/tests/e2e/secret_manager/secret_backends.py b/tests/e2e/secret_manager/secret_backends.py deleted file mode 100644 index 578b56970a3..00000000000 --- a/tests/e2e/secret_manager/secret_backends.py +++ /dev/null @@ -1,25 +0,0 @@ -from __future__ import annotations - -import os -from types import MappingProxyType -from typing import Final - -import pytest - -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from secret_store import SecretBackend -from secret_store_cyberark import CYBERARK -from secret_store_hashicorp_vault import HASHICORP_VAULT - -BACKENDS: Final = MappingProxyType({backend.system: backend for backend in (HASHICORP_VAULT, CYBERARK)}) - - -def selected_backend() -> SecretBackend: - system: Final = os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip() - backend: Final = BACKENDS.get(system) - if backend is None: - pytest.fail( - f"{SECRET_MANAGER_OPT_IN_ENV}={system!r} names no secret manager backend; " - f"set it to one of {sorted(BACKENDS)}" - ) - return backend diff --git a/tests/e2e/secret_manager/secret_store.py b/tests/e2e/secret_manager/secret_store.py deleted file mode 100644 index cff5005b73a..00000000000 --- a/tests/e2e/secret_manager/secret_store.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from typing import Final, Literal, Protocol - -SECRET_MANAGER_CONFIG_DIR: Final = "gateway" - - -class SecretStore(Protocol): - def write(self, name: str, value: str) -> None: ... - - def read(self, name: str) -> str | None: ... - - def destroy(self, name: str) -> None: ... - - -Capability = Literal["deletes_stored_keys"] - - -@dataclass(frozen=True, slots=True) -class SecretBackend: - system: str - from_env: Callable[[], SecretStore] - capabilities: frozenset[Capability] - - @property - def proxy_config(self) -> str: - return f"{SECRET_MANAGER_CONFIG_DIR}/secret_manager_{self.system}_ci_config.yml" diff --git a/tests/e2e/secret_manager/secret_store_cyberark.py b/tests/e2e/secret_manager/secret_store_cyberark.py deleted file mode 100644 index 22c7c458d86..00000000000 --- a/tests/e2e/secret_manager/secret_store_cyberark.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -import base64 -import os -import time -from dataclasses import dataclass, field -from typing import Final, Literal -from urllib.parse import quote - -import pytest -import yaml -from e2e_http import ExternalWrite, Headers, send_text_external -from e2e_metadata import step -from pydantic import Field - -from secret_store import SecretBackend - -CYBERARK_API_BASE_ENV: Final = "E2E_CYBERARK_API_BASE" -CYBERARK_ACCOUNT_ENV: Final = "E2E_CYBERARK_ACCOUNT" -CYBERARK_USERNAME_ENV: Final = "E2E_CYBERARK_USERNAME" -CYBERARK_API_KEY_ENV: Final = "E2E_CYBERARK_API_KEY" - -# The same defaults CyberArkSecretManager falls back to for CYBERARK_*. -DEFAULT_API_BASE: Final = "http://127.0.0.1:8080" -DEFAULT_ACCOUNT: Final = "default" -DEFAULT_USERNAME: Final = "admin" - -SYSTEM: Final = "cyberark" - -_POLICY_LOAD_ATTEMPTS: Final = 5 -_POLICY_LOAD_RETRY_DELAY_SECONDS: Final = 0.2 - -_START_HINT: Final = ( - f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " - f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" -) - - -class ConjurHeaders(Headers): - authorization: str = Field(repr=False) - content_type: str | None = Field(default=None, serialization_alias="Content-Type") - - -def _policy_scalar(name: str) -> str: - # Quoted the way CyberArkSecretManager._ensure_variable_exists quotes it. - return yaml.safe_dump(name, default_style='"').strip() - - -@dataclass(frozen=True, slots=True) -class Conjur: - base_url: str - account: str - username: str - api_key: str = field(repr=False) - - def _fail_unless_reached(self, result: ExternalWrite, action: str) -> None: - if result.status_code == -1: - pytest.fail(f"No live Conjur at {self.base_url}: {result.body}. {_START_HINT}") - if result.status_code == 401: - pytest.fail(f"Conjur rejected {self.username}'s credentials while trying to {action}. {_START_HINT}") - - def _headers(self, content_type: str | None = None) -> ConjurHeaders: - # Tokens last about eight minutes, so each call authenticates afresh rather than - # letting a long session outlive a cached one. - auth: Final = send_text_external( - "POST", - f"{self.base_url}/authn/{self.account}/{quote(self.username, safe='')}/authenticate", - headers=Headers(), - content=self.api_key, - ) - self._fail_unless_reached(auth, "authenticate") - if not auth.ok: - pytest.fail(f"Conjur refused to authenticate {self.username}: HTTP {auth.status_code} {auth.body[:300]}") - token: Final = base64.b64encode(auth.body.encode()).decode() - return ConjurHeaders(authorization=f'Token token="{token}"', content_type=content_type) - - def _secret_url(self, name: str) -> str: - return f"{self.base_url}/secrets/{self.account}/variable/{quote(name, safe='')}" - - def _load_root_policy(self, method: Literal["POST", "PATCH"], policy: str, attempt: int = 0) -> ExternalWrite: - result: Final = send_text_external( - method, - f"{self.base_url}/policies/{self.account}/policy/root", - headers=self._headers(content_type="application/x-yaml"), - content=policy, - ) - if result.status_code != 409 or attempt + 1 == _POLICY_LOAD_ATTEMPTS: - return result - time.sleep(_POLICY_LOAD_RETRY_DELAY_SECONDS * (1 << attempt)) - return self._load_root_policy(method, policy, attempt + 1) - - def _update_root_policy(self, method: Literal["POST", "PATCH"], policy: str, action: str) -> None: - result: Final = self._load_root_policy(method, policy) - self._fail_unless_reached(result, action) - if not result.ok: - pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}") - - @step("Write the secret {name} to CyberArk Conjur") - def write(self, name: str, value: str) -> None: - self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}") - result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value) - self._fail_unless_reached(result, f"write {name}") - if not result.ok: - pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}") - - @step("Read the secret {name} from CyberArk Conjur") - def read(self, name: str) -> str | None: - result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers()) - self._fail_unless_reached(result, f"read {name}") - if result.status_code == 404: - return None - if not result.ok: - pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}") - return result.body - - @step("Delete the secret {name} from CyberArk Conjur") - def destroy(self, name: str) -> None: - self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}") - - -def conjur_from_env() -> Conjur: - api_key: Final = os.environ.get(CYBERARK_API_KEY_ENV, "").strip() - if not api_key: - pytest.fail(f"The {SYSTEM} lane needs {CYBERARK_API_KEY_ENV} to reach its Conjur. {_START_HINT}") - return Conjur( - base_url=os.environ.get(CYBERARK_API_BASE_ENV, "").strip().rstrip("/") or DEFAULT_API_BASE, - account=os.environ.get(CYBERARK_ACCOUNT_ENV, "").strip() or DEFAULT_ACCOUNT, - username=os.environ.get(CYBERARK_USERNAME_ENV, "").strip() or DEFAULT_USERNAME, - api_key=api_key, - ) - - -CYBERARK: Final = SecretBackend(system=SYSTEM, from_env=conjur_from_env, capabilities=frozenset()) diff --git a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py b/tests/e2e/secret_manager/secret_store_hashicorp_vault.py deleted file mode 100644 index 5954719ccbb..00000000000 --- a/tests/e2e/secret_manager/secret_store_hashicorp_vault.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import os -from dataclasses import dataclass, field -from typing import Final - -import pytest -from e2e_http import ( - Headers, - NetworkError, - Success, - UnknownApiError, - delete_external, - get_external, - post_json_external, -) -from e2e_metadata import step -from pydantic import BaseModel, Field - -from secret_store import SecretBackend - -VAULT_ADDR_ENV: Final = "E2E_VAULT_ADDR" -VAULT_TOKEN_ENV: Final = "E2E_VAULT_TOKEN" -VAULT_MOUNT_ENV: Final = "E2E_VAULT_MOUNT_NAME" - -DEFAULT_VAULT_ADDR: Final = "http://127.0.0.1:8200" -DEFAULT_MOUNT: Final = "secret" - -SYSTEM: Final = "hashicorp_vault" - -_START_HINT: Final = ( - f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for " - f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests" -) - - -class VaultHeaders(Headers): - x_vault_token: str = Field(serialization_alias="X-Vault-Token", repr=False) - - -class KvData(BaseModel): - key: str = Field(repr=False) - - -class KvWriteBody(BaseModel): - data: KvData - - -class KvReadData(BaseModel): - data: KvData - - -class KvReadResponse(BaseModel): - data: KvReadData - - -@dataclass(frozen=True, slots=True) -class Vault: - base_url: str - token: str = field(repr=False) - mount: str = DEFAULT_MOUNT - - def _headers(self) -> VaultHeaders: - return VaultHeaders(x_vault_token=self.token) - - def _data_url(self, name: str) -> str: - return f"{self.base_url}/v1/{self.mount}/data/{name}" - - def _metadata_url(self, name: str) -> str: - return f"{self.base_url}/v1/{self.mount}/metadata/{name}" - - @step("Write the secret {name} to HashiCorp Vault") - def write(self, name: str, value: str) -> None: - write: Final = post_json_external( - self._data_url(name), headers=self._headers(), json=KvWriteBody(data=KvData(key=value)) - ) - if write.status_code == -1: - pytest.fail(f"No live Vault at {self.base_url}: {write.body}. {_START_HINT}") - if not write.ok: - pytest.fail(f"Vault refused to write {name}: HTTP {write.status_code} {write.body[:300]}") - - @step("Read the secret {name} from HashiCorp Vault") - def read(self, name: str) -> str | None: - result: Final = get_external(self._data_url(name), headers=self._headers(), response_type=KvReadResponse) - match result: - case Success(data=body): - return body.data.data.key - case UnknownApiError(status_code=404): - return None - case NetworkError(message=message): - return pytest.fail(f"No live Vault at {self.base_url}: {message}. {_START_HINT}") - case _: - return pytest.fail(f"Vault refused to read {name}: {result}") - - @step("Delete the secret {name} from HashiCorp Vault") - def destroy(self, name: str) -> None: - write: Final = delete_external(self._metadata_url(name), headers=self._headers()) - if not write.ok and write.status_code != 404: - pytest.fail(f"Vault refused to destroy {name}: HTTP {write.status_code} {write.body[:300]}") - - -def vault_from_env() -> Vault: - token: Final = os.environ.get(VAULT_TOKEN_ENV, "").strip() - if not token: - pytest.fail(f"The hashicorp_vault lane needs {VAULT_TOKEN_ENV} to reach its Vault. {_START_HINT}") - return Vault( - base_url=os.environ.get(VAULT_ADDR_ENV, DEFAULT_VAULT_ADDR).rstrip("/"), - token=token, - mount=os.environ.get(VAULT_MOUNT_ENV, "").strip() or DEFAULT_MOUNT, - ) - - -HASHICORP_VAULT: Final = SecretBackend( - system=SYSTEM, - from_env=vault_from_env, - capabilities=frozenset({"deletes_stored_keys"}), -) diff --git a/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py b/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py deleted file mode 100644 index 5bd67abd2f6..00000000000 --- a/tests/e2e/secret_manager/test_aws_secret_manager_e2e.py +++ /dev/null @@ -1,34 +0,0 @@ -import hashlib -import os -from typing import Final - -import pytest -from e2e_config import SECRET_MANAGER_OPT_IN_ENV -from e2e_metadata import Domain, Subject, meta -from pydantic import TypeAdapter - -from litellm.secret_managers.aws_secret_manager_v2 import AWSSecretsManagerV2 -from litellm.secret_managers.main import get_secret - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.secret_manager, - pytest.mark.skipif( - os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip() != "aws", - reason=f"runs only on the AWS Secrets Manager lane ({SECRET_MANAGER_OPT_IN_ENV}=aws)", - ), -] - -_AWS_FIXTURE_MASTER_KEY_SHA256: Final = "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b" -_STRING_MAP: Final = TypeAdapter(dict[str, str]) -_SECRET_STRING: Final = TypeAdapter(str) - - -class TestAwsSecretManager: - @meta(Subject(domain=Domain.DEPLOY_OPS)) - def test_aws_secret_manager(self) -> None: - AWSSecretsManagerV2.load_aws_secret_manager(use_aws_secret_manager=True) - secret_value: Final = _STRING_MAP.validate_json( - _SECRET_STRING.validate_python(get_secret("litellm_master_key")) - ) - assert hashlib.sha256(secret_value["litellm_master_key"].encode()).hexdigest() == _AWS_FIXTURE_MASTER_KEY_SHA256 diff --git a/tests/e2e/secret_manager/test_secret_manager_e2e.py b/tests/e2e/secret_manager/test_secret_manager_e2e.py deleted file mode 100644 index 8eaabcaae6f..00000000000 --- a/tests/e2e/secret_manager/test_secret_manager_e2e.py +++ /dev/null @@ -1,155 +0,0 @@ -from __future__ import annotations - -import os -import time -from collections.abc import Callable -from typing import Final - -import pytest - -from e2e_config import unique_marker -from e2e_http import Result, Success, UnauthorizedError, unwrap -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatResponse, KeyGenerateBody, LiteLLMParamsBody -from proxy_client import ProxyClient -from secret_store import SecretStore -from e2e_metadata import Domain, Mode, Provider, Subject, meta - -pytestmark = [pytest.mark.e2e, pytest.mark.secret_manager] - -BACKEND_MODEL: Final = "openai/gpt-4o-mini" -VIRTUAL_KEY_PREFIX: Final = "litellm-e2e/virtual-keys/" -PROVIDER_KEY_ENV: Final = "OPENAI_API_KEY" - - -# The proxy's env never holds OPENAI_API_KEY and each test seeds it under a fresh name, so a passing -# call proves the key came from the manager and not get_secret's os.environ fallback. -def _provider_key() -> str: - key: Final = os.environ.get(PROVIDER_KEY_ENV, "").strip() - if not key: - pytest.fail(f"The secret manager suite seeds the manager with the runner's {PROVIDER_KEY_ENV}, which is unset") - return key - - -def _seed(store: SecretStore, resources: ResourceManager, value: str) -> str: - name: Final = f"litellm-e2e-openai-{unique_marker()}" - store.write(name, value) - resources.defer(lambda: store.destroy(name)) - return name - - -def _deploy(proxy: ProxyClient, resources: ResourceManager, secret_name: str) -> str: - model_name: Final = f"secret-manager-backed-{unique_marker()}" - model_id: Final = proxy.create_model( - model_name, - LiteLLMParamsBody(model=BACKEND_MODEL, api_key=f"os.environ/{secret_name}"), - provider_live=True, - ) - resources.defer(lambda: proxy.delete_model(model_id)) - return model_name - - -def _chat(proxy: ProxyClient, key: str, model: str) -> Result[ChatResponse]: - return proxy.chat( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], - max_tokens=16, - ), - ) - - -def _eventually(proxy: ProxyClient, read: Callable[[], str | None], expected: str | None, context: str) -> None: - deadline: Final = time.monotonic() + proxy.poll_timeout - last: str | None = read() - while last != expected and time.monotonic() < deadline: - time.sleep(proxy.poll_interval) - last = read() - if last != expected: - pytest.fail( - f"{context}: the secret manager still holds {'a value' if last is not None else 'nothing'} after the deadline" - ) - - -class TestSecretManager: - @pytest.mark.covers("other.config.secret_resolution.kms_integration") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_deployment_key_resolves_from_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str - ) -> None: - model: Final = _deploy(proxy, resources, _seed(store, resources, _provider_key())) - - response: Final = unwrap(_chat(proxy, scoped_key, model)) - - assert response.choices, f"the manager-backed deployment answered with no choices: {response}" - - @pytest.mark.covers("other.config.secret_resolution.manager_value_used") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - providers=(Provider.OPENAI,), - models=(BACKEND_MODEL,), - mode=Mode.NONSTREAM, - ) - ) - def test_deployment_uses_the_value_the_manager_holds( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore, scoped_key: str - ) -> None: - bogus: Final = f"sk-litellm-e2e-not-a-key-{unique_marker()}" - model: Final = _deploy(proxy, resources, _seed(store, resources, bogus)) - - result: Final = _chat(proxy, scoped_key, model) - - match result: - case UnauthorizedError(body=body): - assert "AuthenticationError" in body, f"the 401 did not come from the provider: {body[:300]}" - case Success(): - pytest.fail("a deployment whose managed secret is not a real key still reached the provider") - case _: - pytest.fail(f"expected the provider to reject the manager-held key with 401, got {result}") - - @pytest.mark.covers("other.config.secret_manager.virtual_key_stored") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - ) - ) - def test_generated_key_is_written_to_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore - ) -> None: - alias: Final = f"litellm-e2e-vk-{unique_marker()}" - secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" - resources.defer(lambda: store.destroy(secret_name)) - key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) - resources.defer(lambda: proxy.delete_key(key)) - - _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") - - @pytest.mark.requires_capability("deletes_stored_keys") - @pytest.mark.covers("other.config.secret_manager.virtual_key_deleted") - @meta( - Subject( - domain=Domain.DEPLOY_OPS, - ) - ) - def test_deleted_key_is_removed_from_the_manager( - self, proxy: ProxyClient, resources: ResourceManager, store: SecretStore - ) -> None: - alias: Final = f"litellm-e2e-vk-{unique_marker()}" - secret_name: Final = f"{VIRTUAL_KEY_PREFIX}{alias}" - resources.defer(lambda: store.destroy(secret_name)) - key: Final = proxy.generate_key(KeyGenerateBody(key_alias=alias)) - resources.defer(lambda: proxy.delete_key(key)) - _eventually(proxy, lambda: store.read(secret_name), key, f"the generated key {alias}") - - proxy.delete_key(key) - - _eventually(proxy, lambda: store.read(secret_name), None, f"the deleted key {alias}")