mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
feat(redis-client): decrypt an encrypted client key with ssl_password
rustls only reads unencrypted keys, so ssl_password was dropped with a warning and the TLS handshake failed. The resolver now reads it into TlsOptions and redis-client decrypts a PKCS#8 ENCRYPTED PRIVATE KEY before handing it to redis-rs, like the sync redis-py client's load_cert_chain(password=...). A missing or wrong password, or a legacy Proc-Type encrypted PEM key, fails at client build with a TlsKey error naming the file Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee
parent
b23980dc08
commit
febe994888
19 changed files with 390 additions and 49 deletions
108
litellm-rust/Cargo.lock
generated
108
litellm-rust/Cargo.lock
generated
|
|
@ -8,6 +8,17 @@ version = "2.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cipher",
|
||||
"cpufeatures 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
|
|
@ -982,6 +993,15 @@ dependencies = [
|
|||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-padding"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bollard"
|
||||
version = "0.20.2"
|
||||
|
|
@ -1134,6 +1154,15 @@ dependencies = [
|
|||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbc"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.3.0"
|
||||
|
|
@ -1210,6 +1239,16 @@ dependencies = [
|
|||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
|
|
@ -1774,6 +1813,15 @@ dependencies = [
|
|||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "des"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
|
|
@ -3105,6 +3153,16 @@ dependencies = [
|
|||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding",
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.12.0"
|
||||
|
|
@ -4372,6 +4430,7 @@ dependencies = [
|
|||
"litellm-redis-types",
|
||||
"litellm-secret-value",
|
||||
"pem",
|
||||
"pkcs8",
|
||||
"redis",
|
||||
"rstest",
|
||||
"rustls 0.23.42",
|
||||
|
|
@ -4403,6 +4462,7 @@ dependencies = [
|
|||
name = "litellm-redis-testing"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"pkcs8",
|
||||
"rcgen",
|
||||
"tempfile",
|
||||
"testcontainers-modules",
|
||||
|
|
@ -5283,6 +5343,16 @@ version = "0.2.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "pbkdf2"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
|
||||
dependencies = [
|
||||
"digest 0.10.7",
|
||||
"hmac 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem"
|
||||
version = "4.0.0"
|
||||
|
|
@ -5393,6 +5463,22 @@ dependencies = [
|
|||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs5"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
|
||||
dependencies = [
|
||||
"aes",
|
||||
"cbc",
|
||||
"der",
|
||||
"des",
|
||||
"pbkdf2",
|
||||
"scrypt",
|
||||
"sha2 0.10.9",
|
||||
"spki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.10.2"
|
||||
|
|
@ -5400,6 +5486,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der",
|
||||
"pkcs5",
|
||||
"rand_core 0.6.4",
|
||||
"spki",
|
||||
]
|
||||
|
||||
|
|
@ -6408,6 +6496,15 @@ version = "1.0.23"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "salsa20"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
|
||||
dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
|
|
@ -6471,6 +6568,17 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scrypt"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f"
|
||||
dependencies = [
|
||||
"pbkdf2",
|
||||
"salsa20",
|
||||
"sha2 0.10.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sct"
|
||||
version = "0.7.1"
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
- `resolve` takes a `SecretSnapshot` and an environment `Lookup`. The snapshot holds every name `ConfiguredRedis::secret_names` lists, read the way Python's `get_secret` reads it (secret manager, then environment). The host fills it once before resolving, python-bridge through `get_secret` and rust-proxy from its environment, so the resolver never calls back into the host. The environment lookup is only for what Python reads straight from `os.environ` (`REDIS_USERNAME` for Azure). Reading a name the snapshot does not list fails a debug assertion
|
||||
- Every Redis setting Python reads is one row in `SETTINGS` in `src/redis/keys.rs`: how Rust handles it (read, or warn and drop), whether `_get_redis_env_kwarg_mapping` reads it as `REDIS_<KWARG>`, and the names `_get_redis_client_logic` falls back to. The env layer, the ignored list and `secret_names` all come from that table. A read row is also a field of `Options`
|
||||
- Lookups go through `litellm_core_utils::settings::Lookup`, never a closure type of our own
|
||||
- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_ca_data`, `ssl_ca_path`, `ssl_certfile`, `ssl_keyfile` and `ssl_check_hostname` query parameters are read like `parse_url` reads them
|
||||
- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_ca_data`, `ssl_ca_path`, `ssl_certfile`, `ssl_keyfile`, `ssl_password` and `ssl_check_hostname` query parameters are read like `parse_url` reads them
|
||||
- TLS options and `db` are only read for the endpoint that uses them, so a bad `ssl_cert_reqs` never fails a url target. A cluster with a non-zero `db`, and Sentinel with a `host` or `port`, are errors because redis-py raises building those clients
|
||||
- An empty value (`""`, `[]`, `{}`) is the same as an unset one, so it falls through to its environment fallback the way Python's `x or get_secret(...)` does. Nulls depend on the surface. `coordination_redis` drops every null like `model_dump(exclude_none=True)`, `cache_params` keeps a null as an explicit `None` that hides the `REDIS_*` value unless `Cache` or `RedisCache` names that parameter, and `RedisCache.redis_kwargs` keeps every null. A missing `os.environ/` reference is a null, as `load_config` resolves it
|
||||
- Precedence matches Python. The target is cluster, then URL, then Sentinel (`sentinel_nodes` plus `service_name`), then host. Auth is an explicit `credential_provider`, then GCP, then Azure, then AWS, and the first one wins with a warning
|
||||
|
|
@ -15,8 +15,8 @@
|
|||
- Flags (`ssl`, `ssl_check_hostname`, `aws_iam_auth`, `aws_iam_serverless`) are read one way on every surface. `true`, `1` and `yes` are true and `false`, `0`, `no`, `off`, `f` and `n` are false, the text redis-py's `_str_to_bool` and the proxy's pydantic models agree on. Any other text is `RedisError::Invalid`, so a TLS setting is never guessed
|
||||
- `RedisSettings::coordination` mirrors the proxy's `CoordinationRedisParams` pydantic model, which rejects a malformed `port` before redis-py coerces anything
|
||||
- The resolver never fails on a setting it cannot honor. A `Warn` row is dropped and returned as an `Ignored` naming its source (`ssl_ciphers` or `REDIS_SSL_CIPHERS`) and the consequence from `keys::consequence`, because a Python deployment that worked must keep working when its stores move to Rust. Only a value that is malformed (`RedisError::Invalid`) or a target that is missing fails. Promoting a `Warn` row to `Read` is then a pure improvement with no behavior cliff
|
||||
- `ssl_ciphers`, `ssl_min_version` above TLS 1.2, `ssl_password`, the OCSP options, `connection_pool`, `event_dispatcher`, `redis_connect_func` and `retry` are ignored. Ignoring OCSP connects without revocation checks, and the consequence says so
|
||||
- `ssl_ca_data` and `ssl_ca_path` are read into `TlsOptions` and trusted next to `ssl_ca_certs`. `unix_socket_path` resolves to `Target::Unix`, which like redis-py's `UnixDomainSocketConnection` replaces host, port and TLS. Python still requires `host` next to it, so Rust does too
|
||||
- `ssl_ciphers`, `ssl_min_version` above TLS 1.2, the OCSP options, `connection_pool`, `event_dispatcher`, `redis_connect_func` and `retry` are ignored. Ignoring OCSP connects without revocation checks, and the consequence says so
|
||||
- `ssl_ca_data` and `ssl_ca_path` are read into `TlsOptions` and trusted next to `ssl_ca_certs`. `ssl_password` is read into `TlsOptions::key_password`, which `redis-client` uses to decrypt a PKCS#8 key like the sync client's `load_cert_chain(password=...)`. redis-py 5.3.1's async client refuses it, so this is a superset of the path the proxy uses, like `ssl_ca_path` `unix_socket_path` resolves to `Target::Unix`, which like redis-py's `UnixDomainSocketConnection` replaces host, port and TLS. Python still requires `host` next to it, so Rust does too
|
||||
- A url's `?db=`, `?username=` and `?password=` are read like `parse_url` reads them, so `?db=` wins over the path and the userinfo wins over `?username=` and `?password=`. Any other query parameter except the TLS ones on a `rediss://` url is ignored naming `url ?<name>`. redis-py turns each one into a client kwarg
|
||||
- `retry_on_timeout`, `retry_on_error`, `protocol`, `cache`, `cache_config`, `socket_keepalive` and `max_connections` are ignored. Cluster clients still turn TCP keepalive on, like Python's async cluster default
|
||||
- `health_check_interval` is read as an `int` of seconds like `_coerce_redis_kwargs_types` into `HealthCheck`. Zero is `Disabled` and stays off on every target, a negative interval checks before every command like redis-py, and only `Unset` takes a default. `redis-connect` gives an unset cluster Python's 25s default and `redis-client` pings every node before the next command once the client sat idle that long, outside the breaker and the call events
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ pub(super) const SETTINGS: &[Setting] = &[
|
|||
Setting::new("ssl_min_version", Handling::Warn, true, &[]),
|
||||
Setting::new("ssl_ocsp_context", Handling::Warn, true, &[]),
|
||||
Setting::new("ssl_ocsp_expected_cert", Handling::Warn, true, &[]),
|
||||
Setting::new("ssl_password", Handling::Warn, true, &[]),
|
||||
Setting::new("ssl_password", Handling::Read, true, &[]),
|
||||
Setting::new("ssl_validate_ocsp", Handling::Warn, true, &[]),
|
||||
Setting::new("ssl_validate_ocsp_stapled", Handling::Warn, true, &[]),
|
||||
Setting::new(
|
||||
|
|
@ -224,7 +224,6 @@ fn consequence(kwarg: &str) -> &'static str {
|
|||
"ssl_ciphers" | "ssl_min_version" => {
|
||||
"rustls negotiates TLS 1.2+ with its own ECDHE AEAD suites"
|
||||
}
|
||||
"ssl_password" => "an encrypted client key cannot be read, so the TLS handshake fails",
|
||||
"credential_provider" | "redis_connect_func" => {
|
||||
"the Python object is not callable from Rust, so static username and password are used"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ pub(super) struct TlsInput {
|
|||
pub(super) ca_path: Option<Entry>,
|
||||
pub(super) certfile: Option<Entry>,
|
||||
pub(super) keyfile: Option<Entry>,
|
||||
pub(super) key_password: Option<Entry>,
|
||||
pub(super) check_hostname: Option<Entry>,
|
||||
}
|
||||
|
||||
|
|
@ -172,6 +173,7 @@ impl Options {
|
|||
ca_path: get("ssl_ca_path"),
|
||||
certfile: get("ssl_certfile"),
|
||||
keyfile: get("ssl_keyfile"),
|
||||
key_password: get("ssl_password"),
|
||||
check_hostname: get("ssl_check_hostname"),
|
||||
},
|
||||
namespace: get("namespace"),
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ const URL_TLS_SETTINGS: &[&str] = &[
|
|||
"ssl_ca_path",
|
||||
"ssl_certfile",
|
||||
"ssl_keyfile",
|
||||
"ssl_password",
|
||||
"ssl_check_hostname",
|
||||
];
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ pub(super) fn tls_options(
|
|||
ca_path: text(input.ca_path.as_ref()).map(Into::into),
|
||||
certfile: text(input.certfile.as_ref()).map(Into::into),
|
||||
keyfile: text(input.keyfile.as_ref()).map(Into::into),
|
||||
key_password: secret(input.key_password.as_ref()),
|
||||
verify_certificate: verify_certificate(input.cert_reqs.as_ref())?,
|
||||
check_hostname: flag(input.check_hostname.as_ref())?,
|
||||
})
|
||||
|
|
@ -99,6 +101,7 @@ pub(super) fn url_query(url: &str) -> Result<UrlQuery, RedisError> {
|
|||
ca_path: text(get("ssl_ca_path").as_ref()).map(Into::into),
|
||||
certfile: text(get("ssl_certfile").as_ref()).map(Into::into),
|
||||
keyfile: text(get("ssl_keyfile").as_ref()).map(Into::into),
|
||||
key_password: secret(get("ssl_password").as_ref()),
|
||||
verify_certificate: verify_certificate(get("ssl_cert_reqs").as_ref())?,
|
||||
check_hostname: text(get("ssl_check_hostname").as_ref())
|
||||
.is_some_and(|text| url_bool(&text)),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use litellm_redis_types::{
|
|||
Auth, AwsIamAuth, AzureAdAuth, HealthCheck, Node, ProviderHandle, Resolution, StaticAuth,
|
||||
Target, TlsOptions,
|
||||
};
|
||||
use litellm_secret_value::SecretValue;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
|
|
@ -327,22 +328,22 @@ fn incomplete_aws_iam_auth_fails(#[case] yaml: &str, #[case] message: &str) {
|
|||
#[case::defaults_verify_the_certificate_but_not_the_hostname(
|
||||
" ssl: true",
|
||||
&[],
|
||||
Some(TlsOptions { ca_certs: None, ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false })
|
||||
Some(TlsOptions { ca_certs: None, ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false })
|
||||
)]
|
||||
#[case::every_option(
|
||||
" ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_cert_reqs: none\n ssl_check_hostname: true",
|
||||
" ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_password: key-secret\n ssl_cert_reqs: none\n ssl_check_hostname: true",
|
||||
&[],
|
||||
Some(TlsOptions { ca_certs: Some("/ca.pem".into()), ca_data: None, ca_path: None, certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), verify_certificate: false, check_hostname: true })
|
||||
Some(TlsOptions { ca_certs: Some("/ca.pem".into()), ca_data: None, ca_path: None, certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), key_password: Some(SecretValue::new("key-secret")), verify_certificate: false, check_hostname: true })
|
||||
)]
|
||||
#[case::options_from_the_environment(
|
||||
" ssl: true",
|
||||
&[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional")],
|
||||
Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false })
|
||||
&[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional"), ("REDIS_SSL_PASSWORD", "env-key-secret")],
|
||||
Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: Some(SecretValue::new("env-key-secret")), verify_certificate: true, check_hostname: false })
|
||||
)]
|
||||
#[case::gcp_ca_certs_replace_the_configured_ones(
|
||||
" ssl: true\n ssl_ca_certs: /ca.pem\n gcp_service_account: account\n gcp_ssl_ca_certs: /gcp-ca.pem",
|
||||
&[],
|
||||
Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, verify_certificate: true, check_hostname: false })
|
||||
Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false })
|
||||
)]
|
||||
#[case::options_without_ssl_stay_plaintext(" ssl_ca_certs: /ca.pem", &[], None)]
|
||||
fn tls_options_resolve_like_redis_py(
|
||||
|
|
@ -357,6 +358,22 @@ fn tls_options_resolve_like_redis_py(
|
|||
assert!(matches!(resolved.target, Target::Standalone(host) if host.tls == expected));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_rediss_url_reads_ssl_password_from_its_query() {
|
||||
let resolved = resolve_kwargs(
|
||||
json!({"url": "rediss://cache:6380/0?ssl_password=query-key-secret"}),
|
||||
&HashMap::new(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let Target::Url(parsed) = resolved.target else {
|
||||
panic!("expected a url target, got {:?}", resolved.target);
|
||||
};
|
||||
let key_password = parsed.tls.and_then(|tls| tls.key_password);
|
||||
assert_eq!(expose(key_password.as_ref()), Some("query-key-secret"));
|
||||
assert!(resolved.ignored.is_empty(), "{:?}", resolved.ignored);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn an_unknown_cert_requirement_fails() {
|
||||
let error = resolve(
|
||||
|
|
@ -958,6 +975,11 @@ fn expected_view(fields: serde_json::Value) -> serde_json::Value {
|
|||
&[],
|
||||
Ok(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": "/ca", "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))),
|
||||
)]
|
||||
#[case::ssl_password_is_unsupported(
|
||||
json!({"host": "redis", "ssl": true, "ssl_certfile": "/client.pem", "ssl_password": "key-secret"}),
|
||||
&[],
|
||||
Ok(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": "/client.pem", "keyfile": null, "verify_certificate": true, "check_hostname": false}}))),
|
||||
)]
|
||||
#[case::host_with_every_basic_setting(
|
||||
json!({"db": 2, "host": "redis", "password": "pass", "port": 6380, "socket_timeout": 1.5, "ssl": true, "username": "user"}),
|
||||
&[],
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@
|
|||
|
||||
## TLS, credentials and Sentinel
|
||||
- `TlsOptions` follows redis-py 5.3.1. It verifies the certificate by default, skips the hostname check by default (`ssl_check_hostname=False`), and trusts the system roots plus `ca_certs`
|
||||
- rustls only reads unencrypted keys, so a PKCS#8 `ENCRYPTED PRIVATE KEY` is decrypted with `key_password` before it reaches redis-rs. A legacy `Proc-Type: 4,ENCRYPTED` PEM key is refused with `KeyError::LegacyEncryption` rather than decrypted by hand-rolled OpenSSL key derivation
|
||||
- redis-rs loads the system trust store for every TLS connection and fails if that fails, even with `ca_certs` set. Where the macOS keychain is blocked (agent sandboxes), run the TLS tests with `SSL_CERT_FILE=/etc/ssl/cert.pem`
|
||||
- `Credentials` are static and sent at connect. URL userinfo wins over them, like redis-py's `from_url`
|
||||
- Sentinel discovers the master on each new connection and drops it on a transport failure or READONLY, like redis-py's `SentinelConnectionPool`
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ redis = { workspace = true, features = [
|
|||
"tokio-rustls-comp",
|
||||
] }
|
||||
pem = "4.0.0"
|
||||
pkcs8 = { version = "0.10.2", features = ["3des", "encryption", "std"] }
|
||||
rustls-native-certs.workspace = true
|
||||
socket2 = "0.6.5"
|
||||
thiserror.workspace = true
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ use redis::{
|
|||
use litellm_circuit_breaker::Breaker;
|
||||
use litellm_secret_value::SecretValue;
|
||||
|
||||
use crate::{ConnectError, Namespace, RedisEvents};
|
||||
use pkcs8::EncryptedPrivateKeyInfo;
|
||||
|
||||
use crate::{ConnectError, KeyError, Namespace, RedisEvents};
|
||||
use litellm_redis_types::{Cluster, RedisUrl, Target, TlsOptions};
|
||||
|
||||
// litellm/constants.py REDIS_SOCKET_TIMEOUT, which _coerce_redis_kwargs_types sets when socket_timeout is unset
|
||||
|
|
@ -72,7 +74,7 @@ pub(crate) fn tls_certificates(tls: &TlsOptions) -> Result<TlsCertificates, Conn
|
|||
let client_tls = match (&tls.certfile, &tls.keyfile) {
|
||||
(Some(certfile), keyfile) => Some(ClientTlsConfig {
|
||||
client_cert: read(certfile)?,
|
||||
client_key: read(keyfile.as_ref().unwrap_or(certfile))?,
|
||||
client_key: client_key(keyfile.as_ref().unwrap_or(certfile), tls.key_password.as_ref())?,
|
||||
}),
|
||||
(None, _) => None,
|
||||
};
|
||||
|
|
@ -152,6 +154,43 @@ fn with_system_roots(configured: Vec<u8>) -> Vec<u8> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// The key as unencrypted PEM, decrypting a PKCS#8 `ENCRYPTED PRIVATE KEY` with `ssl_password` the way
|
||||
/// `load_cert_chain(password=...)` does, since rustls only reads unencrypted keys. Like OpenSSL, a password next to
|
||||
/// an unencrypted key is unused
|
||||
fn client_key(path: &PathBuf, password: Option<&SecretValue>) -> Result<Vec<u8>, ConnectError> {
|
||||
let contents = read(path)?;
|
||||
let key_error = |reason| ConnectError::TlsKey {
|
||||
path: path.display().to_string(),
|
||||
reason,
|
||||
};
|
||||
let Ok(sections) = pem::parse_many(&contents) else {
|
||||
return Ok(contents);
|
||||
};
|
||||
let encrypted = sections
|
||||
.iter()
|
||||
.find(|section| section.tag() == "ENCRYPTED PRIVATE KEY");
|
||||
match (encrypted, password) {
|
||||
(Some(_), None) => Err(key_error(KeyError::MissingPassword)),
|
||||
(Some(section), Some(password)) => {
|
||||
let decrypted = EncryptedPrivateKeyInfo::try_from(section.contents())
|
||||
.and_then(|info| info.decrypt(password.expose()))
|
||||
.map_err(|source| key_error(KeyError::Decrypt(source)))?;
|
||||
Ok(pem::encode(&pem::Pem::new("PRIVATE KEY", decrypted.as_bytes())).into_bytes())
|
||||
}
|
||||
(None, _) if sections.iter().any(is_legacy_encrypted) => {
|
||||
Err(key_error(KeyError::LegacyEncryption))
|
||||
}
|
||||
(None, _) => Ok(contents),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_legacy_encrypted(section: &pem::Pem) -> bool {
|
||||
section
|
||||
.headers()
|
||||
.get("Proc-Type")
|
||||
.is_some_and(|proc_type| proc_type.contains("ENCRYPTED"))
|
||||
}
|
||||
|
||||
fn read(path: &PathBuf) -> Result<Vec<u8>, ConnectError> {
|
||||
std::fs::read(path).map_err(|source| ConnectError::TlsFile {
|
||||
path: path.display().to_string(),
|
||||
|
|
|
|||
|
|
@ -293,6 +293,11 @@ impl SentinelConnection {
|
|||
"reading a Redis TLS file",
|
||||
path,
|
||||
)),
|
||||
ConnectError::TlsKey { path, reason } => RedisError::from((
|
||||
ErrorKind::InvalidClientConfig,
|
||||
"reading the Redis TLS key",
|
||||
format!("{path}: {reason}"),
|
||||
)),
|
||||
},
|
||||
)?;
|
||||
redis::Client::open(info)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,24 @@ pub enum ConnectError {
|
|||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
#[error("reading the Redis TLS key {path}")]
|
||||
TlsKey {
|
||||
path: String,
|
||||
#[source]
|
||||
reason: KeyError,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum KeyError {
|
||||
#[error("the key is encrypted, so ssl_password is required")]
|
||||
MissingPassword,
|
||||
#[error("ssl_password does not decrypt the key, or the key uses an unsupported cipher")]
|
||||
Decrypt(#[source] pkcs8::Error),
|
||||
#[error(
|
||||
"the key uses legacy OpenSSL PEM encryption, which Rust cannot read. Convert it to PKCS#8 with `openssl pkcs8 -topk8`"
|
||||
)]
|
||||
LegacyEncryption,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ mod script;
|
|||
pub use client::{RedisClient, Ttl};
|
||||
pub use commands::{Amount, Expiry};
|
||||
pub use config::{Credentials, DEFAULT_TIMEOUT, RedisConfig, Timeouts};
|
||||
pub use error::{ConnectError, Error};
|
||||
pub use error::{ConnectError, Error, KeyError};
|
||||
pub use events::{CallEvent, FailureKind, Operation, Outcome, RedisEvents};
|
||||
pub use litellm_redis_types::{
|
||||
Cluster, Node, RedisUrl, Sentinel, Standalone, StaticAuth, Target, TlsOptions, UnixSocket,
|
||||
|
|
|
|||
|
|
@ -33,31 +33,28 @@ fn directory_holding(ca: &std::path::Path) -> std::path::PathBuf {
|
|||
directory
|
||||
}
|
||||
|
||||
async fn ping_over_tls(
|
||||
require_client_certificate: bool,
|
||||
host: &str,
|
||||
ca: Ca,
|
||||
verify_certificate: bool,
|
||||
check_hostname: bool,
|
||||
client_certificate: bool,
|
||||
) -> Result<(), litellm_redis_client::Error> {
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum ClientKey {
|
||||
Absent,
|
||||
Plain,
|
||||
Encrypted { password: Option<&'static str> },
|
||||
}
|
||||
|
||||
async fn tls_server(require_client_certificate: bool) -> litellm_redis_testing::RedisTlsServer {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
}
|
||||
let server = litellm_redis_testing::RedisTlsServer::start(require_client_certificate)
|
||||
litellm_redis_testing::RedisTlsServer::start(require_client_certificate)
|
||||
.await
|
||||
.unwrap();
|
||||
let files = server.files();
|
||||
let tls = TlsOptions {
|
||||
ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()),
|
||||
ca_data: matches!(ca, Ca::TrustedData).then(|| std::fs::read_to_string(&files.ca).unwrap()),
|
||||
ca_path: matches!(ca, Ca::TrustedDirectory).then(|| directory_holding(&files.ca)),
|
||||
certfile: client_certificate.then(|| files.client_cert.clone()),
|
||||
keyfile: client_certificate.then(|| files.client_key.clone()),
|
||||
verify_certificate,
|
||||
check_hostname,
|
||||
};
|
||||
let client = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn tls_client(
|
||||
server: &litellm_redis_testing::RedisTlsServer,
|
||||
host: &str,
|
||||
tls: TlsOptions,
|
||||
) -> Result<litellm_redis_client::RedisClient, litellm_redis_client::ConnectError> {
|
||||
litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig {
|
||||
target: Target::Standalone(litellm_redis_client::Standalone {
|
||||
host: host.to_owned(),
|
||||
port: server.port(),
|
||||
|
|
@ -71,8 +68,46 @@ async fn ping_over_tls(
|
|||
events: None,
|
||||
health_check: None,
|
||||
})
|
||||
.unwrap();
|
||||
client.ping().await
|
||||
}
|
||||
|
||||
fn tls_options(
|
||||
files: &litellm_redis_testing::TlsFiles,
|
||||
ca: Ca,
|
||||
verify_certificate: bool,
|
||||
check_hostname: bool,
|
||||
client_key: ClientKey,
|
||||
) -> TlsOptions {
|
||||
let (keyfile, key_password) = match client_key {
|
||||
ClientKey::Absent => (None, None),
|
||||
ClientKey::Plain => (Some(files.client_key.clone()), None),
|
||||
ClientKey::Encrypted { password } => (
|
||||
Some(files.encrypted_client_key.clone()),
|
||||
password.map(SecretValue::new),
|
||||
),
|
||||
};
|
||||
TlsOptions {
|
||||
ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()),
|
||||
ca_data: matches!(ca, Ca::TrustedData).then(|| std::fs::read_to_string(&files.ca).unwrap()),
|
||||
ca_path: matches!(ca, Ca::TrustedDirectory).then(|| directory_holding(&files.ca)),
|
||||
certfile: keyfile.is_some().then(|| files.client_cert.clone()),
|
||||
keyfile,
|
||||
key_password,
|
||||
verify_certificate,
|
||||
check_hostname,
|
||||
}
|
||||
}
|
||||
|
||||
async fn ping_over_tls(
|
||||
require_client_certificate: bool,
|
||||
host: &str,
|
||||
ca: Ca,
|
||||
verify_certificate: bool,
|
||||
check_hostname: bool,
|
||||
client_key: ClientKey,
|
||||
) -> Result<(), litellm_redis_client::Error> {
|
||||
let server = tls_server(require_client_certificate).await;
|
||||
let tls = tls_options(server.files(), ca, verify_certificate, check_hostname, client_key);
|
||||
tls_client(&server, host, tls).unwrap().ping().await
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -103,7 +138,15 @@ async fn tls_options_decide_which_servers_are_trusted(
|
|||
#[case] check_hostname: bool,
|
||||
#[case] connects: bool,
|
||||
) {
|
||||
let result = ping_over_tls(false, host, ca, verify_certificate, check_hostname, false).await;
|
||||
let result = ping_over_tls(
|
||||
false,
|
||||
host,
|
||||
ca,
|
||||
verify_certificate,
|
||||
check_hostname,
|
||||
ClientKey::Absent,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.is_ok(), connects, "{result:?}");
|
||||
if let Err(error) = result {
|
||||
|
|
@ -115,24 +158,46 @@ async fn tls_options_decide_which_servers_are_trusted(
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::with_the_client_certificate(true, true)]
|
||||
#[case::without_a_client_certificate(false, false)]
|
||||
#[case::with_the_client_certificate(ClientKey::Plain, true)]
|
||||
#[case::with_an_encrypted_key_and_its_ssl_password(
|
||||
ClientKey::Encrypted { password: Some(litellm_redis_testing::CLIENT_KEY_PASSWORD) },
|
||||
true
|
||||
)]
|
||||
#[case::without_a_client_certificate(ClientKey::Absent, false)]
|
||||
#[tokio::test]
|
||||
async fn mutual_tls_sends_the_configured_client_certificate(
|
||||
#[case] client_certificate: bool,
|
||||
#[case] client_key: ClientKey,
|
||||
#[case] connects: bool,
|
||||
) {
|
||||
let result = ping_over_tls(
|
||||
true,
|
||||
"127.0.0.1",
|
||||
let result = ping_over_tls(true, "127.0.0.1", Ca::Trusted, true, false, client_key).await;
|
||||
|
||||
assert_eq!(result.is_ok(), connects, "{result:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::without_ssl_password(None, "ssl_password is required")]
|
||||
#[case::with_the_wrong_ssl_password(Some("wrong-password"), "does not decrypt the key")]
|
||||
#[tokio::test]
|
||||
async fn an_encrypted_key_that_cannot_be_decrypted_fails_before_connecting(
|
||||
#[case] password: Option<&'static str>,
|
||||
#[case] reason: &str,
|
||||
) {
|
||||
let server = tls_server(true).await;
|
||||
let tls = tls_options(
|
||||
server.files(),
|
||||
Ca::Trusted,
|
||||
true,
|
||||
false,
|
||||
client_certificate,
|
||||
)
|
||||
.await;
|
||||
ClientKey::Encrypted { password },
|
||||
);
|
||||
|
||||
assert_eq!(result.is_ok(), connects, "{result:?}");
|
||||
let error = tls_client(&server, "127.0.0.1", tls).err().unwrap();
|
||||
|
||||
let litellm_redis_client::ConnectError::TlsKey { path, reason: cause } = &error else {
|
||||
panic!("expected a TLS key error, got {error:?}");
|
||||
};
|
||||
assert_eq!(*path, server.files().encrypted_client_key.display().to_string());
|
||||
assert!(cause.to_string().contains(reason), "{cause}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
|
@ -161,3 +226,49 @@ fn a_missing_tls_file_names_the_path() {
|
|||
|
||||
assert!(error.to_string().contains("/nonexistent/ca.pem"), "{error}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_legacy_encrypted_pem_key_is_refused_with_a_conversion_hint() {
|
||||
let directory = std::env::temp_dir().join(format!("litellm-legacy-key-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&directory).unwrap();
|
||||
let keyfile = directory.join("client.key");
|
||||
std::fs::write(
|
||||
&keyfile,
|
||||
"-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,00112233445566778899AABBCCDDEEFF\n\nAAAA\n-----END RSA PRIVATE KEY-----\n",
|
||||
)
|
||||
.unwrap();
|
||||
let tls = TlsOptions {
|
||||
certfile: Some(keyfile.clone()),
|
||||
keyfile: Some(keyfile),
|
||||
key_password: Some(SecretValue::new("legacy-password")),
|
||||
..TlsOptions::verified()
|
||||
};
|
||||
|
||||
let error = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig {
|
||||
target: Target::Standalone(litellm_redis_client::Standalone {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 6379,
|
||||
tls: Some(tls),
|
||||
db: 0,
|
||||
}),
|
||||
credentials: Credentials::NONE,
|
||||
namespace: litellm_redis_client::Namespace::default(),
|
||||
timeouts: litellm_redis_client::Timeouts::default(),
|
||||
breaker: None,
|
||||
events: None,
|
||||
health_check: None,
|
||||
})
|
||||
.err()
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
litellm_redis_client::ConnectError::TlsKey {
|
||||
reason: litellm_redis_client::KeyError::LegacyEncryption,
|
||||
..
|
||||
}
|
||||
),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
pkcs8 = { version = "0.10.2", features = ["encryption", "pem", "std"] }
|
||||
rcgen = "0.14.10"
|
||||
tempfile.workspace = true
|
||||
testcontainers-modules = { workspace = true, features = ["redis"] }
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ pub enum Error {
|
|||
Command { command: String, code: i64 },
|
||||
#[error("generating test TLS certificates")]
|
||||
Certificate(#[from] rcgen::Error),
|
||||
#[error("encrypting the test client key")]
|
||||
KeyEncryption(#[from] pkcs8::Error),
|
||||
#[error("writing test TLS files")]
|
||||
Files(#[source] io::Error),
|
||||
#[error("Sentinel did not promote the replica")]
|
||||
|
|
|
|||
|
|
@ -12,4 +12,4 @@ pub use deployment::{ClientConnection, RedisDeployment, Topology, eventually};
|
|||
pub use error::Error;
|
||||
pub use sentinel::{RedisSentinel, SERVICE_NAME};
|
||||
pub use standalone::RedisServer;
|
||||
pub use tls::{RedisTlsServer, SERVER_NAME, TlsFiles};
|
||||
pub use tls::{CLIENT_KEY_PASSWORD, RedisTlsServer, SERVER_NAME, TlsFiles};
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use pkcs8::{LineEnding, PrivateKeyInfo, pkcs5::pbes2};
|
||||
use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair};
|
||||
use tempfile::TempDir;
|
||||
use testcontainers_modules::testcontainers::{
|
||||
|
|
@ -15,12 +16,15 @@ use crate::{
|
|||
};
|
||||
|
||||
pub const SERVER_NAME: &str = "localhost";
|
||||
pub const CLIENT_KEY_PASSWORD: &str = "redis-testing-key-password";
|
||||
|
||||
pub struct TlsFiles {
|
||||
_directory: TempDir,
|
||||
pub ca: PathBuf,
|
||||
pub client_cert: PathBuf,
|
||||
pub client_key: PathBuf,
|
||||
/// `client_key` as a PKCS#8 `ENCRYPTED PRIVATE KEY` under `CLIENT_KEY_PASSWORD`
|
||||
pub encrypted_client_key: PathBuf,
|
||||
}
|
||||
|
||||
struct Generated {
|
||||
|
|
@ -117,6 +121,7 @@ fn generate() -> Result<Generated, Error> {
|
|||
let ca_path = write("ca.pem", ca.pem())?;
|
||||
let client_cert = write("client.pem", client.pem())?;
|
||||
let client_key_path = write("client.key", client_key.serialize_pem())?;
|
||||
let encrypted_client_key = write("client-encrypted.key", encrypt(&client_key)?)?;
|
||||
Ok(Generated {
|
||||
ca_pem: ca.pem(),
|
||||
server_cert: server.pem(),
|
||||
|
|
@ -126,6 +131,16 @@ fn generate() -> Result<Generated, Error> {
|
|||
ca: ca_path,
|
||||
client_cert,
|
||||
client_key: client_key_path,
|
||||
encrypted_client_key,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// The parameters `openssl pkcs8 -topk8` uses by default: PBES2 with PBKDF2-HMAC-SHA256 and AES-256-CBC
|
||||
fn encrypt(key: &KeyPair) -> Result<String, pkcs8::Error> {
|
||||
let der = key.serialize_der();
|
||||
let params = pbes2::Parameters::pbkdf2_sha256_aes256cbc(2048, b"redis-testing-salt", &[7; 16])?;
|
||||
let encrypted =
|
||||
PrivateKeyInfo::try_from(der.as_slice())?.encrypt_with_params(params, CLIENT_KEY_PASSWORD)?;
|
||||
Ok(encrypted.to_pem("ENCRYPTED PRIVATE KEY", LineEnding::LF)?.to_string())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ pub struct TlsOptions {
|
|||
pub ca_path: Option<PathBuf>,
|
||||
pub certfile: Option<PathBuf>,
|
||||
pub keyfile: Option<PathBuf>,
|
||||
/// Unlocks an encrypted `keyfile`, redis-py's `ssl_password`
|
||||
pub key_password: Option<SecretValue>,
|
||||
pub verify_certificate: bool,
|
||||
pub check_hostname: bool,
|
||||
}
|
||||
|
|
@ -123,6 +125,7 @@ impl TlsOptions {
|
|||
ca_path: None,
|
||||
certfile: None,
|
||||
keyfile: None,
|
||||
key_password: None,
|
||||
verify_certificate: true,
|
||||
check_hostname: false,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2136,6 +2136,17 @@ def _expected_view(fields: dict[str, object]) -> dict[str, object]:
|
|||
},
|
||||
id="ssl_ca_path_is_unsupported",
|
||||
),
|
||||
pytest.param(
|
||||
{"host": "redis", "ssl": True, "ssl_certfile": "/client.pem", "ssl_password": "key-secret"},
|
||||
{},
|
||||
{
|
||||
"rejected": {
|
||||
"message": "Redis.__init__() got an unexpected keyword argument 'ssl_password'",
|
||||
"type": "TypeError",
|
||||
}
|
||||
},
|
||||
id="ssl_password_is_unsupported",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"db": 2,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue