From a493f9f77f23eeeaf2dcdf043730c63f38457317 Mon Sep 17 00:00:00 2001 From: Haquant <76264891+Steeve-Crypto@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:51:59 -0400 Subject: [PATCH 1/2] fix(proxy): log undecryptable DB models --- .../common_utils/encrypt_decrypt_utils.py | 23 +++++++++ litellm/proxy/proxy_server.py | 41 +++++++++++++++ .../test_encrypt_decrypt_utils.py | 25 ++++++++++ .../proxy/proxy_server/test_proxy_config.py | 50 +++++++++++++++++++ 4 files changed, 139 insertions(+) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index e5f6bc5dda8..dfc324003b4 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,5 +1,6 @@ import base64 import os +import re from collections.abc import Mapping from typing import Final, Literal, cast @@ -186,6 +187,28 @@ def decrypt_if_encrypted_with(value: str, signing_key: str) -> str | None: return None +# A legacy nacl ciphertext is base64url(nonce(24) || MAC(16) || plaintext), so it is +# never shorter than 40 bytes once decoded. +_NACL_MIN_SEALED_BYTES: Final = 40 +_BASE64URL_PADDED: Final = re.compile(r"[A-Za-z0-9_-]+={0,2}") + + +def is_undecryptable_ciphertext(value: object, signing_key: str | None) -> bool: + """True when value has the shape of a stored ciphertext but does not open under signing_key. + + Used to flag values written under another key (e.g. LITELLM_SALT_KEY added after rows were + encrypted with the master key). Plaintext such as "gpt-4o" or "os.environ/X" is never flagged. + """ + if not isinstance(value, str) or not signing_key: + return False + if not value.startswith(V2_GCM_PREFIX): + if len(value) % 4 != 0 or _BASE64URL_PADDED.fullmatch(value) is None: + return False + if len(base64.urlsafe_b64decode(value)) < _NACL_MIN_SEALED_BYTES: + return False + return decrypt_if_encrypted_with(value=value, signing_key=signing_key) is None + + def decrypt_value_helper( value: str, key: str, # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 000ffa9c305..442f547fdf7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -427,6 +427,8 @@ from litellm.proxy.common_utils.discoverable_model_filter import discoverable_ro from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, + get_salt_key, + is_undecryptable_ciphertext, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.fips import ( @@ -5617,6 +5619,7 @@ class ProxyConfig: self._last_cleanup_schedule_attempt: tuple[object, ...] | None = None self._cleanup_reschedule_failed: bool = False self._warned_db_mcp_stdio_flag_ignored: bool = False + self._warned_undecryptable_db_models: frozenset[tuple[str, tuple[str, ...]]] = frozenset() self._cyberark_boot_env: dict[str, str | None] | None = None # mutable-ok: deployment env snapshot, set once self.worker_registry: list[WorkerRegistryEntry] = [] self.config_sync_subscriber: ConfigSyncSubscriber | None = None @@ -7600,6 +7603,36 @@ class ProxyConfig: return get_secret(decrypted_value) return decrypted_value + def _log_undecryptable_db_model_params(self, model: object, litellm_params: Mapping[str, object]) -> None: + """Log (once per model and key set) DB model params that do not decrypt under the current key. + + Without this, a row encrypted under an older key is passed on as ciphertext and the model + silently disappears from the router. Only key names are logged, never values. + """ + signing_key: Final = get_salt_key() + failed_keys: Final = tuple( + sorted( + k for k, v in litellm_params.items() if is_undecryptable_ciphertext(value=v, signing_key=signing_key) + ) + ) + if not failed_keys: + return + model_name: Final = str(getattr(model, "model_name", None)) + model_id: Final = str(getattr(model, "model_id", None)) + warned_key: Final = (model_id, failed_keys) + if warned_key in self._warned_undecryptable_db_models: + return + self._warned_undecryptable_db_models = self._warned_undecryptable_db_models | {warned_key} + verbose_proxy_logger.error( + "Could not decrypt litellm_params %s of DB model model_name=%s model_id=%s. " + "The row was likely encrypted with a different key (LITELLM_SALT_KEY added or changed, " + "or master_key changed), so this model will not be routable. Restore the original key " + "or re-save the model. See https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key", + list(failed_keys), + model_name, + model_id, + ) + def _add_deployment(self, db_models: list) -> int: """ Iterate through db models @@ -7618,6 +7651,10 @@ class ProxyConfig: for m in db_models: _litellm_params = m.litellm_params if isinstance(_litellm_params, dict): + self._log_undecryptable_db_model_params( + model=m, # pyright: ignore[reportUnknownArgumentType] # DB model rows are untyped here + litellm_params=_litellm_params, # pyright: ignore[reportUnknownArgumentType] # same untyped row + ) # decrypt values for k, v in _litellm_params.items(): _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v) @@ -7649,6 +7686,10 @@ class ProxyConfig: if isinstance(_litellm_params, BaseModel): _litellm_params = _litellm_params.model_dump() if isinstance(_litellm_params, dict): + self._log_undecryptable_db_model_params( + model=m, # pyright: ignore[reportUnknownArgumentType] # DB model rows are untyped here + litellm_params=_litellm_params, # pyright: ignore[reportUnknownArgumentType] # same untyped row + ) # decrypt values for k, v in _litellm_params.items(): _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v) diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 1163090c510..e5fa22bff6b 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -20,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_bearer_token, encrypt_value, encrypt_value_helper, + is_undecryptable_ciphertext, ) @@ -266,3 +267,27 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +@pytest.mark.parametrize("use_aes", [False, True]) +def test_is_undecryptable_ciphertext_flags_value_written_under_another_key( + monkeypatch: pytest.MonkeyPatch, use_aes: bool +) -> None: + if use_aes: + _use_aes(monkeypatch) + ciphertext = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key") + assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-salt-aes-1234") is True + assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-old-master-key") is False + + +@pytest.mark.parametrize( + "value", + ["gpt-4o", "openai/gpt-4o", "os.environ/OPENAI_API_KEY", "", "abcd", 12, None, "https://example.com/v1"], +) +def test_is_undecryptable_ciphertext_ignores_plaintext(value: object) -> None: + assert is_undecryptable_ciphertext(value=value, signing_key="sk-salt-aes-1234") is False + + +def test_is_undecryptable_ciphertext_without_signing_key_is_false() -> None: + ciphertext = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key") + assert is_undecryptable_ciphertext(value=ciphertext, signing_key=None) is False diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index d1b80627b3a..b63f0c89e4f 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -3373,6 +3373,56 @@ def test_ProxyConfig_decrypt_credentials_returns_an_encrypted_empty_value_as_emp assert decrypted.credential_values == {"api_base": "", "openai_service_account_id": "user-1"} +def test_ProxyConfig_decrypt_model_list_from_db_logs_rows_encrypted_under_another_key( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A row encrypted before LITELLM_SALT_KEY was set must be reported, not dropped silently (#45574).""" + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-new-salt-key") + old_api_key = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key") + old_model = encrypt_value_helper("openai/gpt-4o", new_encryption_key="sk-old-master-key") + pc = ProxyConfig() + + def stale_row() -> SimpleNamespace: + return SimpleNamespace( + model_id="m-stale", + model_name="memory-model", + model_info={"id": "m-stale"}, + litellm_params={"api_key": old_api_key, "model": old_model, "rpm": 10}, + blocked=False, + ) + + with caplog.at_level(logging.ERROR): + pc.decrypt_model_list_from_db(new_models=[stale_row()]) + pc.decrypt_model_list_from_db(new_models=[stale_row()]) + + errors = [ + r.getMessage() for r in caplog.records if r.levelno == logging.ERROR and "Could not decrypt" in r.getMessage() + ] + assert len(errors) == 1 + assert "['api_key', 'model']" in errors[0] + assert "model_name=memory-model model_id=m-stale" in errors[0] + assert old_api_key not in caplog.text + assert "sk-provider-secret" not in caplog.text + + +def test_ProxyConfig_decrypt_model_list_from_db_does_not_log_rows_under_current_key( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-new-salt-key") + pc = ProxyConfig() + m = SimpleNamespace( + model_id="m-ok", + model_name="gpt-4o", + model_info={"id": "m-ok"}, + litellm_params={"api_key": encrypt_value_helper("sk-provider-secret"), "model": "openai/gpt-4o"}, + blocked=False, + ) + with caplog.at_level(logging.ERROR): + out = pc.decrypt_model_list_from_db(new_models=[m]) + assert out[0]["litellm_params"]["api_key"] == "sk-provider-secret" + assert "Could not decrypt" not in caplog.text + + def test_ProxyConfig_decrypt_model_list_from_db_returns_decrypted(monkeypatch): monkeypatch.setattr( "litellm.proxy.proxy_server.decrypt_value_helper", From 7e52e5f33747da0bcf452a6ad976195e17019415 Mon Sep 17 00:00:00 2001 From: Haquant <76264891+Steeve-Crypto@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:26:33 -0400 Subject: [PATCH 2/2] fix(proxy): accept standard base64 ciphertext --- .../common_utils/encrypt_decrypt_utils.py | 9 ++++--- .../test_encrypt_decrypt_utils.py | 25 ++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index dfc324003b4..56ad5f972b4 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -190,7 +190,10 @@ def decrypt_if_encrypted_with(value: str, signing_key: str) -> str | None: # A legacy nacl ciphertext is base64url(nonce(24) || MAC(16) || plaintext), so it is # never shorter than 40 bytes once decoded. _NACL_MIN_SEALED_BYTES: Final = 40 -_BASE64URL_PADDED: Final = re.compile(r"[A-Za-z0-9_-]+={0,2}") +# The two alphabets _legacy_ciphertext_bytes accepts: base64url (current rows) and standard +# base64 (older rows). A real ciphertext uses exactly one, so a mixed value such as a model path +# "org/model-name" is never treated as ciphertext. +_BASE64_ALPHABETS: Final = (re.compile(r"[A-Za-z0-9_-]+={0,2}"), re.compile(r"[A-Za-z0-9+/]+={0,2}")) def is_undecryptable_ciphertext(value: object, signing_key: str | None) -> bool: @@ -202,9 +205,9 @@ def is_undecryptable_ciphertext(value: object, signing_key: str | None) -> bool: if not isinstance(value, str) or not signing_key: return False if not value.startswith(V2_GCM_PREFIX): - if len(value) % 4 != 0 or _BASE64URL_PADDED.fullmatch(value) is None: + if len(value) % 4 != 0 or not any(alphabet.fullmatch(value) for alphabet in _BASE64_ALPHABETS): return False - if len(base64.urlsafe_b64decode(value)) < _NACL_MIN_SEALED_BYTES: + if len(_legacy_ciphertext_bytes(value)) < _NACL_MIN_SEALED_BYTES: return False return decrypt_if_encrypted_with(value=value, signing_key=signing_key) is None diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index e5fa22bff6b..92180eb4a08 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -280,9 +280,32 @@ def test_is_undecryptable_ciphertext_flags_value_written_under_another_key( assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-old-master-key") is False +def test_is_undecryptable_ciphertext_flags_standard_base64_row_under_another_key() -> None: + # The nonce is random, so draw until the standard-alphabet characters actually appear. + candidates = ( + base64.b64encode(encrypt_value(value="sk-provider-secret-" + "x" * 64, signing_key="sk-old-master-key")).decode( + "utf-8" + ) + for _ in range(200) + ) + ciphertext = next(c for c in candidates if "+" in c or "/" in c) + assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-salt-aes-1234") is True + assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-old-master-key") is False + + @pytest.mark.parametrize( "value", - ["gpt-4o", "openai/gpt-4o", "os.environ/OPENAI_API_KEY", "", "abcd", 12, None, "https://example.com/v1"], + [ + "gpt-4o", + "openai/gpt-4o", + "os.environ/OPENAI_API_KEY", + "", + "abcd", + 12, + None, + "https://example.com/v1", + "huggingface/meta-llama/Meta-Llama-3-70B-Instruct-fine-tuned-abcd", + ], ) def test_is_undecryptable_ciphertext_ignores_plaintext(value: object) -> None: assert is_undecryptable_ciphertext(value=value, signing_key="sk-salt-aes-1234") is False