Merge 7e52e5f337 into c8ed5ccca8

litellm/proxy/common_utils/encrypt_decrypt_utils.py

@@ -1,5 +1,6 @@
 import base64
 import os
+import re
 from collections.abc import Mapping
 from typing import Final, Literal, cast
 
@@ -186,6 +187,31 @@ def decrypt_if_encrypted_with(value: str, signing_key: str) -> str | None:
         return None
 
 
+# A legacy nacl ciphertext is base64url(nonce(24) || MAC(16) || plaintext), so it is
+# never shorter than 40 bytes once decoded.
+_NACL_MIN_SEALED_BYTES: Final = 40
+# The two alphabets _legacy_ciphertext_bytes accepts: base64url (current rows) and standard
+# base64 (older rows). A real ciphertext uses exactly one, so a mixed value such as a model path
+# "org/model-name" is never treated as ciphertext.
+_BASE64_ALPHABETS: Final = (re.compile(r"[A-Za-z0-9_-]+={0,2}"), re.compile(r"[A-Za-z0-9+/]+={0,2}"))
+
+
+def is_undecryptable_ciphertext(value: object, signing_key: str | None) -> bool:
+    """True when value has the shape of a stored ciphertext but does not open under signing_key.
+
+    Used to flag values written under another key (e.g. LITELLM_SALT_KEY added after rows were
+    encrypted with the master key). Plaintext such as "gpt-4o" or "os.environ/X" is never flagged.
+    """
+    if not isinstance(value, str) or not signing_key:
+        return False
+    if not value.startswith(V2_GCM_PREFIX):
+        if len(value) % 4 != 0 or not any(alphabet.fullmatch(value) for alphabet in _BASE64_ALPHABETS):
+            return False
+        if len(_legacy_ciphertext_bytes(value)) < _NACL_MIN_SEALED_BYTES:
+            return False
+    return decrypt_if_encrypted_with(value=value, signing_key=signing_key) is None
+
+
 def decrypt_value_helper(
     value: str,
     key: str,  # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key.

litellm/proxy/proxy_server.py

@@ -484,6 +484,8 @@ from litellm.proxy.common_utils.discoverable_model_filter import (
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
+    get_salt_key,
+    is_undecryptable_ciphertext,
 )
 from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
 from litellm.proxy.common_utils.healthy_model_filter import (
@@ -5643,6 +5645,7 @@ class ProxyConfig:
         self._last_cleanup_schedule_attempt: tuple[object, ...] | None = None
         self._cleanup_reschedule_failed: bool = False
         self._warned_db_mcp_stdio_flag_ignored: bool = False
+        self._warned_undecryptable_db_models: frozenset[tuple[str, tuple[str, ...]]] = frozenset()
         self._cyberark_boot_env: dict[str, str | None] | None = None  # mutable-ok: deployment env snapshot, set once
         self.worker_registry: list[WorkerRegistryEntry] = []
         self.config_sync_subscriber: ConfigSyncSubscriber | None = None
@@ -7670,6 +7673,36 @@ class ProxyConfig:
             return get_secret(decrypted_value)
         return decrypted_value
 
+    def _log_undecryptable_db_model_params(self, model: object, litellm_params: Mapping[str, object]) -> None:
+        """Log (once per model and key set) DB model params that do not decrypt under the current key.
+
+        Without this, a row encrypted under an older key is passed on as ciphertext and the model
+        silently disappears from the router. Only key names are logged, never values.
+        """
+        signing_key: Final = get_salt_key()
+        failed_keys: Final = tuple(
+            sorted(
+                k for k, v in litellm_params.items() if is_undecryptable_ciphertext(value=v, signing_key=signing_key)
+            )
+        )
+        if not failed_keys:
+            return
+        model_name: Final = str(getattr(model, "model_name", None))
+        model_id: Final = str(getattr(model, "model_id", None))
+        warned_key: Final = (model_id, failed_keys)
+        if warned_key in self._warned_undecryptable_db_models:
+            return
+        self._warned_undecryptable_db_models = self._warned_undecryptable_db_models | {warned_key}
+        verbose_proxy_logger.error(
+            "Could not decrypt litellm_params %s of DB model model_name=%s model_id=%s. "
+            "The row was likely encrypted with a different key (LITELLM_SALT_KEY added or changed, "
+            "or master_key changed), so this model will not be routable. Restore the original key "
+            "or re-save the model. See https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key",
+            list(failed_keys),
+            model_name,
+            model_id,
+        )
+
     def _add_deployment(self, db_models: list) -> int:
         """
         Iterate through db models
@@ -7688,6 +7721,10 @@ class ProxyConfig:
         for m in db_models:
             _litellm_params = m.litellm_params
             if isinstance(_litellm_params, dict):
+                self._log_undecryptable_db_model_params(
+                    model=m,  # pyright: ignore[reportUnknownArgumentType]  # DB model rows are untyped here
+                    litellm_params=_litellm_params,  # pyright: ignore[reportUnknownArgumentType]  # same untyped row
+                )
                 # decrypt values
                 for k, v in _litellm_params.items():
                     _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)
@@ -7719,6 +7756,10 @@ class ProxyConfig:
             if isinstance(_litellm_params, BaseModel):
                 _litellm_params = _litellm_params.model_dump()
             if isinstance(_litellm_params, dict):
+                self._log_undecryptable_db_model_params(
+                    model=m,  # pyright: ignore[reportUnknownArgumentType]  # DB model rows are untyped here
+                    litellm_params=_litellm_params,  # pyright: ignore[reportUnknownArgumentType]  # same untyped row
+                )
                 # decrypt values
                 for k, v in _litellm_params.items():
                     _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)

tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py

@@ -20,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     encrypt_bearer_token,
     encrypt_value,
     encrypt_value_helper,
+    is_undecryptable_ciphertext,
 )
 
 
@@ -266,3 +267,50 @@ def test_bearer_token_uses_only_header_safe_characters(length: int):
 
     assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token
     assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length
+
+
+@pytest.mark.parametrize("use_aes", [False, True])
+def test_is_undecryptable_ciphertext_flags_value_written_under_another_key(
+    monkeypatch: pytest.MonkeyPatch, use_aes: bool
+) -> None:
+    if use_aes:
+        _use_aes(monkeypatch)
+    ciphertext = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key")
+    assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-salt-aes-1234") is True
+    assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-old-master-key") is False
+
+
+def test_is_undecryptable_ciphertext_flags_standard_base64_row_under_another_key() -> None:
+    # The nonce is random, so draw until the standard-alphabet characters actually appear.
+    candidates = (
+        base64.b64encode(encrypt_value(value="sk-provider-secret-" + "x" * 64, signing_key="sk-old-master-key")).decode(
+            "utf-8"
+        )
+        for _ in range(200)
+    )
+    ciphertext = next(c for c in candidates if "+" in c or "/" in c)
+    assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-salt-aes-1234") is True
+    assert is_undecryptable_ciphertext(value=ciphertext, signing_key="sk-old-master-key") is False
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "gpt-4o",
+        "openai/gpt-4o",
+        "os.environ/OPENAI_API_KEY",
+        "",
+        "abcd",
+        12,
+        None,
+        "https://example.com/v1",
+        "huggingface/meta-llama/Meta-Llama-3-70B-Instruct-fine-tuned-abcd",
+    ],
+)
+def test_is_undecryptable_ciphertext_ignores_plaintext(value: object) -> None:
+    assert is_undecryptable_ciphertext(value=value, signing_key="sk-salt-aes-1234") is False
+
+
+def test_is_undecryptable_ciphertext_without_signing_key_is_false() -> None:
+    ciphertext = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key")
+    assert is_undecryptable_ciphertext(value=ciphertext, signing_key=None) is False

tests/unit/proxy/proxy_server/test_proxy_config.py

@@ -3614,6 +3614,56 @@ def test_ProxyConfig_decrypt_credentials_returns_an_encrypted_empty_value_as_emp
     assert decrypted.credential_values == {"api_base": "", "openai_service_account_id": "user-1"}
 
 
+def test_ProxyConfig_decrypt_model_list_from_db_logs_rows_encrypted_under_another_key(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    """A row encrypted before LITELLM_SALT_KEY was set must be reported, not dropped silently (#45574)."""
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-new-salt-key")
+    old_api_key = encrypt_value_helper("sk-provider-secret", new_encryption_key="sk-old-master-key")
+    old_model = encrypt_value_helper("openai/gpt-4o", new_encryption_key="sk-old-master-key")
+    pc = ProxyConfig()
+
+    def stale_row() -> SimpleNamespace:
+        return SimpleNamespace(
+            model_id="m-stale",
+            model_name="memory-model",
+            model_info={"id": "m-stale"},
+            litellm_params={"api_key": old_api_key, "model": old_model, "rpm": 10},
+            blocked=False,
+        )
+
+    with caplog.at_level(logging.ERROR):
+        pc.decrypt_model_list_from_db(new_models=[stale_row()])
+        pc.decrypt_model_list_from_db(new_models=[stale_row()])
+
+    errors = [
+        r.getMessage() for r in caplog.records if r.levelno == logging.ERROR and "Could not decrypt" in r.getMessage()
+    ]
+    assert len(errors) == 1
+    assert "['api_key', 'model']" in errors[0]
+    assert "model_name=memory-model model_id=m-stale" in errors[0]
+    assert old_api_key not in caplog.text
+    assert "sk-provider-secret" not in caplog.text
+
+
+def test_ProxyConfig_decrypt_model_list_from_db_does_not_log_rows_under_current_key(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-new-salt-key")
+    pc = ProxyConfig()
+    m = SimpleNamespace(
+        model_id="m-ok",
+        model_name="gpt-4o",
+        model_info={"id": "m-ok"},
+        litellm_params={"api_key": encrypt_value_helper("sk-provider-secret"), "model": "openai/gpt-4o"},
+        blocked=False,
+    )
+    with caplog.at_level(logging.ERROR):
+        out = pc.decrypt_model_list_from_db(new_models=[m])
+    assert out[0]["litellm_params"]["api_key"] == "sk-provider-secret"
+    assert "Could not decrypt" not in caplog.text
+
+
 def test_ProxyConfig_decrypt_model_list_from_db_returns_decrypted(monkeypatch):
     monkeypatch.setattr(
         "litellm.proxy.proxy_server.decrypt_value_helper",