From 6358d0ed2526710a3bd41d2e76a5eced9b07b5ea Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Tue, 3 Mar 2026 15:18:16 -0800 Subject: [PATCH] Mask sensitive fields in GET response, restore env vars on failure, add change detection - GET endpoint now masks sensitive fields (vault_token, approle credentials, client_key) using SensitiveDataMasker instead of returning plaintext - POST merges missing sensitive fields from existing DB record so users can update non-secret fields without re-entering credentials - POST updates _last_hashicorp_vault_config to prevent redundant re-init on next poll - Env var rollback restores previous values on init failure instead of wiping all - Background reload skips reinit when DB config unchanged (change detection cache) - UI strips masked values from form inputs, shows them as placeholder hints instead - Form remounts on data change to prevent stale values after save - Add test for init failure env var rollback --- .../config_override_endpoints.py | 56 +++++++++++-- litellm/proxy/proxy_server.py | 15 +++- .../test_config_override_endpoints.py | 79 +++++++++++++++++-- .../HashicorpVault/HashicorpVault.tsx | 25 +++++- 4 files changed, 157 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/management_endpoints/config_override_endpoints.py b/litellm/proxy/management_endpoints/config_override_endpoints.py index e5532e3b03a..ca7cf68e01d 100644 --- a/litellm/proxy/management_endpoints/config_override_endpoints.py +++ b/litellm/proxy/management_endpoints/config_override_endpoints.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -41,6 +42,8 @@ HASHICORP_SENSITIVE_FIELDS: Set[str] = { "client_key", } +_sensitive_masker = SensitiveDataMasker() + # --- Shared helpers (reusable by future config types) --- @@ -77,6 +80,19 @@ def _decrypt_sensitive_fields( return decrypted +def _mask_sensitive_fields( + data: Dict[str, Any], sensitive_fields: Set[str] +) -> Dict[str, Any]: + """Mask sensitive fields for API responses. Non-sensitive fields are left as-is.""" + masked = {} + for key, value in data.items(): + if value is not None and key in sensitive_fields and isinstance(value, str): + masked[key] = _sensitive_masker._mask_value(value) + else: + masked[key] = value + return masked + + def _get_current_env_values(env_var_mapping: Dict[str, str]) -> Dict[str, Any]: """Read current env var values as fallback when no DB record exists.""" values = {} @@ -161,6 +177,19 @@ async def update_hashicorp_vault_config( config_data = config.model_dump(exclude_none=True) + # Merge with existing DB record: preserve sensitive fields the user didn't re-enter + existing_record = await prisma_client.db.litellm_configoverrides.find_unique( + where={"config_type": "hashicorp_vault"} + ) + if existing_record is not None and existing_record.config_value is not None: + existing_data = _parse_config_value(existing_record.config_value) + existing_decrypted = _decrypt_sensitive_fields( + existing_data, HASHICORP_SENSITIVE_FIELDS + ) + for field in HASHICORP_SENSITIVE_FIELDS: + if field not in config_data and existing_decrypted.get(field): + config_data[field] = existing_decrypted[field] + # Validate that the config has enough fields to initialize has_vault_addr = bool(config_data.get("vault_addr")) has_token_auth = bool(config_data.get("vault_token")) @@ -183,6 +212,9 @@ async def update_hashicorp_vault_config( }, ) + # Snapshot current env vars so we can restore on failure + previous_env = _get_current_env_values(HASHICORP_ENV_VAR_MAPPING) + # Set env vars and verify the secret manager can initialize before persisting _set_env_vars(config_data) @@ -191,7 +223,7 @@ async def update_hashicorp_vault_config( key_management_system="hashicorp_vault" ) except Exception as e: - _set_env_vars({}) + _set_env_vars(previous_env) verbose_proxy_logger.exception( "Error reinitializing Hashicorp Vault secret manager: %s", str(e) ) @@ -204,19 +236,23 @@ async def update_hashicorp_vault_config( # Only persist to DB after successful init encrypted_data = _encrypt_sensitive_fields(config_data, HASHICORP_SENSITIVE_FIELDS) + config_value = json.dumps(encrypted_data) await prisma_client.db.litellm_configoverrides.upsert( where={"config_type": "hashicorp_vault"}, data={ "create": { "config_type": "hashicorp_vault", - "config_value": json.dumps(encrypted_data), + "config_value": config_value, }, "update": { - "config_value": json.dumps(encrypted_data), + "config_value": config_value, }, }, ) + # Update change-detection cache so the background reload doesn't redundantly re-init + proxy_config._last_hashicorp_vault_config = json.loads(config_value) + return { "message": "Hashicorp Vault configuration updated successfully", "status": "success", @@ -260,22 +296,28 @@ async def get_hashicorp_vault_config( if db_record is not None and db_record.config_value is not None: config_data = _parse_config_value(db_record.config_value) - # Decrypt sensitive fields + # Decrypt then mask sensitive fields so plaintext secrets are never sent to the UI decrypted_data = _decrypt_sensitive_fields( config_data, HASHICORP_SENSITIVE_FIELDS ) + masked_data = _mask_sensitive_fields( + decrypted_data, HASHICORP_SENSITIVE_FIELDS + ) return ConfigOverrideSettingsResponse( config_type="hashicorp_vault", - values=decrypted_data, + values=masked_data, field_schema=field_schema, ) - # Fallback to env vars + # Fallback to env vars — also mask sensitive values env_values = _get_current_env_values(HASHICORP_ENV_VAR_MAPPING) + masked_env_values = _mask_sensitive_fields( + env_values, HASHICORP_SENSITIVE_FIELDS + ) return ConfigOverrideSettingsResponse( config_type="hashicorp_vault", - values=env_values, + values=masked_env_values, field_schema=field_schema, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 05d7e892941..c98f166ea49 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2238,6 +2238,7 @@ class ProxyConfig: def __init__(self) -> None: self.config: Dict[str, Any] = {} self._last_semantic_filter_config: Optional[Dict[str, Any]] = None + self._last_hashicorp_vault_config: Optional[Dict[str, Any]] = None def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -4558,8 +4559,10 @@ class ProxyConfig: Called periodically via _init_non_llm_objects_in_db to sync config across pods. """ from litellm.proxy.management_endpoints.config_override_endpoints import ( + HASHICORP_ENV_VAR_MAPPING, HASHICORP_SENSITIVE_FIELDS, _decrypt_sensitive_fields, + _get_current_env_values, _parse_config_value, _set_env_vars, ) @@ -4574,10 +4577,17 @@ class ProxyConfig: config_data = _parse_config_value(db_record.config_value) + # Skip reinit if config hasn't changed since last poll + if self._last_hashicorp_vault_config == config_data: + return + # Decrypt sensitive fields and set env vars decrypted_data = _decrypt_sensitive_fields( config_data, HASHICORP_SENSITIVE_FIELDS ) + + # Snapshot current env vars so we can restore on failure + previous_env = _get_current_env_values(HASHICORP_ENV_VAR_MAPPING) _set_env_vars(decrypted_data) # Reinitialize the secret manager @@ -4586,10 +4596,11 @@ class ProxyConfig: key_management_system="hashicorp_vault" ) except Exception: - # Roll back env vars so the broken config doesn't affect secret lookups - _set_env_vars({}) + # Restore previous working env vars instead of wiping all + _set_env_vars(previous_env) raise + self._last_hashicorp_vault_config = config_data.copy() verbose_proxy_logger.debug( "Hashicorp Vault config override loaded from DB" ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_config_override_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_config_override_endpoints.py index e848848b0c6..f68eb5bcb5a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_config_override_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_config_override_endpoints.py @@ -119,7 +119,7 @@ async def test_get_hashicorp_config_fallback_to_env(client, monkeypatch): @pytest.mark.asyncio async def test_get_hashicorp_config_from_db(client, monkeypatch): - """When a DB record exists, GET should return decrypted values.""" + """When a DB record exists, GET should return masked sensitive values.""" mock_record = MagicMock() mock_record.config_value = { "vault_addr": "https://vault.db.com", @@ -139,7 +139,7 @@ async def test_get_hashicorp_config_from_db(client, monkeypatch): with patch( "litellm.proxy.management_endpoints.config_override_endpoints.decrypt_value_helper" ) as mock_decrypt: - mock_decrypt.return_value = "decrypted_token" + mock_decrypt.return_value = "decrypted_token_value" app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" @@ -150,9 +150,13 @@ async def test_get_hashicorp_config_from_db(client, monkeypatch): assert response.status_code == 200 data = response.json() assert data["config_type"] == "hashicorp_vault" + # Non-sensitive fields returned as-is assert data["values"]["vault_addr"] == "https://vault.db.com" - assert data["values"]["vault_token"] == "decrypted_token" assert data["values"]["vault_namespace"] == "db-ns" + # Sensitive fields should be masked, not plaintext + vault_token_value = data["values"]["vault_token"] + assert "*" in vault_token_value + assert vault_token_value != "decrypted_token_value" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -161,6 +165,7 @@ async def test_get_hashicorp_config_from_db(client, monkeypatch): async def test_update_hashicorp_config_success(client, monkeypatch): """POST should set env vars, encrypt sensitive fields, upsert DB, and reinit secret manager.""" mock_configoverrides = MagicMock() + mock_configoverrides.find_unique = AsyncMock(return_value=None) mock_configoverrides.upsert = AsyncMock(return_value=None) mock_prisma = MagicMock() @@ -169,6 +174,7 @@ async def test_update_hashicorp_config_success(client, monkeypatch): mock_proxy_config = MagicMock() mock_proxy_config.initialize_secret_manager = MagicMock() + mock_proxy_config._last_hashicorp_vault_config = None monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.setattr(ps, "proxy_config", mock_proxy_config) @@ -215,6 +221,9 @@ async def test_update_hashicorp_config_success(client, monkeypatch): mock_proxy_config.initialize_secret_manager.assert_called_once_with( key_management_system="hashicorp_vault" ) + + # Verify change-detection cache was updated + assert mock_proxy_config._last_hashicorp_vault_config is not None finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) # Clean up env vars @@ -227,6 +236,7 @@ async def test_update_hashicorp_config_success(client, monkeypatch): async def test_update_hashicorp_config_excludes_none_fields(client, monkeypatch): """POST with partial fields should only store provided fields (None fields excluded).""" mock_configoverrides = MagicMock() + mock_configoverrides.find_unique = AsyncMock(return_value=None) mock_configoverrides.upsert = AsyncMock(return_value=None) mock_prisma = MagicMock() @@ -235,6 +245,7 @@ async def test_update_hashicorp_config_excludes_none_fields(client, monkeypatch) mock_proxy_config = MagicMock() mock_proxy_config.initialize_secret_manager = MagicMock() + mock_proxy_config._last_hashicorp_vault_config = None monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.setattr(ps, "proxy_config", mock_proxy_config) @@ -274,10 +285,64 @@ async def test_update_hashicorp_config_excludes_none_fields(client, monkeypatch) @pytest.mark.asyncio -async def test_update_hashicorp_config_missing_vault_addr(client, monkeypatch): - """POST without vault_addr should return 400.""" +async def test_update_hashicorp_config_init_failure_restores_env_vars( + client, monkeypatch +): + """When initialize_secret_manager fails, env vars should be restored to previous values and DB should not be updated.""" + mock_configoverrides = MagicMock() + mock_configoverrides.find_unique = AsyncMock(return_value=None) + mock_configoverrides.upsert = AsyncMock(return_value=None) + mock_prisma = MagicMock() mock_prisma.db = MagicMock() + mock_prisma.db.litellm_configoverrides = mock_configoverrides + + mock_proxy_config = MagicMock() + mock_proxy_config.initialize_secret_manager = MagicMock( + side_effect=Exception("Vault connection refused") + ) + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(ps, "proxy_config", mock_proxy_config) + + # Set pre-existing env vars that should be restored on failure + monkeypatch.setenv("HCP_VAULT_ADDR", "https://vault.old.com") + monkeypatch.setenv("HCP_VAULT_TOKEN", "old-token") + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.post( + "/config_overrides/hashicorp_vault", + json={ + "vault_addr": "https://vault.bad.com", + "vault_token": "bad-token", + }, + ) + assert response.status_code == 500 + assert "Vault connection refused" in response.json()["detail"]["error"] + + # Env vars should be restored to previous values, not wiped + assert os.environ.get("HCP_VAULT_ADDR") == "https://vault.old.com" + assert os.environ.get("HCP_VAULT_TOKEN") == "old-token" + + # DB should NOT have been updated + mock_configoverrides.upsert.assert_not_awaited() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_update_hashicorp_config_missing_vault_addr(client, monkeypatch): + """POST without vault_addr should return 400.""" + mock_configoverrides = MagicMock() + mock_configoverrides.find_unique = AsyncMock(return_value=None) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_configoverrides = mock_configoverrides monkeypatch.setattr(ps, "prisma_client", mock_prisma) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( @@ -298,8 +363,12 @@ async def test_update_hashicorp_config_missing_vault_addr(client, monkeypatch): @pytest.mark.asyncio async def test_update_hashicorp_config_missing_auth(client, monkeypatch): """POST with vault_addr but no auth method should return 400.""" + mock_configoverrides = MagicMock() + mock_configoverrides.find_unique = AsyncMock(return_value=None) + mock_prisma = MagicMock() mock_prisma.db = MagicMock() + mock_prisma.db.litellm_configoverrides = mock_configoverrides monkeypatch.setattr(ps, "prisma_client", mock_prisma) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx index 2a707e4f454..2674ce694b1 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx @@ -64,7 +64,17 @@ export default function HashicorpVault() { const schema = data?.field_schema; const properties = schema?.properties ?? {}; - const values = data?.values ?? {}; + const rawValues = data?.values ?? {}; + + // Strip masked sensitive values so form inputs start empty — users enter new values or leave blank to keep existing + const values: Record = {}; + for (const [key, value] of Object.entries(rawValues)) { + if (SENSITIVE_FIELDS.has(key) && typeof value === "string" && value.includes("*")) { + // Don't populate form with masked value — it would be saved literally + continue; + } + values[key] = value; + } const handleSave = (formValues: Record) => { // Only send fields that have a value @@ -94,6 +104,13 @@ export default function HashicorpVault() { ? [{ type: "url" as const, message: "Please enter a valid URL" }] : undefined; + const isSensitive = SENSITIVE_FIELDS.has(fieldName); + const maskedValue = rawValues[fieldName]; + const hasExistingValue = isSensitive && typeof maskedValue === "string" && maskedValue.includes("*"); + const placeholder = hasExistingValue + ? `Current: ${maskedValue}` + : fieldSchema?.description; + return ( - {SENSITIVE_FIELDS.has(fieldName) ? ( - + {isSensitive ? ( + ) : ( )} @@ -128,7 +145,7 @@ export default function HashicorpVault() { )} -
+ {FIELD_GROUPS.map((group, index) => (
{index > 0 && }