feat(database): add encryption support EE-1983 (#6316)

* bootstrap encryption key

* secret key message change in cli and secret key file content trimmed

* Migrate encryption code to latest version

* pull in newer code

* tidying up

* working data encryption layer

* fix tests

* remove stray comment

* fix a few minor issues and improve the comments

* split out databasefilename with param to two methods to be more obvious

* DB encryption integration (#6374)

* json methods moved under DBConnection

* store encryption fixed

* cleaned

* review comments addressed

* newstore value fixed

* backup test updated

* logrus format config updated

* Fix for newStore

Co-authored-by: Matt Hook <hookenz@gmail.com>

* Minor improvements

* Improve the export code.  Add missing webhook for import

* rename HelmUserRepositorys to HelmUserRepositories

* fix logging messages

* when starting portainer with a key (first use) http is disabled by default.  But when starting fresh without a key, http is enabled?

* Fix bug for default settings on new installs

Co-authored-by: Prabhat Khera <prabhat.khera@portainer.io>
Co-authored-by: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com>

api/datastore/datastore.go

@@ -1,10 +1,15 @@
 package datastore
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"path"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/sirupsen/logrus"
 )
 
 func (store *Store) version() (int, error) {
@@ -34,6 +39,12 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
 	newStore = true
+
+	encryptionReq := store.connection.NeedsEncryptionMigration()
+	if encryptionReq {
+		store.encryptDB()
+	}
+
 	err = store.connection.Open()
 	if err != nil {
 		return newStore, err
@@ -45,8 +56,18 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
-		newStore = false
+	version, err := store.VersionService.DBVersion()
+	if err != nil {
+		if store.IsErrObjectNotFound(err) {
+			return newStore, nil
+		}
+
+		return newStore, err
+	}
+
+	if version > 0 {
+		logrus.WithField("version", version).Infof("Opened existing store")
+		return false, nil
 	}
 
 	return newStore, nil
@@ -65,16 +86,81 @@ func (store *Store) BackupTo(w io.Writer) error {
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
 	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
+		return portainerErrors.ErrWrongDBEdition
 	}
 	return nil
 }
 
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	return e == errors.ErrObjectNotFound
+	return e == portainerErrors.ErrObjectNotFound
 }
 
 func (store *Store) Rollback(force bool) error {
 	return store.connectionRollback(force)
 }
+
+func (store *Store) encryptDB() error {
+	store.connection.SetEncrypted(false)
+	err := store.connection.Open()
+	if err != nil {
+		return err
+	}
+
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// The DB is not currently encrypted.  First save the encrypted db filename
+	oldFilename := store.connection.GetDatabaseFilePath()
+	logrus.Infof("Encrypting database")
+
+	// export file path for backup
+	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
+
+	logrus.Infof("Exporting database backup to %s", exportFilename)
+	err = store.Export(exportFilename)
+	if err != nil {
+		logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
+		return err
+	}
+
+	logrus.Infof("Database backup exported")
+
+	// Close existing un-encrypted db so that we can delete the file later
+	store.connection.Close()
+
+	// Tell the db layer to create an encrypted db when opened
+	store.connection.SetEncrypted(true)
+	store.connection.Open()
+
+	// We have to init services before import
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	err = store.Import(exportFilename)
+	if err != nil {
+		// Remove the new encrypted file that we failed to import
+		os.Remove(store.connection.GetDatabaseFilePath())
+		logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
+	}
+
+	err = os.Remove(oldFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the un-encrypted db file")
+	}
+
+	err = os.Remove(exportFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the json backup file")
+	}
+
+	// Close db connection
+	store.connection.Close()
+
+	logrus.Info("Database successfully encrypted")
+	return nil
+}