feat(database): add encryption support EE-1983 (#6316)

* bootstrap encryption key

* secret key message change in cli and secret key file content trimmed

* Migrate encryption code to latest version

* pull in newer code

* tidying up

* working data encryption layer

* fix tests

* remove stray comment

* fix a few minor issues and improve the comments

* split out databasefilename with param to two methods to be more obvious

* DB encryption integration (#6374)

* json methods moved under DBConnection

* store encryption fixed

* cleaned

* review comments addressed

* newstore value fixed

* backup test updated

* logrus format config updated

* Fix for newStore

Co-authored-by: Matt Hook <hookenz@gmail.com>

* Minor improvements

* Improve the export code.  Add missing webhook for import

* rename HelmUserRepositorys to HelmUserRepositories

* fix logging messages

* when starting portainer with a key (first use) http is disabled by default.  But when starting fresh without a key, http is enabled?

* Fix bug for default settings on new installs

Co-authored-by: Prabhat Khera <prabhat.khera@portainer.io>
Co-authored-by: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com>

api/datastore/backup.go

@@ -35,7 +35,7 @@ func (store *Store) createBackupFolders() {
 }
 
 func (store *Store) databasePath() string {
-	return path.Join(store.connection.GetStorePath(), store.connection.GetDatabaseFilename())
+	return store.connection.GetDatabaseFilePath()
 }
 
 func (store *Store) commonBackupDir() string {
@@ -84,7 +84,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 		options.BackupDir = store.commonBackupDir()
 	}
 	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFilename(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
 	}
 	if options.BackupPath == "" {
 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)

api/datastore/backup_test.go

@@ -48,7 +48,7 @@ func TestBackup(t *testing.T) {
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
 		store.backupWithOptions(nil)
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}

api/datastore/datastore.go

@@ -1,10 +1,15 @@
 package datastore
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"path"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/sirupsen/logrus"
 )
 
 func (store *Store) version() (int, error) {
@@ -34,6 +39,12 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
 	newStore = true
+
+	encryptionReq := store.connection.NeedsEncryptionMigration()
+	if encryptionReq {
+		store.encryptDB()
+	}
+
 	err = store.connection.Open()
 	if err != nil {
 		return newStore, err
@@ -45,8 +56,18 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
-		newStore = false
+	version, err := store.VersionService.DBVersion()
+	if err != nil {
+		if store.IsErrObjectNotFound(err) {
+			return newStore, nil
+		}
+
+		return newStore, err
+	}
+
+	if version > 0 {
+		logrus.WithField("version", version).Infof("Opened existing store")
+		return false, nil
 	}
 
 	return newStore, nil
@@ -65,16 +86,81 @@ func (store *Store) BackupTo(w io.Writer) error {
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
 	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
+		return portainerErrors.ErrWrongDBEdition
 	}
 	return nil
 }
 
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	return e == errors.ErrObjectNotFound
+	return e == portainerErrors.ErrObjectNotFound
 }
 
 func (store *Store) Rollback(force bool) error {
 	return store.connectionRollback(force)
 }
+
+func (store *Store) encryptDB() error {
+	store.connection.SetEncrypted(false)
+	err := store.connection.Open()
+	if err != nil {
+		return err
+	}
+
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// The DB is not currently encrypted.  First save the encrypted db filename
+	oldFilename := store.connection.GetDatabaseFilePath()
+	logrus.Infof("Encrypting database")
+
+	// export file path for backup
+	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
+
+	logrus.Infof("Exporting database backup to %s", exportFilename)
+	err = store.Export(exportFilename)
+	if err != nil {
+		logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
+		return err
+	}
+
+	logrus.Infof("Database backup exported")
+
+	// Close existing un-encrypted db so that we can delete the file later
+	store.connection.Close()
+
+	// Tell the db layer to create an encrypted db when opened
+	store.connection.SetEncrypted(true)
+	store.connection.Open()
+
+	// We have to init services before import
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	err = store.Import(exportFilename)
+	if err != nil {
+		// Remove the new encrypted file that we failed to import
+		os.Remove(store.connection.GetDatabaseFilePath())
+		logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
+	}
+
+	err = os.Remove(oldFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the un-encrypted db file")
+	}
+
+	err = os.Remove(exportFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the json backup file")
+	}
+
+	// Close db connection
+	store.connection.Close()
+
+	logrus.Info("Database successfully encrypted")
+	return nil
+}

api/datastore/services.go

@@ -363,118 +363,184 @@ func (store *Store) Export(filename string) (err error) {
 	backup := storeExport{}
 
 	if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Custom Templates")
+		}
 	} else {
 		backup.CustomTemplate = c
 	}
+
 	if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Groups")
+		}
 	} else {
 		backup.EdgeGroup = e
 	}
+
 	if e, err := store.EdgeJob().EdgeJobs(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Jobs")
+		}
 	} else {
 		backup.EdgeJob = e
 	}
+
 	if e, err := store.EdgeStack().EdgeStacks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Stacks")
+		}
 	} else {
 		backup.EdgeStack = e
 	}
+
 	if e, err := store.Endpoint().Endpoints(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoints")
+		}
 	} else {
 		backup.Endpoint = e
 	}
+
 	if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoint Groups")
+		}
 	} else {
 		backup.EndpointGroup = e
 	}
+
 	if r, err := store.EndpointRelation().EndpointRelations(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoint Relations")
+		}
 	} else {
 		backup.EndpointRelation = r
 	}
+
 	if r, err := store.ExtensionService.Extensions(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Extensions")
+		}
 	} else {
 		backup.Extensions = r
 	}
-	if r, err := store.HelmUserRepository().HelmUserRepositorys(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+
+	if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Helm User Repositories")
+		}
 	} else {
 		backup.HelmUserRepository = r
 	}
+
 	if r, err := store.Registry().Registries(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Registries")
+		}
 	} else {
 		backup.Registry = r
 	}
+
 	if c, err := store.ResourceControl().ResourceControls(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Resource Controls")
+		}
 	} else {
 		backup.ResourceControl = c
 	}
+
 	if role, err := store.Role().Roles(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Roles")
+		}
 	} else {
 		backup.Role = role
 	}
+
 	if r, err := store.ScheduleService.Schedules(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Schedules")
+		}
 	} else {
 		backup.Schedules = r
 	}
+
 	if settings, err := store.Settings().Settings(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Settings")
+		}
 	} else {
 		backup.Settings = *settings
 	}
+
 	if settings, err := store.SSLSettings().Settings(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting SSL Settings")
+		}
 	} else {
 		backup.SSLSettings = *settings
 	}
+
 	if t, err := store.Stack().Stacks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Stacks")
+		}
 	} else {
 		backup.Stack = t
 	}
+
 	if t, err := store.Tag().Tags(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Tags")
+		}
 	} else {
 		backup.Tag = t
 	}
+
 	if t, err := store.TeamMembership().TeamMemberships(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Team Memberships")
+		}
 	} else {
 		backup.TeamMembership = t
 	}
+
 	if t, err := store.Team().Teams(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Teams")
+		}
 	} else {
 		backup.Team = t
 	}
+
 	if info, err := store.TunnelServer().Info(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Tunnel Server")
+		}
 	} else {
 		backup.TunnelServer = *info
 	}
+
 	if users, err := store.User().Users(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Users")
+		}
 	} else {
 		backup.User = users
 	}
+
 	if webhooks, err := store.Webhook().Webhooks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Webhooks")
+		}
 	} else {
 		backup.Webhook = webhooks
 	}
+
 	v, err := store.Version().DBVersion()
-	if err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+	if err != nil && !store.IsErrObjectNotFound(err) {
+		logrus.WithError(err).Errorf("Exporting DB version")
 	}
 	instance, _ := store.Version().InstanceID()
 	backup.Version = map[string]string{
@@ -518,50 +584,66 @@ func (store *Store) Import(filename string) (err error) {
 	for _, v := range backup.CustomTemplate {
 		store.CustomTemplate().UpdateCustomTemplate(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeGroup {
 		store.EdgeGroup().UpdateEdgeGroup(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeJob {
 		store.EdgeJob().UpdateEdgeJob(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeStack {
 		store.EdgeStack().UpdateEdgeStack(v.ID, &v)
 	}
+
 	for _, v := range backup.Endpoint {
 		store.Endpoint().UpdateEndpoint(v.ID, &v)
 	}
+
 	for _, v := range backup.EndpointGroup {
 		store.EndpointGroup().UpdateEndpointGroup(v.ID, &v)
 	}
+
 	for _, v := range backup.EndpointRelation {
 		store.EndpointRelation().UpdateEndpointRelation(v.EndpointID, &v)
 	}
+
 	for _, v := range backup.HelmUserRepository {
 		store.HelmUserRepository().UpdateHelmUserRepository(v.ID, &v)
 	}
+
 	for _, v := range backup.Registry {
 		store.Registry().UpdateRegistry(v.ID, &v)
 	}
+
 	for _, v := range backup.ResourceControl {
 		store.ResourceControl().UpdateResourceControl(v.ID, &v)
 	}
+
 	for _, v := range backup.Role {
 		store.Role().UpdateRole(v.ID, &v)
 	}
+
 	store.Settings().UpdateSettings(&backup.Settings)
 	store.SSLSettings().UpdateSettings(&backup.SSLSettings)
+
 	for _, v := range backup.Stack {
 		store.Stack().UpdateStack(v.ID, &v)
 	}
+
 	for _, v := range backup.Tag {
 		store.Tag().UpdateTag(v.ID, &v)
 	}
+
 	for _, v := range backup.TeamMembership {
 		store.TeamMembership().UpdateTeamMembership(v.ID, &v)
 	}
+
 	for _, v := range backup.Team {
 		store.Team().UpdateTeam(v.ID, &v)
 	}
+
 	store.TunnelServer().UpdateInfo(&backup.TunnelServer)
 
 	for _, user := range backup.User {
@@ -570,10 +652,9 @@ func (store *Store) Import(filename string) (err error) {
 		}
 	}
 
-	// backup[store.Webhook().BucketName()], err = store.Webhook().Webhooks()
-	// if err != nil {
-	// 	logrus.WithError(err).Debugf("Export boom")
-	// }
+	for _, v := range backup.Webhook {
+		store.Webhook().UpdateWebhook(v.ID, &v)
+	}
 
 	return nil
 }

api/datastore/teststore.go

@@ -42,7 +42,7 @@ func NewTestStore(init bool) (bool, *Store, func(), error) {
 		return false, nil, nil, err
 	}
 
-	connection, err := database.NewDatabase("boltdb", storePath)
+	connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes"))
 	if err != nil {
 		panic(err)
 	}