diff --git a/api/cli/cli.go b/api/cli/cli.go index ce9504047..973bd0099 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -57,6 +57,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { Logo: kingpin.Flag("logo", "URL for the logo displayed in the UI").String(), Templates: kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(), BaseURL: kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(), + SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/.").Default(defaultSecretKeyName).String(), } kingpin.Parse() diff --git a/api/cli/defaults.go b/api/cli/defaults.go index 84c054bed..fe3dde208 100644 --- a/api/cli/defaults.go +++ b/api/cli/defaults.go @@ -16,10 +16,11 @@ const ( defaultTLSCertPath = "/certs/cert.pem" defaultTLSKeyPath = "/certs/key.pem" defaultHTTPDisabled = "false" - defaultHTTPEnabled = "false" + defaultHTTPEnabled = "true" defaultSSL = "false" defaultSSLCertPath = "/certs/portainer.crt" defaultSSLKeyPath = "/certs/portainer.key" defaultSnapshotInterval = "5m" defaultBaseURL = "/" + defaultSecretKeyName = "portainer" ) diff --git a/api/cli/defaults_windows.go b/api/cli/defaults_windows.go index fdb4be0ab..007c7af69 100644 --- a/api/cli/defaults_windows.go +++ b/api/cli/defaults_windows.go @@ -13,10 +13,11 @@ const ( defaultTLSCertPath = "C:\\certs\\cert.pem" defaultTLSKeyPath = "C:\\certs\\key.pem" defaultHTTPDisabled = "false" - defaultHTTPEnabled = "false" + defaultHTTPEnabled = "true" defaultSSL = "false" defaultSSLCertPath = "C:\\certs\\portainer.crt" defaultSSLKeyPath = "C:\\certs\\portainer.key" defaultSnapshotInterval = "5m" defaultBaseURL = "/" + defaultSecretKeyName = "portainer" ) diff --git a/api/cmd/portainer/import.go b/api/cmd/portainer/import.go index c38cace30..bb71b111f 100644 --- a/api/cmd/portainer/import.go +++ b/api/cmd/portainer/import.go @@ -13,7 +13,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) { importFile := "/data/import.json" if exists, _ := fileService.FileExists(importFile); exists { if err := store.Import(importFile); err != nil { - logrus.WithError(err).Debugf("import %s failed", importFile) + logrus.WithError(err).Debugf("Import %s failed", importFile) // TODO: should really rollback on failure, but then we have nothing. } else { @@ -23,7 +23,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) { // I also suspect that everything from "Init to Init" is potentially a migration err := store.Init() if err != nil { - log.Fatalf("failed initializing data store: %v", err) + log.Fatalf("Failed initializing data store: %v", err) } } } diff --git a/api/cmd/portainer/log.go b/api/cmd/portainer/log.go index 5bab3c5de..570fb3554 100644 --- a/api/cmd/portainer/log.go +++ b/api/cmd/portainer/log.go @@ -1,18 +1,41 @@ package main import ( + "fmt" "log" + "strings" "github.com/sirupsen/logrus" ) +type portainerFormatter struct { + logrus.TextFormatter +} + +func (f *portainerFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var levelColor int + switch entry.Level { + case logrus.DebugLevel, logrus.TraceLevel: + levelColor = 31 // gray + case logrus.WarnLevel: + levelColor = 33 // yellow + case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel: + levelColor = 31 // red + default: + levelColor = 36 // blue + } + return []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m %s %s\n", levelColor, strings.ToUpper(entry.Level.String()), entry.Time.Format(f.TimestampFormat), entry.Message)), nil +} + func configureLogger() { logger := logrus.New() // logger is to implicitly substitute stdlib's log log.SetOutput(logger.Writer()) formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true} + formatterLogrus := &portainerFormatter{logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true, TimestampFormat: "2006/01/02 15:04:05", FullTimestamp: true}} + logger.SetFormatter(formatter) - logrus.SetFormatter(formatter) + logrus.SetFormatter(formatterLogrus) logger.SetLevel(logrus.DebugLevel) logrus.SetLevel(logrus.DebugLevel) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 9c16792d6..3291ebfe4 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "fmt" "log" "os" @@ -47,12 +48,12 @@ func initCLI() *portainer.CLIFlags { var cliService portainer.CLIService = &cli.Service{} flags, err := cliService.ParseFlags(portainer.APIVersion) if err != nil { - log.Fatalf("failed parsing flags: %v", err) + log.Fatalf("Failed parsing flags: %v", err) } err = cliService.ValidateFlags(flags) if err != nil { - log.Fatalf("failed validating flags:%v", err) + log.Fatalf("Failed validating flags:%v", err) } return flags } @@ -60,26 +61,26 @@ func initCLI() *portainer.CLIFlags { func initFileService(dataStorePath string) portainer.FileService { fileService, err := filesystem.NewService(dataStorePath, "") if err != nil { - log.Fatalf("failed creating file service: %v", err) + log.Fatalf("Failed creating file service: %v", err) } return fileService } -func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore { - connection, err := database.NewDatabase("boltdb", *flags.Data) +func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore { + connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey) if err != nil { - panic(err) + panic(err.Error()) } store := datastore.NewStore(*flags.Data, fileService, connection) isNew, err := store.Open() if err != nil { - log.Fatalf("failed opening store: %v", err) + log.Fatalf("Failed opening store: %v", err) } if *flags.Rollback { err := store.Rollback(false) if err != nil { - log.Fatalf("failed rolling back: %s", err) + log.Fatalf("Failed rolling back: %v", err) } log.Println("Exiting rollback") @@ -90,31 +91,27 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, // Init sets some defaults - its basically a migration err = store.Init() if err != nil { - log.Fatalf("failed initializing data store: %v", err) + log.Fatalf("Failed initializing data store: %v", err) } if isNew { // from MigrateData store.VersionService.StoreDBVersion(portainer.DBVersion) - // Disabled for now. Can't use feature flags due to the way that works - // EXPERIMENTAL, will only activate if `/data/import.json` exists - //importFromJson(fileService, store) - err := updateSettingsFromFlags(store, flags) if err != nil { - log.Fatalf("failed updating settings from flags: %v", err) + log.Fatalf("Failed updating settings from flags: %v", err) } - } - - storedVersion, err := store.VersionService.DBVersion() - if err != nil { - log.Fatalf("Something failed during creation of new database: %v", err) - } - if storedVersion != portainer.DBVersion { - err = store.MigrateData() + } else { + storedVersion, err := store.VersionService.DBVersion() if err != nil { - log.Fatalf("failed migration: %v", err) + log.Fatalf("Something Failed during creation of new database: %v", err) + } + if storedVersion != portainer.DBVersion { + err = store.MigrateData() + if err != nil { + log.Fatalf("Failed migration: %v", err) + } } } @@ -127,7 +124,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, err := store.Export(exportFilename) if err != nil { - logrus.WithError(err).Debugf("failed to export to %s", exportFilename) + logrus.WithError(err).Debugf("Failed to export to %s", exportFilename) } else { logrus.Debugf("exported to %s", exportFilename) } @@ -139,7 +136,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager { composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager) if err != nil { - log.Fatalf("failed creating compose manager: %s", err) + log.Fatalf("Failed creating compose manager: %v", err) } return composeWrapper @@ -347,7 +344,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error { existingKeyPair, err := fileService.KeyPairFilesExist() if err != nil { - log.Fatalf("failed checking for existing key pair: %v", err) + log.Fatalf("Failed checking for existing key pair: %v", err) } if existingKeyPair { @@ -491,19 +488,40 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService) } +func loadEncryptionSecretKey(keyfilename string) []byte { + content, err := os.ReadFile(path.Join("/run/secrets", keyfilename)) + if err != nil { + if os.IsNotExist(err) { + log.Printf("Encryption key file `%s` not present", keyfilename) + } else { + log.Printf("Error reading encryption key file: %v", err) + } + + return nil + } + + // return a 32 byte hash of the secret (required for AES) + hash := sha256.Sum256(content) + return hash[:] +} + func buildServer(flags *portainer.CLIFlags) portainer.Server { shutdownCtx, shutdownTrigger := context.WithCancel(context.Background()) fileService := initFileService(*flags.Data) + encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName) + if encryptionKey == nil { + log.Println("proceeding without encryption key") + } - dataStore := initDataStore(flags, fileService, shutdownCtx) + dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx) if err := dataStore.CheckCurrentEdition(); err != nil { log.Fatal(err) } instanceID, err := dataStore.Version().InstanceID() if err != nil { - log.Fatalf("failed getting instance id: %v", err) + log.Fatalf("Failed getting instance id: %v", err) } apiKeyService := initAPIKeyService(dataStore) @@ -514,12 +532,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { } jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore) if err != nil { - log.Fatalf("failed initializing JWT service: %v", err) + log.Fatalf("Failed initializing JWT service: %v", err) } err = enableFeaturesFromFlags(dataStore, flags) if err != nil { - log.Fatalf("failed enabling feature flag: %v", err) + log.Fatalf("Failed enabling feature flag: %v", err) } ldapService := initLDAPService() @@ -538,12 +556,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { sslSettings, err := sslService.GetSSLSettings() if err != nil { - log.Fatalf("failed to get ssl settings: %s", err) + log.Fatalf("Failed to get ssl settings: %s", err) } err = initKeyPair(fileService, digitalSignatureService) if err != nil { - log.Fatalf("failed initializing key pair: %v", err) + log.Fatalf("Failed initializing key pair: %v", err) } reverseTunnelService := chisel.NewService(dataStore, shutdownCtx) @@ -553,7 +571,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx) if err != nil { - log.Fatalf("failed initializing snapshot service: %v", err) + log.Fatalf("Failed initializing snapshot service: %v", err) } snapshotService.Start() @@ -574,37 +592,37 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore) if err != nil { - log.Fatalf("failed initializing swarm stack manager: %s", err) + log.Fatalf("Failed initializing swarm stack manager: %v", err) } kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets) helmPackageManager, err := initHelmPackageManager(*flags.Assets) if err != nil { - log.Fatalf("failed initializing helm package manager: %s", err) + log.Fatalf("Failed initializing helm package manager: %v", err) } err = edge.LoadEdgeJobs(dataStore, reverseTunnelService) if err != nil { - log.Fatalf("failed loading edge jobs from database: %v", err) + log.Fatalf("Failed loading edge jobs from database: %v", err) } applicationStatus := initStatus(instanceID) err = initEndpoint(flags, dataStore, snapshotService) if err != nil { - log.Fatalf("failed initializing environment: %v", err) + log.Fatalf("Failed initializing environment: %v", err) } adminPasswordHash := "" if *flags.AdminPasswordFile != "" { content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "") if err != nil { - log.Fatalf("failed getting admin password file: %v", err) + log.Fatalf("Failed getting admin password file: %v", err) } adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n")) if err != nil { - log.Fatalf("failed hashing admin password: %v", err) + log.Fatalf("Failed hashing admin password: %v", err) } } else if *flags.AdminPassword != "" { adminPasswordHash = *flags.AdminPassword @@ -613,7 +631,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { if adminPasswordHash != "" { users, err := dataStore.User().UsersByRole(portainer.AdministratorRole) if err != nil { - log.Fatalf("failed getting admin user: %v", err) + log.Fatalf("Failed getting admin user: %v", err) } if len(users) == 0 { @@ -625,7 +643,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { } err := dataStore.User().Create(user) if err != nil { - log.Fatalf("failed creating admin user: %v", err) + log.Fatalf("Failed creating admin user: %v", err) } } else { log.Println("Instance already has an administrator user defined. Skipping admin password related flags.") @@ -634,12 +652,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService) if err != nil { - log.Fatalf("failed starting tunnel server: %s", err) + log.Fatalf("Failed starting tunnel server: %v", err) } sslDBSettings, err := dataStore.SSLSettings().Settings() if err != nil { - log.Fatalf("failed to fetch ssl settings from DB") + log.Fatalf("Failed to fetch ssl settings from DB") } scheduler := scheduler.NewScheduler(shutdownCtx) @@ -692,6 +710,6 @@ func main() { server := buildServer(flags) log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion) err := server.Start() - log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err) + log.Printf("[INFO] [cmd,main] Http server exited: %v\n", err) } } diff --git a/api/connection.go b/api/connection.go index 565f9cd51..befadaae3 100644 --- a/api/connection.go +++ b/api/connection.go @@ -11,14 +11,16 @@ type Connection interface { // write the db contents to filename as json (the schema needs defining) ExportRaw(filename string) error - //Rollback(force bool) error - //MigrateData(migratorParams *database.MigratorParameters, force bool) error - // TODO: this one is very database specific atm BackupTo(w io.Writer) error - GetDatabaseFilename() string + GetDatabaseFileName() string + GetDatabaseFilePath() string GetStorePath() string + IsEncryptedStore() bool + NeedsEncryptionMigration() bool + SetEncrypted(encrypted bool) + SetServiceName(bucketName string) error GetObject(bucketName string, key []byte, object interface{}) error UpdateObject(bucketName string, key []byte, object interface{}) error diff --git a/api/database/boltdb/db.go b/api/database/boltdb/db.go index 4a4de29ef..1ccf708fc 100644 --- a/api/database/boltdb/db.go +++ b/api/database/boltdb/db.go @@ -11,39 +11,78 @@ import ( "github.com/boltdb/bolt" "github.com/portainer/portainer/api/dataservices/errors" + "github.com/sirupsen/logrus" ) const ( - DatabaseFileName = "portainer.db" + DatabaseFileName = "portainer.db" + EncryptedDatabaseFileName = "portainer.edb" ) type DbConnection struct { - Path string + Path string + EncryptionKey []byte + isEncrypted bool *bolt.DB } -func (connection *DbConnection) GetDatabaseFilename() string { +// GetDatabaseFileName get the database filename +func (connection *DbConnection) GetDatabaseFileName() string { + if connection.IsEncryptedStore() { + return EncryptedDatabaseFileName + } + return DatabaseFileName } +// GetDataseFilePath get the path + filename for the database file +func (connection *DbConnection) GetDatabaseFilePath() string { + if connection.IsEncryptedStore() { + return path.Join(connection.Path, EncryptedDatabaseFileName) + } + + return path.Join(connection.Path, DatabaseFileName) +} + +// GetStorePath get the filename and path for the database file func (connection *DbConnection) GetStorePath() string { return connection.Path } +func (connection *DbConnection) SetEncrypted(flag bool) { + connection.isEncrypted = flag +} + +// Return true if the database is encrypted +func (connection *DbConnection) IsEncryptedStore() bool { + return connection.getEncryptionKey() != nil +} + +// NeedsEncryptionMigration returns true if database encryption is enabled and +// we have an un-encrypted DB that requires migration to an encrypted DB +func (connection *DbConnection) NeedsEncryptionMigration() bool { + if connection.EncryptionKey != nil { + dbFile := path.Join(connection.Path, DatabaseFileName) + if _, err := os.Stat(dbFile); err == nil { + return true + } + + // This is an existing encrypted store or a new store. + // A new store will open encrypted from the outset + connection.SetEncrypted(true) + } + + return false +} + // Open opens and initializes the BoltDB database. func (connection *DbConnection) Open() error { - // Disabled for now. Can't use feature flags due to the way that works - // databaseExportPath := path.Join(connection.Path, fmt.Sprintf("raw-%s-%d.json", DatabaseFileName, time.Now().Unix())) - // if err := connection.ExportRaw(databaseExportPath); err != nil { - // log.Printf("raw export to %s error: %s", databaseExportPath, err) - // } else { - // log.Printf("raw export to %s success", databaseExportPath) - // } - - databasePath := path.Join(connection.Path, DatabaseFileName) + logrus.Infof("Loading PortainerDB: %s", connection.GetDatabaseFileName()) + // Now we open the db + databasePath := connection.GetDatabaseFilePath() db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second}) if err != nil { return err @@ -71,12 +110,12 @@ func (connection *DbConnection) BackupTo(w io.Writer) error { } func (connection *DbConnection) ExportRaw(filename string) error { - databasePath := path.Join(connection.Path, DatabaseFileName) + databasePath := connection.GetDatabaseFilePath() if _, err := os.Stat(databasePath); err != nil { return fmt.Errorf("stat on %s failed: %s", databasePath, err) } - b, err := exportJson(databasePath) + b, err := connection.exportJson(databasePath) if err != nil { return err } @@ -124,7 +163,15 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object return err } - return UnmarshalObject(data, object) + return connection.UnmarshalObject(data, object) +} + +func (connection *DbConnection) getEncryptionKey() []byte { + if !connection.isEncrypted { + return nil + } + + return connection.EncryptionKey } // UpdateObject is a generic function used to update an object inside a database database. @@ -132,7 +179,7 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje return connection.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) - data, err := MarshalObject(object) + data, err := connection.MarshalObject(object) if err != nil { return err } @@ -163,7 +210,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun cursor := bucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { var obj interface{} - err := UnmarshalObject(v, &obj) + err := connection.UnmarshalObject(v, &obj) if err != nil { return err } @@ -205,7 +252,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) seqId, _ := bucket.NextSequence() id, obj := fn(seqId) - data, err := MarshalObject(obj) + data, err := connection.MarshalObject(obj) if err != nil { return err } @@ -218,8 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error { return connection.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) - - data, err := MarshalObject(obj) + data, err := connection.MarshalObject(obj) if err != nil { return err } @@ -240,7 +286,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i return err } - data, err := MarshalObject(obj) + data, err := connection.MarshalObject(obj) if err != nil { return err } @@ -252,10 +298,9 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error { err := connection.View(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) - cursor := bucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { - err := UnmarshalObject(v, obj) + err := connection.UnmarshalObject(v, obj) if err != nil { return err } @@ -277,7 +322,7 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf cursor := bucket.Cursor() for k, v := cursor.First(); k != nil; k, v = cursor.Next() { - err := UnmarshalObjectWithJsoniter(v, obj) + err := connection.UnmarshalObjectWithJsoniter(v, obj) if err != nil { return err } diff --git a/api/database/boltdb/export.go b/api/database/boltdb/export.go index fa4e56dbc..3d04f2aab 100644 --- a/api/database/boltdb/export.go +++ b/api/database/boltdb/export.go @@ -10,8 +10,9 @@ import ( // inspired by github.com/konoui/boltdb-exporter (which has no license) // but very much simplified, based on how we use boltdb +func (c *DbConnection) exportJson(databasePath string) ([]byte, error) { + logrus.WithField("databasePath", databasePath).Infof("exportJson") -func exportJson(databasePath string) ([]byte, error) { connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true}) if err != nil { return []byte("{}"), err @@ -31,7 +32,7 @@ func exportJson(databasePath string) ([]byte, error) { continue } var obj interface{} - err := UnmarshalObject(v, &obj) + err := c.UnmarshalObject(v, &obj) if err != nil { logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v)) obj = v diff --git a/api/database/boltdb/json.go b/api/database/boltdb/json.go index d2f70dbe8..b4b754e68 100644 --- a/api/database/boltdb/json.go +++ b/api/database/boltdb/json.go @@ -1,42 +1,123 @@ package boltdb import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" "encoding/json" + "fmt" + "io" jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" ) +var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short") + // MarshalObject encodes an object to binary format -func MarshalObject(object interface{}) ([]byte, error) { +func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) { // Special case for the VERSION bucket. Here we're not using json if v, ok := object.(string); ok { - return []byte(v), nil + data = []byte(v) + } else { + data, err = json.Marshal(object) + if err != nil { + return data, err + } } - - return json.Marshal(object) + if connection.getEncryptionKey() == nil { + return data, nil + } + return encrypt(data, connection.getEncryptionKey()) } // UnmarshalObject decodes an object from binary data -func UnmarshalObject(data []byte, object interface{}) error { - // Special case for the VERSION bucket. Here we're not using json - // So we need to return it as a string - err := json.Unmarshal(data, object) - if err != nil { - if s, ok := object.(*string); ok { - *s = string(data) - return nil +func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error { + var err error + if connection.getEncryptionKey() != nil { + data, err = decrypt(data, connection.getEncryptionKey()) + if err != nil { + errors.Wrapf(err, "Failed decrypting object") + } + } + e := json.Unmarshal(data, object) + if e != nil { + // Special case for the VERSION bucket. Here we're not using json + // So we need to return it as a string + s, ok := object.(*string) + if !ok { + return errors.Wrap(err, e.Error()) } - return err + *s = string(data) } - - return nil + return err } // UnmarshalObjectWithJsoniter decodes an object from binary data // using the jsoniter library. It is mainly used to accelerate environment(endpoint) // decoding at the moment. -func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error { +func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error { + if connection.getEncryptionKey() != nil { + var err error + data, err = decrypt(data, connection.getEncryptionKey()) + if err != nil { + return err + } + } var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary return jsoni.Unmarshal(data, &object) } + +// mmm, don't have a KMS .... aes GCM seems the most likely from +// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption + +func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) { + block, _ := aes.NewCipher(passphrase) + gcm, err := cipher.NewGCM(block) + if err != nil { + return encrypted, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return encrypted, err + } + ciphertextByte := gcm.Seal( + nonce, + nonce, + plaintext, + nil) + return ciphertextByte, nil +} + +func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) { + if string(encrypted) == "false" { + return []byte("false"), nil + } + block, err := aes.NewCipher(passphrase) + if err != nil { + return encrypted, errors.Wrap(err, "Error creating cypher block") + } + + gcm, err := cipher.NewGCM(block) + if err != nil { + return encrypted, errors.Wrap(err, "Error creating GCM") + } + + nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return encrypted, errEncryptedStringTooShort + } + + nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:] + plaintextByte, err = gcm.Open( + nil, + nonce, + ciphertextByteClean, + nil) + if err != nil { + return encrypted, errors.Wrap(err, "Error decrypting text") + } + + return plaintextByte, err +} diff --git a/api/database/boltdb/json_test.go b/api/database/boltdb/json_test.go index d60e73d6e..712621f10 100644 --- a/api/database/boltdb/json_test.go +++ b/api/database/boltdb/json_test.go @@ -1,6 +1,7 @@ package boltdb import ( + "crypto/sha256" "fmt" "testing" @@ -8,9 +9,17 @@ import ( "github.com/stretchr/testify/assert" ) -const jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}` +const ( + jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}` + passphrase = "my secret key" +) -func Test_MarshalObject(t *testing.T) { +func secretToEncryptionKey(passphrase string) []byte { + hash := sha256.Sum256([]byte(passphrase)) + return hash[:] +} + +func Test_MarshalObjectUnencrypted(t *testing.T) { is := assert.New(t) uuid := uuid.Must(uuid.NewV4()) @@ -73,16 +82,18 @@ func Test_MarshalObject(t *testing.T) { }, } + conn := DbConnection{} + for _, test := range tests { t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) { - data, err := MarshalObject(test.object) + data, err := conn.MarshalObject(test.object) is.NoError(err) is.Equal(test.expected, string(data)) }) } } -func Test_UnMarshalObject(t *testing.T) { +func Test_UnMarshalObjectUnencrypted(t *testing.T) { is := assert.New(t) // Based on actual data entering and what we expect out of the function @@ -105,18 +116,62 @@ func Test_UnMarshalObject(t *testing.T) { expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6", }, { - // An unmarshalled json object string should return the same as a string without error also + // An un-marshalled json object string should return the same as a string without error also object: []byte(jsonobject), expected: jsonobject, }, } + conn := DbConnection{} + for _, test := range tests { t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) { var object string - err := UnmarshalObject(test.object, &object) + err := conn.UnmarshalObject(test.object, &object) is.NoError(err) is.Equal(test.expected, string(object)) }) } } + +func Test_ObjectMarshallingEncrypted(t *testing.T) { + is := assert.New(t) + + // Based on actual data entering and what we expect out of the function + + tests := []struct { + object []byte + expected string + }{ + { + object: []byte(""), + }, + { + object: []byte("35"), + }, + { + // An unmarshalled byte string should return the same without error + object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"), + }, + { + // An un-marshalled json object string should return the same as a string without error also + object: []byte(jsonobject), + }, + } + + key := secretToEncryptionKey(passphrase) + conn := DbConnection{EncryptionKey: key} + for _, test := range tests { + t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) { + + data, err := conn.MarshalObject(test.object) + is.NoError(err) + + var object []byte + err = conn.UnmarshalObject(data, &object) + + is.NoError(err) + is.Equal(test.object, object) + }) + } +} diff --git a/api/database/database.go b/api/database/database.go index d12381639..03feea8e9 100644 --- a/api/database/database.go +++ b/api/database/database.go @@ -2,15 +2,19 @@ package database import ( "fmt" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/database/boltdb" ) // NewDatabase should use config options to return a connection to the requested database -func NewDatabase(storeType, storePath string) (connection portainer.Connection, err error) { +func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) { switch storeType { case "boltdb": - return &boltdb.DbConnection{Path: storePath}, nil + return &boltdb.DbConnection{ + Path: storePath, + EncryptionKey: encryptionKey, + }, nil } - return nil, fmt.Errorf("Unknown storage database: %s", storeType) + return nil, fmt.Errorf("unknown storage database: %s", storeType) } diff --git a/api/dataservices/endpoint/endpoint.go b/api/dataservices/endpoint/endpoint.go index 2660343ad..a8f9294b5 100644 --- a/api/dataservices/endpoint/endpoint.go +++ b/api/dataservices/endpoint/endpoint.go @@ -69,7 +69,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) { endpoint, ok := obj.(*portainer.Endpoint) if !ok { logrus.WithField("obj", obj).Errorf("Failed to convert to Endpoint object") - return nil, fmt.Errorf("Failed to convert to Endpoint object: %s", obj) + return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj) } endpoints = append(endpoints, *endpoint) return &portainer.Endpoint{}, nil diff --git a/api/dataservices/errors/errors.go b/api/dataservices/errors/errors.go index b88911590..1a06c8972 100644 --- a/api/dataservices/errors/errors.go +++ b/api/dataservices/errors/errors.go @@ -4,6 +4,7 @@ import "errors" var ( // TODO: i'm pretty sure this needs wrapping at several levels - ErrObjectNotFound = errors.New("Object not found inside the database") - ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/") + ErrObjectNotFound = errors.New("object not found inside the database") + ErrWrongDBEdition = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/") + ErrDBImportFailed = errors.New("importing backup failed") ) diff --git a/api/dataservices/helmuserrepository/helmuserrepository.go b/api/dataservices/helmuserrepository/helmuserrepository.go index 28bc139e9..0ae3a3564 100644 --- a/api/dataservices/helmuserrepository/helmuserrepository.go +++ b/api/dataservices/helmuserrepository/helmuserrepository.go @@ -34,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) { } //HelmUserRepository returns an array of all HelmUserRepository -func (service *Service) HelmUserRepositorys() ([]portainer.HelmUserRepository, error) { +func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) { var repos = make([]portainer.HelmUserRepository, 0) err := service.connection.GetAll( diff --git a/api/dataservices/interface.go b/api/dataservices/interface.go index dff951282..fecfe6666 100644 --- a/api/dataservices/interface.go +++ b/api/dataservices/interface.go @@ -124,7 +124,7 @@ type ( // HelmUserRepositoryService represents a service to manage HelmUserRepositories HelmUserRepositoryService interface { - HelmUserRepositorys() ([]portainer.HelmUserRepository, error) + HelmUserRepositories() ([]portainer.HelmUserRepository, error) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) Create(record *portainer.HelmUserRepository) error UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error diff --git a/api/datastore/backup.go b/api/datastore/backup.go index 939058924..f5f972c63 100644 --- a/api/datastore/backup.go +++ b/api/datastore/backup.go @@ -35,7 +35,7 @@ func (store *Store) createBackupFolders() { } func (store *Store) databasePath() string { - return path.Join(store.connection.GetStorePath(), store.connection.GetDatabaseFilename()) + return store.connection.GetDatabaseFilePath() } func (store *Store) commonBackupDir() string { @@ -84,7 +84,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions { options.BackupDir = store.commonBackupDir() } if options.BackupFileName == "" { - options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFilename(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405")) + options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405")) } if options.BackupPath == "" { options.BackupPath = path.Join(options.BackupDir, options.BackupFileName) diff --git a/api/datastore/backup_test.go b/api/datastore/backup_test.go index 8134245fb..19ee34951 100644 --- a/api/datastore/backup_test.go +++ b/api/datastore/backup_test.go @@ -48,7 +48,7 @@ func TestBackup(t *testing.T) { store.VersionService.StoreDBVersion(portainer.DBVersion) store.backupWithOptions(nil) - backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion)) + backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion)) if !isFileExist(backupFileName) { t.Errorf("Expect backup file to be created %s", backupFileName) } diff --git a/api/datastore/datastore.go b/api/datastore/datastore.go index 0decf4c97..652f29db1 100644 --- a/api/datastore/datastore.go +++ b/api/datastore/datastore.go @@ -1,10 +1,15 @@ package datastore import ( + "fmt" "io" + "os" + "path" + "time" portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/dataservices/errors" + portainerErrors "github.com/portainer/portainer/api/dataservices/errors" + "github.com/sirupsen/logrus" ) func (store *Store) version() (int, error) { @@ -34,6 +39,12 @@ func NewStore(storePath string, fileService portainer.FileService, connection po // Open opens and initializes the BoltDB database. func (store *Store) Open() (newStore bool, err error) { newStore = true + + encryptionReq := store.connection.NeedsEncryptionMigration() + if encryptionReq { + store.encryptDB() + } + err = store.connection.Open() if err != nil { return newStore, err @@ -45,8 +56,18 @@ func (store *Store) Open() (newStore bool, err error) { } // if we have DBVersion in the database then ensure we flag this as NOT a new store - if _, err := store.VersionService.DBVersion(); err == nil { - newStore = false + version, err := store.VersionService.DBVersion() + if err != nil { + if store.IsErrObjectNotFound(err) { + return newStore, nil + } + + return newStore, err + } + + if version > 0 { + logrus.WithField("version", version).Infof("Opened existing store") + return false, nil } return newStore, nil @@ -65,16 +86,81 @@ func (store *Store) BackupTo(w io.Writer) error { // CheckCurrentEdition checks if current edition is community edition func (store *Store) CheckCurrentEdition() error { if store.edition() != portainer.PortainerCE { - return errors.ErrWrongDBEdition + return portainerErrors.ErrWrongDBEdition } return nil } // TODO: move the use of this to dataservices.IsErrObjectNotFound()? func (store *Store) IsErrObjectNotFound(e error) bool { - return e == errors.ErrObjectNotFound + return e == portainerErrors.ErrObjectNotFound } func (store *Store) Rollback(force bool) error { return store.connectionRollback(force) } + +func (store *Store) encryptDB() error { + store.connection.SetEncrypted(false) + err := store.connection.Open() + if err != nil { + return err + } + + err = store.initServices() + if err != nil { + return err + } + + // The DB is not currently encrypted. First save the encrypted db filename + oldFilename := store.connection.GetDatabaseFilePath() + logrus.Infof("Encrypting database") + + // export file path for backup + exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix())) + + logrus.Infof("Exporting database backup to %s", exportFilename) + err = store.Export(exportFilename) + if err != nil { + logrus.WithError(err).Debugf("Failed to export to %s", exportFilename) + return err + } + + logrus.Infof("Database backup exported") + + // Close existing un-encrypted db so that we can delete the file later + store.connection.Close() + + // Tell the db layer to create an encrypted db when opened + store.connection.SetEncrypted(true) + store.connection.Open() + + // We have to init services before import + err = store.initServices() + if err != nil { + return err + } + + err = store.Import(exportFilename) + if err != nil { + // Remove the new encrypted file that we failed to import + os.Remove(store.connection.GetDatabaseFilePath()) + logrus.Fatal(portainerErrors.ErrDBImportFailed.Error()) + } + + err = os.Remove(oldFilename) + if err != nil { + logrus.Errorf("Failed to remove the un-encrypted db file") + } + + err = os.Remove(exportFilename) + if err != nil { + logrus.Errorf("Failed to remove the json backup file") + } + + // Close db connection + store.connection.Close() + + logrus.Info("Database successfully encrypted") + return nil +} diff --git a/api/datastore/services.go b/api/datastore/services.go index b88eb9596..0d23fa612 100644 --- a/api/datastore/services.go +++ b/api/datastore/services.go @@ -363,118 +363,184 @@ func (store *Store) Export(filename string) (err error) { backup := storeExport{} if c, err := store.CustomTemplate().CustomTemplates(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Custom Templates") + } } else { backup.CustomTemplate = c } + if e, err := store.EdgeGroup().EdgeGroups(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Edge Groups") + } } else { backup.EdgeGroup = e } + if e, err := store.EdgeJob().EdgeJobs(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Edge Jobs") + } } else { backup.EdgeJob = e } + if e, err := store.EdgeStack().EdgeStacks(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Edge Stacks") + } } else { backup.EdgeStack = e } + if e, err := store.Endpoint().Endpoints(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Endpoints") + } } else { backup.Endpoint = e } + if e, err := store.EndpointGroup().EndpointGroups(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Endpoint Groups") + } } else { backup.EndpointGroup = e } + if r, err := store.EndpointRelation().EndpointRelations(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Endpoint Relations") + } } else { backup.EndpointRelation = r } + if r, err := store.ExtensionService.Extensions(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Extensions") + } } else { backup.Extensions = r } - if r, err := store.HelmUserRepository().HelmUserRepositorys(); err != nil { - logrus.WithError(err).Debugf("Export boom") + + if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil { + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Helm User Repositories") + } } else { backup.HelmUserRepository = r } + if r, err := store.Registry().Registries(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Registries") + } } else { backup.Registry = r } + if c, err := store.ResourceControl().ResourceControls(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Resource Controls") + } } else { backup.ResourceControl = c } + if role, err := store.Role().Roles(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Roles") + } } else { backup.Role = role } + if r, err := store.ScheduleService.Schedules(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Schedules") + } } else { backup.Schedules = r } + if settings, err := store.Settings().Settings(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Settings") + } } else { backup.Settings = *settings } + if settings, err := store.SSLSettings().Settings(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting SSL Settings") + } } else { backup.SSLSettings = *settings } + if t, err := store.Stack().Stacks(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Stacks") + } } else { backup.Stack = t } + if t, err := store.Tag().Tags(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Tags") + } } else { backup.Tag = t } + if t, err := store.TeamMembership().TeamMemberships(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Team Memberships") + } } else { backup.TeamMembership = t } + if t, err := store.Team().Teams(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Teams") + } } else { backup.Team = t } + if info, err := store.TunnelServer().Info(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Tunnel Server") + } } else { backup.TunnelServer = *info } + if users, err := store.User().Users(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Users") + } } else { backup.User = users } + if webhooks, err := store.Webhook().Webhooks(); err != nil { - logrus.WithError(err).Debugf("Export boom") + if !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting Webhooks") + } } else { backup.Webhook = webhooks } + v, err := store.Version().DBVersion() - if err != nil { - logrus.WithError(err).Debugf("Export boom") + if err != nil && !store.IsErrObjectNotFound(err) { + logrus.WithError(err).Errorf("Exporting DB version") } instance, _ := store.Version().InstanceID() backup.Version = map[string]string{ @@ -518,50 +584,66 @@ func (store *Store) Import(filename string) (err error) { for _, v := range backup.CustomTemplate { store.CustomTemplate().UpdateCustomTemplate(v.ID, &v) } + for _, v := range backup.EdgeGroup { store.EdgeGroup().UpdateEdgeGroup(v.ID, &v) } + for _, v := range backup.EdgeJob { store.EdgeJob().UpdateEdgeJob(v.ID, &v) } + for _, v := range backup.EdgeStack { store.EdgeStack().UpdateEdgeStack(v.ID, &v) } + for _, v := range backup.Endpoint { store.Endpoint().UpdateEndpoint(v.ID, &v) } + for _, v := range backup.EndpointGroup { store.EndpointGroup().UpdateEndpointGroup(v.ID, &v) } + for _, v := range backup.EndpointRelation { store.EndpointRelation().UpdateEndpointRelation(v.EndpointID, &v) } + for _, v := range backup.HelmUserRepository { store.HelmUserRepository().UpdateHelmUserRepository(v.ID, &v) } + for _, v := range backup.Registry { store.Registry().UpdateRegistry(v.ID, &v) } + for _, v := range backup.ResourceControl { store.ResourceControl().UpdateResourceControl(v.ID, &v) } + for _, v := range backup.Role { store.Role().UpdateRole(v.ID, &v) } + store.Settings().UpdateSettings(&backup.Settings) store.SSLSettings().UpdateSettings(&backup.SSLSettings) + for _, v := range backup.Stack { store.Stack().UpdateStack(v.ID, &v) } + for _, v := range backup.Tag { store.Tag().UpdateTag(v.ID, &v) } + for _, v := range backup.TeamMembership { store.TeamMembership().UpdateTeamMembership(v.ID, &v) } + for _, v := range backup.Team { store.Team().UpdateTeam(v.ID, &v) } + store.TunnelServer().UpdateInfo(&backup.TunnelServer) for _, user := range backup.User { @@ -570,10 +652,9 @@ func (store *Store) Import(filename string) (err error) { } } - // backup[store.Webhook().BucketName()], err = store.Webhook().Webhooks() - // if err != nil { - // logrus.WithError(err).Debugf("Export boom") - // } + for _, v := range backup.Webhook { + store.Webhook().UpdateWebhook(v.ID, &v) + } return nil } diff --git a/api/datastore/teststore.go b/api/datastore/teststore.go index b97ea465b..7402365d4 100644 --- a/api/datastore/teststore.go +++ b/api/datastore/teststore.go @@ -42,7 +42,7 @@ func NewTestStore(init bool) (bool, *Store, func(), error) { return false, nil, nil, err } - connection, err := database.NewDatabase("boltdb", storePath) + connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes")) if err != nil { panic(err) } diff --git a/api/portainer.go b/api/portainer.go index 93e7a45f3..0a6a63ae4 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -97,6 +97,7 @@ type ( Rollback *bool SnapshotInterval *string BaseURL *string + SecretKeyName *string } // CustomTemplate represents a custom template