Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-25 08:19:40 +02:00
Code Issues Releases Activity

feat(database): add encryption support EE-1983 (#6316)

Browse source 
* bootstrap encryption key

* secret key message change in cli and secret key file content trimmed

* Migrate encryption code to latest version

* pull in newer code

* tidying up

* working data encryption layer

* fix tests

* remove stray comment

* fix a few minor issues and improve the comments

* split out databasefilename with param to two methods to be more obvious

* DB encryption integration (#6374)

* json methods moved under DBConnection

* store encryption fixed

* cleaned

* review comments addressed

* newstore value fixed

* backup test updated

* logrus format config updated

* Fix for newStore

Co-authored-by: Matt Hook <hookenz@gmail.com>

* Minor improvements

* Improve the export code.  Add missing webhook for import

* rename HelmUserRepositorys to HelmUserRepositories

* fix logging messages

* when starting portainer with a key (first use) http is disabled by default.  But when starting fresh without a key, http is enabled?

* Fix bug for default settings on new installs

Co-authored-by: Prabhat Khera <prabhat.khera@portainer.io>
Co-authored-by: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com>
This commit is contained in:
Matt Hook 2022-01-17 16:40:02 +13:00 • committed by GitHub
parent 59ec22f706
commit 34cc8ea96a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 548 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

93
api/database/boltdb/db.go
View file

@ -11,39 +11,78 @@ import (
"github.com/boltdb/bolt"
"github.com/portainer/portainer/api/dataservices/errors"
"github.com/sirupsen/logrus"
)
const (
DatabaseFileName = "portainer.db"
DatabaseFileName = "portainer.db"
EncryptedDatabaseFileName = "portainer.edb"
)
type DbConnection struct {
Path string
Path string
EncryptionKey []byte
isEncrypted bool
*bolt.DB
}
func (connection *DbConnection) GetDatabaseFilename() string {
// GetDatabaseFileName get the database filename
func (connection *DbConnection) GetDatabaseFileName() string {
if connection.IsEncryptedStore() {
return EncryptedDatabaseFileName
}
return DatabaseFileName
}
// GetDataseFilePath get the path + filename for the database file
func (connection *DbConnection) GetDatabaseFilePath() string {
if connection.IsEncryptedStore() {
return path.Join(connection.Path, EncryptedDatabaseFileName)
}
return path.Join(connection.Path, DatabaseFileName)
}
// GetStorePath get the filename and path for the database file
func (connection *DbConnection) GetStorePath() string {
return connection.Path
}
func (connection *DbConnection) SetEncrypted(flag bool) {
connection.isEncrypted = flag
}
// Return true if the database is encrypted
func (connection *DbConnection) IsEncryptedStore() bool {
return connection.getEncryptionKey() != nil
}
// NeedsEncryptionMigration returns true if database encryption is enabled and
// we have an un-encrypted DB that requires migration to an encrypted DB
func (connection *DbConnection) NeedsEncryptionMigration() bool {
if connection.EncryptionKey != nil {
dbFile := path.Join(connection.Path, DatabaseFileName)
if _, err := os.Stat(dbFile); err == nil {
return true
}
// This is an existing encrypted store or a new store.
// A new store will open encrypted from the outset
connection.SetEncrypted(true)
}
return false
}
// Open opens and initializes the BoltDB database.
func (connection *DbConnection) Open() error {
// Disabled for now. Can't use feature flags due to the way that works
// databaseExportPath := path.Join(connection.Path, fmt.Sprintf("raw-%s-%d.json", DatabaseFileName, time.Now().Unix()))
// if err := connection.ExportRaw(databaseExportPath); err != nil {
// log.Printf("raw export to %s error: %s", databaseExportPath, err)
// } else {
// log.Printf("raw export to %s success", databaseExportPath)
// }
databasePath := path.Join(connection.Path, DatabaseFileName)
logrus.Infof("Loading PortainerDB: %s", connection.GetDatabaseFileName())
// Now we open the db
databasePath := connection.GetDatabaseFilePath()
db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
return err
@ -71,12 +110,12 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
}
func (connection *DbConnection) ExportRaw(filename string) error {
databasePath := path.Join(connection.Path, DatabaseFileName)
databasePath := connection.GetDatabaseFilePath()
if _, err := os.Stat(databasePath); err != nil {
return fmt.Errorf("stat on %s failed: %s", databasePath, err)
}
b, err := exportJson(databasePath)
b, err := connection.exportJson(databasePath)
if err != nil {
return err
}
@ -124,7 +163,15 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
return err
}
return UnmarshalObject(data, object)
return connection.UnmarshalObject(data, object)
}
func (connection *DbConnection) getEncryptionKey() []byte {
if !connection.isEncrypted {
return nil
}
return connection.EncryptionKey
}
// UpdateObject is a generic function used to update an object inside a database database.
@ -132,7 +179,7 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje
return connection.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
data, err := MarshalObject(object)
data, err := connection.MarshalObject(object)
if err != nil {
return err
}
@ -163,7 +210,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var obj interface{}
err := UnmarshalObject(v, &obj)
err := connection.UnmarshalObject(v, &obj)
if err != nil {
return err
}
@ -205,7 +252,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
seqId, _ := bucket.NextSequence()
id, obj := fn(seqId)
data, err := MarshalObject(obj)
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
@ -218,8 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
return connection.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
data, err := MarshalObject(obj)
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
@ -240,7 +286,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
return err
}
data, err := MarshalObject(obj)
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
@ -252,10 +298,9 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
err := connection.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
err := UnmarshalObject(v, obj)
err := connection.UnmarshalObject(v, obj)
if err != nil {
return err
}
@ -277,7 +322,7 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
err := UnmarshalObjectWithJsoniter(v, obj)
err := connection.UnmarshalObjectWithJsoniter(v, obj)
if err != nil {
return err
}

5
api/database/boltdb/export.go
View file

@ -10,8 +10,9 @@ import (
// inspired by github.com/konoui/boltdb-exporter (which has no license)
// but very much simplified, based on how we use boltdb
func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
logrus.WithField("databasePath", databasePath).Infof("exportJson")
func exportJson(databasePath string) ([]byte, error) {
connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
if err != nil {
return []byte("{}"), err
@ -31,7 +32,7 @@ func exportJson(databasePath string) ([]byte, error) {
continue
}
var obj interface{}
err := UnmarshalObject(v, &obj)
err := c.UnmarshalObject(v, &obj)
if err != nil {
logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v))
obj = v

113
api/database/boltdb/json.go
View file

@ -1,42 +1,123 @@
package boltdb
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/json"
"fmt"
"io"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
)
var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
// MarshalObject encodes an object to binary format
func MarshalObject(object interface{}) ([]byte, error) {
func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
// Special case for the VERSION bucket. Here we're not using json
if v, ok := object.(string); ok {
return []byte(v), nil
data = []byte(v)
} else {
data, err = json.Marshal(object)
if err != nil {
return data, err
}
}
return json.Marshal(object)
if connection.getEncryptionKey() == nil {
return data, nil
}
return encrypt(data, connection.getEncryptionKey())
}
// UnmarshalObject decodes an object from binary data
func UnmarshalObject(data []byte, object interface{}) error {
// Special case for the VERSION bucket. Here we're not using json
// So we need to return it as a string
err := json.Unmarshal(data, object)
if err != nil {
if s, ok := object.(*string); ok {
*s = string(data)
return nil
func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
var err error
if connection.getEncryptionKey() != nil {
data, err = decrypt(data, connection.getEncryptionKey())
if err != nil {
errors.Wrapf(err, "Failed decrypting object")
}
}
e := json.Unmarshal(data, object)
if e != nil {
// Special case for the VERSION bucket. Here we're not using json
// So we need to return it as a string
s, ok := object.(*string)
if !ok {
return errors.Wrap(err, e.Error())
}
return err
*s = string(data)
}
return nil
return err
}
// UnmarshalObjectWithJsoniter decodes an object from binary data
// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
// decoding at the moment.
func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
if connection.getEncryptionKey() != nil {
var err error
data, err = decrypt(data, connection.getEncryptionKey())
if err != nil {
return err
}
}
var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
return jsoni.Unmarshal(data, &object)
}
// mmm, don't have a KMS .... aes GCM seems the most likely from
// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
block, _ := aes.NewCipher(passphrase)
gcm, err := cipher.NewGCM(block)
if err != nil {
return encrypted, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return encrypted, err
}
ciphertextByte := gcm.Seal(
nonce,
nonce,
plaintext,
nil)
return ciphertextByte, nil
}
func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
if string(encrypted) == "false" {
return []byte("false"), nil
}
block, err := aes.NewCipher(passphrase)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating cypher block")
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating GCM")
}
nonceSize := gcm.NonceSize()
if len(encrypted) < nonceSize {
return encrypted, errEncryptedStringTooShort
}
nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
plaintextByte, err = gcm.Open(
nil,
nonce,
ciphertextByteClean,
nil)
if err != nil {
return encrypted, errors.Wrap(err, "Error decrypting text")
}
return plaintextByte, err
}

67
api/database/boltdb/json_test.go
View file

@ -1,6 +1,7 @@
package boltdb
import (
"crypto/sha256"
"fmt"
"testing"
@ -8,9 +9,17 @@ import (
"github.com/stretchr/testify/assert"
)
const jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
const (
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
passphrase = "my secret key"
)
func Test_MarshalObject(t *testing.T) {
func secretToEncryptionKey(passphrase string) []byte {
hash := sha256.Sum256([]byte(passphrase))
return hash[:]
}
func Test_MarshalObjectUnencrypted(t *testing.T) {
is := assert.New(t)
uuid := uuid.Must(uuid.NewV4())
@ -73,16 +82,18 @@ func Test_MarshalObject(t *testing.T) {
},
}
conn := DbConnection{}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
data, err := MarshalObject(test.object)
data, err := conn.MarshalObject(test.object)
is.NoError(err)
is.Equal(test.expected, string(data))
})
}
}
func Test_UnMarshalObject(t *testing.T) {
func Test_UnMarshalObjectUnencrypted(t *testing.T) {
is := assert.New(t)
// Based on actual data entering and what we expect out of the function
@ -105,18 +116,62 @@ func Test_UnMarshalObject(t *testing.T) {
expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
},
{
// An unmarshalled json object string should return the same as a string without error also
// An un-marshalled json object string should return the same as a string without error also
object: []byte(jsonobject),
expected: jsonobject,
},
}
conn := DbConnection{}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
var object string
err := UnmarshalObject(test.object, &object)
err := conn.UnmarshalObject(test.object, &object)
is.NoError(err)
is.Equal(test.expected, string(object))
})
}
}
func Test_ObjectMarshallingEncrypted(t *testing.T) {
is := assert.New(t)
// Based on actual data entering and what we expect out of the function
tests := []struct {
object []byte
expected string
}{
{
object: []byte(""),
},
{
object: []byte("35"),
},
{
// An unmarshalled byte string should return the same without error
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
},
{
// An un-marshalled json object string should return the same as a string without error also
object: []byte(jsonobject),
},
}
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
data, err := conn.MarshalObject(test.object)
is.NoError(err)
var object []byte
err = conn.UnmarshalObject(data, &object)
is.NoError(err)
is.Equal(test.object, object)
})
}
}

10
api/database/database.go
View file

@ -2,15 +2,19 @@ package database
import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
)
// NewDatabase should use config options to return a connection to the requested database
func NewDatabase(storeType, storePath string) (connection portainer.Connection, err error) {
func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
switch storeType {
case "boltdb":
return &boltdb.DbConnection{Path: storePath}, nil
return &boltdb.DbConnection{
Path: storePath,
EncryptionKey: encryptionKey,
}, nil
}
return nil, fmt.Errorf("Unknown storage database: %s", storeType)
return nil, fmt.Errorf("unknown storage database: %s", storeType)
}