From 50b2f789a385fdd2b129c43739aeab8a3275175f Mon Sep 17 00:00:00 2001 From: andres-portainer <91705312+andres-portainer@users.noreply.github.com> Date: Mon, 17 Jan 2022 19:25:29 -0300 Subject: [PATCH] feat(performance): add settings to tune the performance of the database EE-2363 (#6389) * feat(performance): add settings to tune the performance of the database EE-2363 * Change panics to log.Fatals. Co-authored-by: andres-portainer --- api/cli/cli.go | 3 ++ api/cmd/portainer/main.go | 12 ++++- api/database/boltdb/db.go | 52 ++++++++++--------- api/database/boltdb/json.go | 12 ++++- .../endpoints/endpoint_status_inspect.go | 21 ++++---- api/portainer.go | 3 ++ 6 files changed, 65 insertions(+), 38 deletions(-) diff --git a/api/cli/cli.go b/api/cli/cli.go index 973bd0099..6be5fe0ec 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -57,6 +57,9 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { Logo: kingpin.Flag("logo", "URL for the logo displayed in the UI").String(), Templates: kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(), BaseURL: kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(), + InitialMmapSize: kingpin.Flag("initial-mmap-size", "Initial mmap size of the database in bytes").Int(), + MaxBatchSize: kingpin.Flag("max-batch-size", "Maximum size of a batch").Int(), + MaxBatchDelay: kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(), SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/.").Default(defaultSecretKeyName).String(), } diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 3291ebfe4..76afcf4f3 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -20,6 +20,7 @@ import ( "github.com/portainer/portainer/api/cli" "github.com/portainer/portainer/api/crypto" "github.com/portainer/portainer/api/database" + "github.com/portainer/portainer/api/database/boltdb" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/datastore" "github.com/portainer/portainer/api/docker" @@ -69,8 +70,17 @@ func initFileService(dataStorePath string) portainer.FileService { func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore { connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey) if err != nil { - panic(err.Error()) + log.Fatalf("failed creating database connection: %s", err) } + + if bconn, ok := connection.(*boltdb.DbConnection); ok { + bconn.MaxBatchSize = *flags.MaxBatchSize + bconn.MaxBatchDelay = *flags.MaxBatchDelay + bconn.InitialMmapSize = *flags.InitialMmapSize + } else { + log.Fatalf("failed creating database connection: expecting a boltdb database type but a different one was received") + } + store := datastore.NewStore(*flags.Data, fileService, connection) isNew, err := store.Open() if err != nil { diff --git a/api/database/boltdb/db.go b/api/database/boltdb/db.go index 1ccf708fc..d177bb4eb 100644 --- a/api/database/boltdb/db.go +++ b/api/database/boltdb/db.go @@ -20,9 +20,12 @@ const ( ) type DbConnection struct { - Path string - EncryptionKey []byte - isEncrypted bool + Path string + MaxBatchSize int + MaxBatchDelay time.Duration + InitialMmapSize int + EncryptionKey []byte + isEncrypted bool *bolt.DB } @@ -83,10 +86,15 @@ func (connection *DbConnection) Open() error { // Now we open the db databasePath := connection.GetDatabaseFilePath() - db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second}) + db, err := bolt.Open(databasePath, 0600, &bolt.Options{ + Timeout: 1 * time.Second, + InitialMmapSize: connection.InitialMmapSize, + }) if err != nil { return err } + db.MaxBatchSize = connection.MaxBatchSize + db.MaxBatchDelay = connection.MaxBatchDelay connection.DB = db return nil } @@ -133,7 +141,7 @@ func (connection *DbConnection) ConvertToKey(v int) []byte { // CreateBucket is a generic function used to create a bucket inside a database database. func (connection *DbConnection) SetServiceName(bucketName string) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(bucketName)) if err != nil { return err @@ -163,7 +171,7 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object return err } - return connection.UnmarshalObject(data, object) + return connection.UnmarshalObjectWithJsoniter(data, object) } func (connection *DbConnection) getEncryptionKey() []byte { @@ -176,26 +184,20 @@ func (connection *DbConnection) getEncryptionKey() []byte { // UpdateObject is a generic function used to update an object inside a database database. func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error { - return connection.Update(func(tx *bolt.Tx) error { + data, err := connection.MarshalObject(object) + if err != nil { + return err + } + + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) - - data, err := connection.MarshalObject(object) - if err != nil { - return err - } - - err = bucket.Put(key, data) - if err != nil { - return err - } - - return nil + return bucket.Put(key, data) }) } // DeleteObject is a generic function used to delete an object inside a database database. func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) return bucket.Delete(key) }) @@ -204,7 +206,7 @@ func (connection *DbConnection) DeleteObject(bucketName string, key []byte) erro // DeleteAllObjects delete all objects where matching() returns (id, ok). // TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"? func (connection *DbConnection) DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) cursor := bucket.Cursor() @@ -231,7 +233,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun func (connection *DbConnection) GetNextIdentifier(bucketName string) int { var identifier int - connection.Update(func(tx *bolt.Tx) error { + connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) id, err := bucket.NextSequence() if err != nil { @@ -246,7 +248,7 @@ func (connection *DbConnection) GetNextIdentifier(bucketName string) int { // CreateObject creates a new object in the bucket, using the next bucket sequence id func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) seqId, _ := bucket.NextSequence() @@ -263,7 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) // CreateObjectWithId creates a new object in the bucket, using the specified id func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) data, err := connection.MarshalObject(obj) if err != nil { @@ -277,7 +279,7 @@ func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, ob // CreateObjectWithSetSequence creates a new object in the bucket, using the specified id, and sets the bucket sequence // avoid this :) func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error { - return connection.Update(func(tx *bolt.Tx) error { + return connection.Batch(func(tx *bolt.Tx) error { bucket := tx.Bucket([]byte(bucketName)) // We manually manage sequences for schedules diff --git a/api/database/boltdb/json.go b/api/database/boltdb/json.go index b4b754e68..35694b95c 100644 --- a/api/database/boltdb/json.go +++ b/api/database/boltdb/json.go @@ -66,7 +66,17 @@ func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object } } var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary - return jsoni.Unmarshal(data, &object) + err := jsoni.Unmarshal(data, &object) + if err != nil { + if s, ok := object.(*string); ok { + *s = string(data) + return nil + } + + return err + } + + return nil } // mmm, don't have a KMS .... aes GCM seems the most likely from diff --git a/api/http/handler/endpoints/endpoint_status_inspect.go b/api/http/handler/endpoints/endpoint_status_inspect.go index b66a6f311..afefd9f25 100644 --- a/api/http/handler/endpoints/endpoint_status_inspect.go +++ b/api/http/handler/endpoints/endpoint_status_inspect.go @@ -103,6 +103,15 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req } } + if endpoint.EdgeCheckinInterval == 0 { + settings, err := handler.DataStore.Settings().Settings() + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err} + } + + endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval + } + endpoint.LastCheckInDate = time.Now().Unix() err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint) @@ -110,18 +119,8 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req return &httperror.HandlerError{http.StatusInternalServerError, "Unable to Unable to persist environment changes inside the database", err} } - settings, err := handler.DataStore.Settings().Settings() - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err} - } - tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID) - checkinInterval := settings.EdgeAgentCheckinInterval - if endpoint.EdgeCheckinInterval != 0 { - checkinInterval = endpoint.EdgeCheckinInterval - } - schedules := []edgeJobResponse{} for _, job := range tunnel.Jobs { schedule := edgeJobResponse{ @@ -146,7 +145,7 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req Status: tunnel.Status, Port: tunnel.Port, Schedules: schedules, - CheckinInterval: checkinInterval, + CheckinInterval: endpoint.EdgeCheckinInterval, Credentials: tunnel.Credentials, } diff --git a/api/portainer.go b/api/portainer.go index 0a6a63ae4..aded693f5 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -97,6 +97,9 @@ type ( Rollback *bool SnapshotInterval *string BaseURL *string + InitialMmapSize *int + MaxBatchSize *int + MaxBatchDelay *time.Duration SecretKeyName *string }