Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-05 05:45:22 +02:00
Code Issues Releases Activity

feat(logging): replace all the loggers with zerolog EE-4186 (#7663)

Browse source
This commit is contained in:
andres-portainer 2022-09-16 13:18:44 -03:00 • committed by GitHub
parent 53025178ef
commit 36e7981ab7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
109 changed files with 1101 additions and 662 deletions
Download patch file Download diff file Expand all files Collapse all files

34
api/datastore/backup.go
View file

@ -6,7 +6,7 @@ import (
"path"
"time"
plog "github.com/portainer/portainer/api/datastore/log"
"github.com/rs/zerolog/log"
)
var backupDefaults = struct {
@ -17,8 +17,6 @@ var backupDefaults = struct {
"common",
}
var backupLog = plog.NewScopedLog("database, backup")
//
// Backup Helpers
//
@ -29,7 +27,7 @@ func (store *Store) createBackupFolders() {
commonDir := store.commonBackupDir()
if exists, _ := store.fileService.FileExists(commonDir); !exists {
if err := os.MkdirAll(commonDir, 0700); err != nil {
backupLog.Error("Error while creating common backup folder", err)
log.Error().Err(err).Msg("error while creating common backup folder")
}
}
}
@ -43,11 +41,13 @@ func (store *Store) commonBackupDir() string {
}
func (store *Store) copyDBFile(from string, to string) error {
backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
err := store.fileService.Copy(from, to, true)
if err != nil {
backupLog.Error("Failed", err)
log.Error().Err(err).Msg("failed")
}
return err
}
@ -99,7 +99,8 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
// BackupWithOptions backup current database with options
func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
backupLog.Info("creating db backup")
log.Info().Msg("creating DB backup")
store.createBackupFolders()
options = store.setupOptions(options)
@ -122,6 +123,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
err,
)
}
return options.BackupPath, nil
}
@ -135,17 +137,19 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
// Check if backup file exist before restoring
_, err := os.Stat(options.BackupPath)
if os.IsNotExist(err) {
backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
return err
}
err = store.Close()
if err != nil {
backupLog.Error("Error while closing store before restore", err)
log.Error().Err(err).Msg("error while closing store before restore")
return err
}
backupLog.Info("Restoring db backup")
log.Info().Msg("restoring DB backup")
err = store.copyDBFile(options.BackupPath, store.databasePath())
if err != nil {
return err
@ -157,20 +161,22 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
// RemoveWithOptions removes backup database based on supplied options
func (store *Store) removeWithOptions(options *BackupOptions) error {
backupLog.Info("Removing db backup")
log.Info().Msg("removing DB backup")
options = store.setupOptions(options)
_, err := os.Stat(options.BackupPath)
if os.IsNotExist(err) {
backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
return err
}
backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
log.Info().Str("path", options.BackupPath).Msg("removing DB file")
err = os.Remove(options.BackupPath)
if err != nil {
backupLog.Error("Failed", err)
log.Error().Err(err).Msg("failed")
return err
}

25
api/datastore/datastore.go
View file

@ -9,7 +9,8 @@ import (
portainer "github.com/portainer/portainer/api"
portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
)
func (store *Store) version() (int, error) {
@ -73,7 +74,8 @@ func (store *Store) Open() (newStore bool, err error) {
}
if version > 0 {
logrus.WithField("version", version).Infof("Opened existing store")
log.Debug().Int("version", version).Msg("opened existing store")
return false, nil
}
@ -121,19 +123,21 @@ func (store *Store) encryptDB() error {
// The DB is not currently encrypted. First save the encrypted db filename
oldFilename := store.connection.GetDatabaseFilePath()
logrus.Infof("Encrypting database")
log.Info().Msg("encrypting database")
// export file path for backup
exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
logrus.Infof("Exporting database backup to %s", exportFilename)
log.Info().Str("filename", exportFilename).Msg("exporting database backup")
err = store.Export(exportFilename)
if err != nil {
logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
return err
}
logrus.Infof("Database backup exported")
log.Info().Msg("database backup exported")
// Close existing un-encrypted db so that we can delete the file later
store.connection.Close()
@ -152,22 +156,23 @@ func (store *Store) encryptDB() error {
if err != nil {
// Remove the new encrypted file that we failed to import
os.Remove(store.connection.GetDatabaseFilePath())
logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
log.Fatal().Err(portainerErrors.ErrDBImportFailed).Msg("")
}
err = os.Remove(oldFilename)
if err != nil {
logrus.Errorf("Failed to remove the un-encrypted db file")
log.Error().Msg("failed to remove the un-encrypted db file")
}
err = os.Remove(exportFilename)
if err != nil {
logrus.Errorf("Failed to remove the json backup file")
log.Error().Msg("failed to remove the json backup file")
}
// Close db connection
store.connection.Close()
logrus.Info("Database successfully encrypted")
log.Info().Msg("database successfully encrypted")
return nil
}

51
api/datastore/log/log.go
View file

@ -1,51 +0,0 @@
package log
import (
"fmt"
"log"
)
const (
INFO = "INFO"
ERROR = "ERROR"
DEBUG = "DEBUG"
FATAL = "FATAL"
)
type ScopedLog struct {
scope string
}
func NewScopedLog(scope string) *ScopedLog {
return &ScopedLog{scope: scope}
}
func (slog *ScopedLog) print(kind string, message string) {
log.Printf("[%s] [%s] %s", kind, slog.scope, message)
}
func (slog *ScopedLog) Debug(message string) {
slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Debugf(message string, vars ...interface{}) {
message = fmt.Sprintf(message, vars...)
slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Info(message string) {
slog.print(INFO, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Infof(message string, vars ...interface{}) {
message = fmt.Sprintf(message, vars...)
slog.print(INFO, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Error(message string, err error) {
slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
}
func (slog *ScopedLog) NotImplemented(method string) {
log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
}

39
api/datastore/migrate_data.go
View file

@ -4,21 +4,18 @@ import (
"fmt"
"runtime/debug"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/cli"
"github.com/portainer/portainer/api/dataservices/errors"
plog "github.com/portainer/portainer/api/datastore/log"
"github.com/portainer/portainer/api/datastore/migrator"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/sirupsen/logrus"
werrors "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
var migrateLog = plog.NewScopedLog("database, migrate")
func (store *Store) MigrateData() error {
version, err := store.version()
if err != nil {
@ -56,20 +53,19 @@ func (store *Store) MigrateData() error {
// restore on error
err = store.connectionMigrateData(migratorParams)
if err != nil {
logrus.Errorf("While DB migration %v. Restoring DB", err)
log.Error().Err(err).Msg("while DB migration, restoring DB")
// Restore options
options := BackupOptions{
BackupPath: backupPath,
}
err := store.restoreWithOptions(&options)
if err != nil {
logrus.Fatalf(
"Failed restoring the backup. portainer database file needs to restored manually by "+
"replacing %s database file with recent backup %s. Error %v",
store.databasePath(),
options.BackupPath,
err,
)
log.Fatal().
Str("database_file", store.databasePath()).
Str("backup", options.BackupPath).Err(err).
Msg("failed restoring the backup, Portainer database file needs to restored manually by replacing the database file with a recent backup")
}
}
@ -111,10 +107,15 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
}
if migrator.Version() < portainer.DBVersion {
migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
log.Info().
Int("migrator_version", migrator.Version()).
Int("db_version", portainer.DBVersion).
Msg("migrating database")
err = store.FailSafeMigrate(migrator)
if err != nil {
migrateLog.Error("An error occurred during database migration", err)
log.Error().Err(err).Msg("an error occurred during database migration")
return err
}
}
@ -124,17 +125,19 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
// backupVersion will backup the database or panic if any errors occur
func (store *Store) backupVersion(migrator *migrator.Migrator) error {
migrateLog.Info("Backing up database prior to version upgrade...")
log.Info().Msg("backing up database prior to version upgrade")
options := getBackupRestoreOptions(store.commonBackupDir())
_, err := store.backupWithOptions(options)
if err != nil {
migrateLog.Error("An error occurred during database backup", err)
log.Error().Err(err).Msg("an error occurred during database backup")
removalErr := store.removeWithOptions(options)
if removalErr != nil {
migrateLog.Error("An error occurred during store removal prior to backup", err)
log.Error().Err(err).Msg("an error occurred during store removal prior to backup")
}
return err
}

15
api/datastore/migrate_data_test.go
View file

@ -5,15 +5,16 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/google/go-cmp/cmp"
"github.com/rs/zerolog/log"
)
// testVersion is a helper which tests current store version against wanted version
@ -157,7 +158,6 @@ func TestMigrateData(t *testing.T) {
t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
}
})
}
func Test_getBackupRestoreOptions(t *testing.T) {
@ -168,12 +168,12 @@ func Test_getBackupRestoreOptions(t *testing.T) {
wantDir := store.commonBackupDir()
if !strings.HasSuffix(options.BackupDir, wantDir) {
log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir")
}
wantFilename := "portainer.db.bak"
if options.BackupFileName != wantFilename {
log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file")
}
}
@ -186,19 +186,20 @@ func TestRollback(t *testing.T) {
_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
if err != nil {
log.Fatal(err)
log.Fatal().Err(err).Msg("")
}
// Change the current edition
err = store.VersionService.StoreDBVersion(version + 10)
if err != nil {
log.Fatal(err)
log.Fatal().Err(err).Msg("")
}
err = store.Rollback(true)
if err != nil {
t.Logf("Rollback failed: %s", err)
t.Fail()
return
}

12
api/datastore/migrator/migrate_ce.go
View file

@ -5,8 +5,10 @@ import (
"reflect"
"runtime"
werrors "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
werrors "github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type migration struct {
@ -114,7 +116,7 @@ func (m *Migrator) Migrate() error {
// Print the next line only when the version changes
if migration.dbversion > lastDbVersion {
migrateLog.Infof("Migrating DB to version %d", migration.dbversion)
log.Info().Int("to_version", migration.dbversion).Msg("migrating DB")
}
err := migration.migrate()
@ -125,12 +127,14 @@ func (m *Migrator) Migrate() error {
lastDbVersion = migration.dbversion
}
migrateLog.Infof("Setting DB version to %d", portainer.DBVersion)
log.Info().Int("version", portainer.DBVersion).Msg("setting DB version")
err = m.versionService.StoreDBVersion(portainer.DBVersion)
if err != nil {
return migrationError(err, "StoreDBVersion")
}
migrateLog.Infof("Updated DB version to %d", portainer.DBVersion)
log.Info().Int("version", portainer.DBVersion).Msg("updated DB version")
// reset DB updating status
return m.versionService.StoreIsUpdating(false)

14
api/datastore/migrator/migrate_dbversion17.go
View file

@ -2,10 +2,13 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateUsersToDBVersion18() error {
migrateLog.Info("- updating users")
log.Info().Msg("updating users")
legacyUsers, err := m.userService.Users()
if err != nil {
return err
@ -40,7 +43,8 @@ func (m *Migrator) updateUsersToDBVersion18() error {
}
func (m *Migrator) updateEndpointsToDBVersion18() error {
migrateLog.Info("- updating endpoints")
log.Info().Msg("updating endpoints")
legacyEndpoints, err := m.endpointService.Endpoints()
if err != nil {
return err
@ -71,7 +75,8 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
}
func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
migrateLog.Info("- updating endpoint groups")
log.Info().Msg("updating endpoint groups")
legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
if err != nil {
return err
@ -102,7 +107,8 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
}
func (m *Migrator) updateRegistriesToDBVersion18() error {
migrateLog.Info("- updating registries")
log.Info().Msg("updating registries")
legacyRegistries, err := m.registryService.Registries()
if err != nil {
return err

9
api/datastore/migrator/migrate_dbversion18.go
View file

@ -1,9 +1,14 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateSettingsToDBVersion19() error {
migrateLog.Info("- updating settings")
log.Info().Msg("updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
return err

8
api/datastore/migrator/migrate_dbversion19.go
View file

@ -2,12 +2,15 @@ package migrator
import (
"strings"
"github.com/rs/zerolog/log"
)
const scheduleScriptExecutionJobType = 1
func (m *Migrator) updateUsersToDBVersion20() error {
migrateLog.Info("- updating user authentication")
log.Info().Msg("updating user authentication")
return m.authorizationService.UpdateUsersAuthorizations()
}
@ -23,7 +26,8 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
}
func (m *Migrator) updateSchedulesToDBVersion20() error {
migrateLog.Info("- updating schedules")
log.Info().Msg("updating schedules")
legacySchedules, err := m.scheduleService.Schedules()
if err != nil {
return err

8
api/datastore/migrator/migrate_dbversion20.go
View file

@ -3,10 +3,13 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateResourceControlsToDBVersion22() error {
migrateLog.Info("- updating resource controls")
log.Info().Msg("updating resource controls")
legacyResourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err
@ -25,7 +28,8 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
}
func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
migrateLog.Info("- updating users and roles")
log.Info().Msg("updating users and roles")
legacyUsers, err := m.userService.Users()
if err != nil {
return err

13
api/datastore/migrator/migrate_dbversion22.go
View file

@ -1,9 +1,14 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateTagsToDBVersion23() error {
migrateLog.Info("- Updating tags")
log.Info().Msg("updating tags")
tags, err := m.tagService.Tags()
if err != nil {
return err
@ -21,7 +26,8 @@ func (m *Migrator) updateTagsToDBVersion23() error {
}
func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
migrateLog.Info("- updating endpoints and endpoint groups")
log.Info().Msg("updating endpoints and endpoint groups")
tags, err := m.tagService.Tags()
if err != nil {
return err
@ -90,5 +96,6 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
return err
}
}
return nil
}

11
api/datastore/migrator/migrate_dbversion23.go
View file

@ -1,9 +1,13 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateSettingsToDB24() error {
migrateLog.Info("- updating Settings")
log.Info().Msg("updating Settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
@ -18,7 +22,8 @@ func (m *Migrator) updateSettingsToDB24() error {
}
func (m *Migrator) updateStacksToDB24() error {
migrateLog.Info("- updating stacks")
log.Info().Msg("updating stacks")
stacks, err := m.stackService.Stacks()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion24.go
View file

@ -2,10 +2,12 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateSettingsToDB25() error {
migrateLog.Info("- updating settings")
log.Info().Msg("updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {

5
api/datastore/migrator/migrate_dbversion25.go
View file

@ -2,10 +2,13 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateEndpointSettingsToDB25() error {
migrateLog.Info("- updating endpoint settings")
log.Info().Msg("updating endpoint settings")
settings, err := m.settingsService.Settings()
if err != nil {
return err

5
api/datastore/migrator/migrate_dbversion26.go
View file

@ -4,10 +4,13 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices/errors"
"github.com/portainer/portainer/api/internal/stackutils"
"github.com/rs/zerolog/log"
)
func (m *Migrator) updateStackResourceControlToDB27() error {
migrateLog.Info("- updating stack resource controls")
log.Info().Msg("updating stack resource controls")
resourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err

11
api/datastore/migrator/migrate_dbversion29.go
View file

@ -1,12 +1,11 @@
package migrator
func (m *Migrator) migrateDBVersionToDB30() error {
migrateLog.Info("- updating legacy settings")
if err := m.MigrateSettingsToDB30(); err != nil {
return err
}
import "github.com/rs/zerolog/log"
return nil
func (m *Migrator) migrateDBVersionToDB30() error {
log.Info().Msg("updating legacy settings")
return m.MigrateSettingsToDB30()
}
// so setting to false and "", is what would happen without this code

36
api/datastore/migrator/migrate_dbversion31.go
View file

@ -2,14 +2,14 @@ package migrator
import (
"fmt"
"log"
"github.com/docker/docker/api/types/volume"
"github.com/portainer/portainer/api/dataservices/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices/errors"
"github.com/portainer/portainer/api/internal/endpointutils"
snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
"github.com/docker/docker/api/types/volume"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB32() error {
@ -39,7 +39,8 @@ func (m *Migrator) migrateDBVersionToDB32() error {
}
func (m *Migrator) updateRegistriesToDB32() error {
migrateLog.Info("- updating registries")
log.Info().Msg("updating registries")
registries, err := m.registryService.Registries()
if err != nil {
return err
@ -82,7 +83,8 @@ func (m *Migrator) updateRegistriesToDB32() error {
}
func (m *Migrator) updateDockerhubToDB32() error {
migrateLog.Info("- updating dockerhub")
log.Info().Msg("updating dockerhub")
dockerhub, err := m.dockerhubService.DockerHub()
if err == errors.ErrObjectNotFound {
return nil
@ -171,7 +173,8 @@ func (m *Migrator) updateDockerhubToDB32() error {
}
func (m *Migrator) updateVolumeResourceControlToDB32() error {
migrateLog.Info("- updating resource controls")
log.Info().Msg("updating resource controls")
endpoints, err := m.endpointService.Endpoints()
if err != nil {
return fmt.Errorf("failed fetching environments: %w", err)
@ -199,7 +202,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
totalSnapshots := len(endpoint.Snapshots)
if totalSnapshots == 0 {
log.Println("[DEBUG] [volume migration] [message: no snapshot found]")
log.Debug().Msg("no snapshot found")
continue
}
@ -207,13 +210,13 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
if err != nil {
log.Printf("[WARN] [database,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
log.Warn().Err(err).Msg("failed fetching environment docker id")
continue
}
volumesData := snapshot.SnapshotRaw.Volumes
if volumesData.Volumes == nil {
log.Println("[DEBUG] [volume migration] [message: no volume data found]")
log.Debug().Msg("no volume data found")
continue
}
@ -224,17 +227,18 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
for _, resourceControl := range volumeResourceControls {
if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
resourceControl.ResourceID = newResourceID
err := m.resourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
if err != nil {
return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
}
} else {
err := m.resourceControlService.DeleteResourceControl(resourceControl.ID)
if err != nil {
return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
}
log.Printf("[DEBUG] [volume migration] [message: legacy resource control(%s) has been deleted]", resourceControl.ResourceID)
log.Debug().Str("resource_id", resourceControl.ResourceID).Msg("legacy resource control has been deleted")
}
}
@ -257,21 +261,25 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData volume.VolumeList
}
func (m *Migrator) kubeconfigExpiryToDB32() error {
migrateLog.Info("- updating kubeconfig expiry")
log.Info().Msg("updating kubeconfig expiry")
settings, err := m.settingsService.Settings()
if err != nil {
return err
}
settings.KubeconfigExpiry = portainer.DefaultKubeconfigExpiry
return m.settingsService.UpdateSettings(settings)
}
func (m *Migrator) helmRepositoryURLToDB32() error {
migrateLog.Info("- setting default helm repository URL")
log.Info().Msg("setting default helm repository URL")
settings, err := m.settingsService.Settings()
if err != nil {
return err
}
settings.HelmRepositoryURL = portainer.DefaultHelmRepositoryURL
return m.settingsService.UpdateSettings(settings)
}

12
api/datastore/migrator/migrate_dbversion32.go
View file

@ -2,15 +2,14 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB33() error {
migrateLog.Info("- updating settings")
if err := m.migrateSettingsToDB33(); err != nil {
return err
}
log.Info().Msg("updating settings")
return nil
return m.migrateSettingsToDB33()
}
func (m *Migrator) migrateSettingsToDB33() error {
@ -19,7 +18,8 @@ func (m *Migrator) migrateSettingsToDB33() error {
return err
}
migrateLog.Info("- setting default kubectl shell image")
log.Info().Msg("setting default kubectl shell image")
settings.KubectlShellImage = portainer.DefaultKubectlShellImage
return m.settingsService.UpdateSettings(settings)
}

13
api/datastore/migrator/migrate_dbversion33.go
View file

@ -2,16 +2,14 @@ package migrator
import (
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB34() error {
migrateLog.Info("- updating stacks")
err := MigrateStackEntryPoint(m.stackService)
if err != nil {
return err
}
log.Info().Msg("updating stacks")
return nil
return MigrateStackEntryPoint(m.stackService)
}
// MigrateStackEntryPoint exported for testing (blah.)
@ -20,15 +18,18 @@ func MigrateStackEntryPoint(stackService dataservices.StackService) error {
if err != nil {
return err
}
for i := range stacks {
stack := &stacks[i]
if stack.GitConfig == nil {
continue
}
stack.GitConfig.ConfigFilePath = stack.EntryPoint
if err := stackService.UpdateStack(stack.ID, stack); err != nil {
return err
}
}
return nil
}

12
api/datastore/migrator/migrate_dbversion34.go
View file

@ -1,12 +1,12 @@
package migrator
import "github.com/rs/zerolog/log"
func (m *Migrator) migrateDBVersionToDB35() error {
// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
// calling it again will now fix the issue as the function has been repaired.
migrateLog.Info("- updating dockerhub registries")
err := m.updateDockerhubToDB32()
if err != nil {
return err
}
return nil
log.Info().Msg("updating dockerhub registries")
return m.updateDockerhubToDB32()
}

9
api/datastore/migrator/migrate_dbversion35.go
View file

@ -3,15 +3,14 @@ package migrator
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB36() error {
migrateLog.Info("Updating user authorizations")
if err := m.migrateUsersToDB36(); err != nil {
return err
}
log.Info().Msg("updating user authorizations")
return nil
return m.migrateUsersToDB36()
}
func (m *Migrator) migrateUsersToDB36() error {

15
api/datastore/migrator/migrate_dbversion40.go
View file

@ -1,17 +1,18 @@
package migrator
import "github.com/portainer/portainer/api/internal/endpointutils"
import (
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB40() error {
if err := m.trustCurrentEdgeEndpointsDB40(); err != nil {
return err
}
return nil
return m.trustCurrentEdgeEndpointsDB40()
}
func (m *Migrator) trustCurrentEdgeEndpointsDB40() error {
migrateLog.Info("- trusting current edge endpoints")
log.Info().Msg("trusting current edge endpoints")
endpoints, err := m.endpointService.Endpoints()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion50.go
View file

@ -2,6 +2,7 @@ package migrator
import (
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB50() error {
@ -9,7 +10,8 @@ func (m *Migrator) migrateDBVersionToDB50() error {
}
func (m *Migrator) migratePasswordLengthSettings() error {
migrateLog.Info("Updating required password length")
log.Info().Msg("updating required password length")
s, err := m.settingsService.Settings()
if err != nil {
return errors.Wrap(err, "unable to retrieve settings")

15
api/datastore/migrator/migrate_dbversion60.go
View file

@ -1,17 +1,18 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
"github.com/rs/zerolog/log"
)
func (m *Migrator) migrateDBVersionToDB60() error {
if err := m.addGpuInputFieldDB60(); err != nil {
return err
}
return nil
return m.addGpuInputFieldDB60()
}
func (m *Migrator) addGpuInputFieldDB60() error {
migrateLog.Info("- add gpu input field")
log.Info().Msg("add gpu input field")
endpoints, err := m.endpointService.Endpoints()
if err != nil {
return err

3
api/datastore/migrator/migrator.go
View file

@ -18,12 +18,9 @@ import (
"github.com/portainer/portainer/api/dataservices/teammembership"
"github.com/portainer/portainer/api/dataservices/user"
"github.com/portainer/portainer/api/dataservices/version"
plog "github.com/portainer/portainer/api/datastore/log"
"github.com/portainer/portainer/api/internal/authorization"
)
var migrateLog = plog.NewScopedLog("database, migrate")
type (
// Migrator defines a service to migrate data after a Portainer version update.
Migrator struct {

59
api/datastore/services.go
View file

@ -2,6 +2,7 @@ package datastore
import (
"encoding/json"
"fmt"
"io/ioutil"
"strconv"
@ -34,7 +35,8 @@ import (
"github.com/portainer/portainer/api/dataservices/user"
"github.com/portainer/portainer/api/dataservices/version"
"github.com/portainer/portainer/api/dataservices/webhook"
"github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
)
// Store defines the implementation of portainer.DataStore using
@ -391,7 +393,7 @@ func (store *Store) Export(filename string) (err error) {
if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Custom Templates")
log.Error().Err(err).Msg("exporting Custom Templates")
}
} else {
backup.CustomTemplate = c
@ -399,7 +401,7 @@ func (store *Store) Export(filename string) (err error) {
if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Edge Groups")
log.Error().Err(err).Msg("exporting Edge Groups")
}
} else {
backup.EdgeGroup = e
@ -407,7 +409,7 @@ func (store *Store) Export(filename string) (err error) {
if e, err := store.EdgeJob().EdgeJobs(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Edge Jobs")
log.Error().Err(err).Msg("exporting Edge Jobs")
}
} else {
backup.EdgeJob = e
@ -415,7 +417,7 @@ func (store *Store) Export(filename string) (err error) {
if e, err := store.EdgeStack().EdgeStacks(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Edge Stacks")
log.Error().Err(err).Msg("exporting Edge Stacks")
}
} else {
backup.EdgeStack = e
@ -423,7 +425,7 @@ func (store *Store) Export(filename string) (err error) {
if e, err := store.Endpoint().Endpoints(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Endpoints")
log.Error().Err(err).Msg("exporting Endpoints")
}
} else {
backup.Endpoint = e
@ -431,7 +433,7 @@ func (store *Store) Export(filename string) (err error) {
if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Endpoint Groups")
log.Error().Err(err).Msg("exporting Endpoint Groups")
}
} else {
backup.EndpointGroup = e
@ -439,7 +441,7 @@ func (store *Store) Export(filename string) (err error) {
if r, err := store.EndpointRelation().EndpointRelations(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Endpoint Relations")
log.Error().Err(err).Msg("exporting Endpoint Relations")
}
} else {
backup.EndpointRelation = r
@ -447,7 +449,7 @@ func (store *Store) Export(filename string) (err error) {
if r, err := store.ExtensionService.Extensions(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Extensions")
log.Error().Err(err).Msg("exporting Extensions")
}
} else {
backup.Extensions = r
@ -455,7 +457,7 @@ func (store *Store) Export(filename string) (err error) {
if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Helm User Repositories")
log.Error().Err(err).Msg("exporting Helm User Repositories")
}
} else {
backup.HelmUserRepository = r
@ -463,7 +465,7 @@ func (store *Store) Export(filename string) (err error) {
if r, err := store.Registry().Registries(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Registries")
log.Error().Err(err).Msg("exporting Registries")
}
} else {
backup.Registry = r
@ -471,7 +473,7 @@ func (store *Store) Export(filename string) (err error) {
if c, err := store.ResourceControl().ResourceControls(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Resource Controls")
log.Error().Err(err).Msg("exporting Resource Controls")
}
} else {
backup.ResourceControl = c
@ -479,7 +481,7 @@ func (store *Store) Export(filename string) (err error) {
if role, err := store.Role().Roles(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Roles")
log.Error().Err(err).Msg("exporting Roles")
}
} else {
backup.Role = role
@ -487,7 +489,7 @@ func (store *Store) Export(filename string) (err error) {
if r, err := store.ScheduleService.Schedules(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Schedules")
log.Error().Err(err).Msg("exporting Schedules")
}
} else {
backup.Schedules = r
@ -495,7 +497,7 @@ func (store *Store) Export(filename string) (err error) {
if settings, err := store.Settings().Settings(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Settings")
log.Error().Err(err).Msg("exporting Settings")
}
} else {
backup.Settings = *settings
@ -503,7 +505,7 @@ func (store *Store) Export(filename string) (err error) {
if settings, err := store.SSLSettings().Settings(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting SSL Settings")
log.Error().Err(err).Msg("exporting SSL Settings")
}
} else {
backup.SSLSettings = *settings
@ -511,7 +513,7 @@ func (store *Store) Export(filename string) (err error) {
if t, err := store.Stack().Stacks(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Stacks")
log.Error().Err(err).Msg("exporting Stacks")
}
} else {
backup.Stack = t
@ -519,7 +521,7 @@ func (store *Store) Export(filename string) (err error) {
if t, err := store.Tag().Tags(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Tags")
log.Error().Err(err).Msg("exporting Tags")
}
} else {
backup.Tag = t
@ -527,7 +529,7 @@ func (store *Store) Export(filename string) (err error) {
if t, err := store.TeamMembership().TeamMemberships(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Team Memberships")
log.Error().Err(err).Msg("exporting Team Memberships")
}
} else {
backup.TeamMembership = t
@ -535,7 +537,7 @@ func (store *Store) Export(filename string) (err error) {
if t, err := store.Team().Teams(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Teams")
log.Error().Err(err).Msg("exporting Teams")
}
} else {
backup.Team = t
@ -543,7 +545,7 @@ func (store *Store) Export(filename string) (err error) {
if info, err := store.TunnelServer().Info(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Tunnel Server")
log.Error().Err(err).Msg("exporting Tunnel Server")
}
} else {
backup.TunnelServer = *info
@ -551,7 +553,7 @@ func (store *Store) Export(filename string) (err error) {
if users, err := store.User().Users(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Users")
log.Error().Err(err).Msg("exporting Users")
}
} else {
backup.User = users
@ -559,7 +561,7 @@ func (store *Store) Export(filename string) (err error) {
if webhooks, err := store.Webhook().Webhooks(); err != nil {
if !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting Webhooks")
log.Error().Err(err).Msg("exporting Webhooks")
}
} else {
backup.Webhook = webhooks
@ -567,7 +569,7 @@ func (store *Store) Export(filename string) (err error) {
v, err := store.Version().DBVersion()
if err != nil && !store.IsErrObjectNotFound(err) {
logrus.WithError(err).Errorf("Exporting DB version")
log.Error().Err(err).Msg("exporting DB version")
}
instance, _ := store.Version().InstanceID()
backup.Version = map[string]string{
@ -577,7 +579,7 @@ func (store *Store) Export(filename string) (err error) {
backup.Metadata, err = store.connection.BackupMetadata()
if err != nil {
logrus.WithError(err).Errorf("Exporting Metadata")
log.Error().Err(err).Msg("exporting Metadata")
}
b, err := json.MarshalIndent(backup, "", " ")
@ -588,7 +590,6 @@ func (store *Store) Export(filename string) (err error) {
}
func (store *Store) Import(filename string) (err error) {
backup := storeExport{}
s, err := ioutil.ReadFile(filename)
@ -604,13 +605,13 @@ func (store *Store) Import(filename string) (err error) {
if dbversion, ok := backup.Version["DB_VERSION"]; ok {
if v, err := strconv.Atoi(dbversion); err == nil {
if err := store.Version().StoreDBVersion(v); err != nil {
logrus.WithError(err).Errorf("DB_VERSION import issue")
log.Error().Err(err).Msg("DB_VERSION import issue")
}
}
}
if instanceID, ok := backup.Version["INSTANCE_ID"]; ok {
if err := store.Version().StoreInstanceID(instanceID); err != nil {
logrus.WithError(err).Errorf("INSTANCE_ID import issue")
log.Error().Err(err).Msg("INSTANCE_ID import issue")
}
}
@ -681,7 +682,7 @@ func (store *Store) Import(filename string) (err error) {
for _, user := range backup.User {
if err := store.User().UpdateUser(user.ID, &user); err != nil {
logrus.WithField("user", user).WithError(err).Errorf("User: Failed to Update Database")
log.Debug().Str("user", fmt.Sprintf("%+v", user)).Err(err).Msg("user: failed to Update Database")
}
}

14
api/datastore/teststore.go
View file

@ -1,14 +1,14 @@
package datastore
import (
"log"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database"
"github.com/portainer/portainer/api/filesystem"
"github.com/pkg/errors"
"github.com/portainer/portainer/api/filesystem"
"github.com/rs/zerolog/log"
)
var errTempDir = errors.New("can't create a temp dir")
@ -23,7 +23,8 @@ func MustNewTestStore(t *testing.T, init, secure bool) (bool, *Store, func()) {
if !errors.Is(err, errTempDir) {
teardown()
}
log.Fatal(err)
log.Fatal().Err(err).Msg("")
}
return newStore, store, teardown
@ -46,12 +47,15 @@ func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error)
if err != nil {
panic(err)
}
store := NewStore(storePath, fileService, connection)
newStore, err := store.Open()
if err != nil {
return newStore, nil, nil, err
}
log.Debug().Msg("opened")
if init {
err = store.Init()
if err != nil {
@ -59,6 +63,8 @@ func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error)
}
}
log.Debug().Msg("initialised")
if newStore {
// from MigrateData
store.VersionService.StoreDBVersion(portainer.DBVersion)
@ -77,6 +83,6 @@ func NewTestStore(t *testing.T, init, secure bool) (bool, *Store, func(), error)
func teardown(store *Store) {
err := store.Close()
if err != nil {
log.Fatalln(err)
log.Fatal().Err(err).Msg("")
}
}