Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-24 15:59:41 +02:00
Code Issues Releases Activity

fix(database): db migration improvements EE-2688 (#6662)

Browse source
This commit is contained in:
Prabhat Khera 2022-04-14 16:25:13 +12:00 • committed by GitHub
parent 85ad4e334a
commit e4241207cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 4276 additions and 258 deletions
Download patch file Download diff file Expand all files Collapse all files

258
api/datastore/migrator/migrate_ce.go
View file

@ -1,16 +1,38 @@
package migrator
import (
"fmt"
"errors"
"reflect"
"runtime"
werrors "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
)
type migration struct {
dbversion int
migrate func() error
}
func migrationError(err error, context string) error {
return werrors.Wrap(err, "failed in "+context)
}
func newMigration(dbversion int, migrate func() error) migration {
return migration{
dbversion: dbversion,
migrate: migrate,
}
}
func dbTooOldError() error {
return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
}
func GetFunctionName(i interface{}) string {
return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}
// Migrate checks the database version and migrate the existing data to the most recent data model.
func (m *Migrator) Migrate() error {
// set DB to updating status
@ -19,181 +41,87 @@ func (m *Migrator) Migrate() error {
return migrationError(err, "StoreIsUpdating")
}
if m.currentDBVersion < 17 {
return migrationError(err, "migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
migrations := []migration{
// Portainer < 1.21.0
newMigration(17, dbTooOldError),
// Portainer 1.21.0
newMigration(18, m.updateUsersToDBVersion18),
newMigration(18, m.updateEndpointsToDBVersion18),
newMigration(18, m.updateEndpointGroupsToDBVersion18),
newMigration(18, m.updateRegistriesToDBVersion18),
// 1.22.0
newMigration(19, m.updateSettingsToDBVersion19),
// 1.22.1
newMigration(20, m.updateUsersToDBVersion20),
newMigration(20, m.updateSettingsToDBVersion20),
newMigration(20, m.updateSchedulesToDBVersion20),
// Portainer 1.23.0
// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
newMigration(22, m.updateResourceControlsToDBVersion22),
newMigration(22, m.updateUsersAndRolesToDBVersion22),
// Portainer 1.24.0
newMigration(23, m.updateTagsToDBVersion23),
newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
// Portainer 1.24.1
newMigration(24, m.updateSettingsToDB24),
// Portainer 2.0.0
newMigration(25, m.updateSettingsToDB25),
newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
// Portainer 2.1.0
newMigration(26, m.updateEndpointSettingsToDB25),
// Portainer 2.2.0
newMigration(27, m.updateStackResourceControlToDB27),
// Portainer 2.6.0
newMigration(30, m.migrateDBVersionToDB30),
// Portainer 2.9.0
newMigration(32, m.migrateDBVersionToDB32),
// Portainer 2.9.1, 2.9.2
newMigration(33, m.migrateDBVersionToDB33),
// Portainer 2.10
newMigration(34, m.migrateDBVersionToDB34),
// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
newMigration(35, m.migrateDBVersionToDB35),
newMigration(36, m.migrateDBVersionToDB36),
}
// Portainer 1.21.0
if m.currentDBVersion < 18 {
err := m.updateUsersToDBVersion18()
if err != nil {
return migrationError(err, "updateUsersToDBVersion18")
}
var lastDbVersion int
for _, migration := range migrations {
if m.currentDBVersion < migration.dbversion {
err = m.updateEndpointsToDBVersion18()
if err != nil {
return migrationError(err, "updateEndpointsToDBVersion18")
}
// Print the next line only when the version changes
if migration.dbversion > lastDbVersion {
migrateLog.Infof("Migrating DB to version %d", migration.dbversion)
}
err = m.updateEndpointGroupsToDBVersion18()
if err != nil {
return migrationError(err, "updateEndpointGroupsToDBVersion18")
}
err = m.updateRegistriesToDBVersion18()
if err != nil {
return migrationError(err, "updateRegistriesToDBVersion18")
err := migration.migrate()
if err != nil {
return migrationError(err, GetFunctionName(migration.migrate))
}
}
lastDbVersion = migration.dbversion
}
// Portainer 1.22.0
if m.currentDBVersion < 19 {
err := m.updateSettingsToDBVersion19()
if err != nil {
return migrationError(err, "updateSettingsToDBVersion19")
}
}
// Portainer 1.22.1
if m.currentDBVersion < 20 {
err := m.updateUsersToDBVersion20()
if err != nil {
return migrationError(err, "updateUsersToDBVersion20")
}
err = m.updateSettingsToDBVersion20()
if err != nil {
return migrationError(err, "updateSettingsToDBVersion20")
}
err = m.updateSchedulesToDBVersion20()
if err != nil {
return migrationError(err, "updateSchedulesToDBVersion20")
}
}
// Portainer 1.23.0
// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
if m.currentDBVersion < 22 {
err := m.updateResourceControlsToDBVersion22()
if err != nil {
return migrationError(err, "updateResourceControlsToDBVersion22")
}
err = m.updateUsersAndRolesToDBVersion22()
if err != nil {
return migrationError(err, "updateUsersAndRolesToDBVersion22")
}
}
// Portainer 1.24.0
if m.currentDBVersion < 23 {
migrateLog.Info("Migrating to DB 23")
err := m.updateTagsToDBVersion23()
if err != nil {
return migrationError(err, "updateTagsToDBVersion23")
}
err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
if err != nil {
return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
}
}
// Portainer 1.24.1
if m.currentDBVersion < 24 {
migrateLog.Info("Migrating to DB 24")
err := m.updateSettingsToDB24()
if err != nil {
return migrationError(err, "updateSettingsToDB24")
}
}
// Portainer 2.0.0
if m.currentDBVersion < 25 {
migrateLog.Info("Migrating to DB 25")
err := m.updateSettingsToDB25()
if err != nil {
return migrationError(err, "updateSettingsToDB25")
}
err = m.updateStacksToDB24()
if err != nil {
return migrationError(err, "updateStacksToDB24")
}
}
// Portainer 2.1.0
if m.currentDBVersion < 26 {
migrateLog.Info("Migrating to DB 26")
err := m.updateEndpointSettingsToDB25()
if err != nil {
return migrationError(err, "updateEndpointSettingsToDB25")
}
}
// Portainer 2.2.0
if m.currentDBVersion < 27 {
migrateLog.Info("Migrating to DB 27")
err := m.updateStackResourceControlToDB27()
if err != nil {
return migrationError(err, "updateStackResourceControlToDB27")
}
}
// Portainer 2.6.0
if m.currentDBVersion < 30 {
migrateLog.Info("Migrating to DB 30")
err := m.migrateDBVersionToDB30()
if err != nil {
return migrationError(err, "migrateDBVersionToDB30")
}
}
// Portainer 2.9.0
if m.currentDBVersion < 32 {
err := m.migrateDBVersionToDB32()
if err != nil {
return migrationError(err, "migrateDBVersionToDB32")
}
}
// Portainer 2.9.1, 2.9.2
if m.currentDBVersion < 33 {
migrateLog.Info("Migrating to DB 33")
err := m.migrateDBVersionToDB33()
if err != nil {
return migrationError(err, "migrateDBVersionToDB33")
}
}
// Portainer 2.10
if m.currentDBVersion < 34 {
migrateLog.Info("Migrating to DB 34")
if err := m.migrateDBVersionToDB34(); err != nil {
return migrationError(err, "migrateDBVersionToDB34")
}
}
// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
if m.currentDBVersion < 35 {
migrateLog.Info("Migrating to DB 35")
if err := m.migrateDBVersionToDB35(); err != nil {
return migrationError(err, "migrateDBVersionToDB35")
}
}
if m.currentDBVersion < 36 {
migrateLog.Info("Migrating to DB 36")
if err := m.migrateDBVersionToDB36(); err != nil {
return migrationError(err, "migrateDBVersionToDB36")
}
}
migrateLog.Infof("Setting DB version to %d", portainer.DBVersion)
err = m.versionService.StoreDBVersion(portainer.DBVersion)
if err != nil {
return migrationError(err, "StoreDBVersion")
}
migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
migrateLog.Infof("Updated DB version to %d", portainer.DBVersion)
// reset DB updating status
return m.versionService.StoreIsUpdating(false)

4
api/datastore/migrator/migrate_dbversion17.go
View file

@ -5,6 +5,7 @@ import (
)
func (m *Migrator) updateUsersToDBVersion18() error {
migrateLog.Info("- updating users")
legacyUsers, err := m.userService.Users()
if err != nil {
return err
@ -39,6 +40,7 @@ func (m *Migrator) updateUsersToDBVersion18() error {
}
func (m *Migrator) updateEndpointsToDBVersion18() error {
migrateLog.Info("- updating endpoints")
legacyEndpoints, err := m.endpointService.Endpoints()
if err != nil {
return err
@ -69,6 +71,7 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
}
func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
migrateLog.Info("- updating endpoint groups")
legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
if err != nil {
return err
@ -99,6 +102,7 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
}
func (m *Migrator) updateRegistriesToDBVersion18() error {
migrateLog.Info("- updating registries")
legacyRegistries, err := m.registryService.Registries()
if err != nil {
return err

1
api/datastore/migrator/migrate_dbversion18.go
View file

@ -3,6 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateSettingsToDBVersion19() error {
migrateLog.Info("- updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion19.go
View file

@ -7,6 +7,7 @@ import (
const scheduleScriptExecutionJobType = 1
func (m *Migrator) updateUsersToDBVersion20() error {
migrateLog.Info("- updating user authentication")
return m.authorizationService.UpdateUsersAuthorizations()
}
@ -22,6 +23,7 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
}
func (m *Migrator) updateSchedulesToDBVersion20() error {
migrateLog.Info("- updating schedules")
legacySchedules, err := m.scheduleService.Schedules()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion20.go
View file

@ -6,6 +6,7 @@ import (
)
func (m *Migrator) updateResourceControlsToDBVersion22() error {
migrateLog.Info("- updating resource controls")
legacyResourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err
@ -24,6 +25,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
}
func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
migrateLog.Info("- updating users and roles")
legacyUsers, err := m.userService.Users()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion22.go
View file

@ -3,7 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateTagsToDBVersion23() error {
migrateLog.Info("Updating tags")
migrateLog.Info("- Updating tags")
tags, err := m.tagService.Tags()
if err != nil {
return err
@ -21,7 +21,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
}
func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
migrateLog.Info("Updating endpoints and endpoint groups")
migrateLog.Info("- updating endpoints and endpoint groups")
tags, err := m.tagService.Tags()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion23.go
View file

@ -3,7 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateSettingsToDB24() error {
migrateLog.Info("Updating Settings")
migrateLog.Info("- updating Settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
@ -18,7 +18,7 @@ func (m *Migrator) updateSettingsToDB24() error {
}
func (m *Migrator) updateStacksToDB24() error {
migrateLog.Info("Updating stacks")
migrateLog.Info("- updating stacks")
stacks, err := m.stackService.Stacks()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion24.go
View file

@ -5,7 +5,7 @@ import (
)
func (m *Migrator) updateSettingsToDB25() error {
migrateLog.Info("Updating settings")
migrateLog.Info("- updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {

2
api/datastore/migrator/migrate_dbversion25.go
View file

@ -5,7 +5,7 @@ import (
)
func (m *Migrator) updateEndpointSettingsToDB25() error {
migrateLog.Info("Updating endpoint settings")
migrateLog.Info("- updating endpoint settings")
settings, err := m.settingsService.Settings()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion26.go
View file

@ -7,7 +7,7 @@ import (
)
func (m *Migrator) updateStackResourceControlToDB27() error {
migrateLog.Info("Updating stack resource controls")
migrateLog.Info("- updating stack resource controls")
resourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion29.go
View file

@ -1,7 +1,7 @@
package migrator
func (m *Migrator) migrateDBVersionToDB30() error {
migrateLog.Info("Updating legacy settings")
migrateLog.Info("- updating legacy settings")
if err := m.MigrateSettingsToDB30(); err != nil {
return err
}

13
api/datastore/migrator/migrate_dbversion31.go
View file

@ -2,38 +2,34 @@ package migrator
import (
"fmt"
"github.com/portainer/portainer/api/dataservices/errors"
"log"
"github.com/portainer/portainer/api/dataservices/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/endpointutils"
snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
)
func (m *Migrator) migrateDBVersionToDB32() error {
migrateLog.Info("Updating registries")
err := m.updateRegistriesToDB32()
if err != nil {
return err
}
migrateLog.Info("Updating dockerhub")
err = m.updateDockerhubToDB32()
if err != nil {
return err
}
migrateLog.Info("Updating resource controls")
if err := m.updateVolumeResourceControlToDB32(); err != nil {
return err
}
migrateLog.Info("Updating kubeconfig expiry")
if err := m.kubeconfigExpiryToDB32(); err != nil {
return err
}
migrateLog.Info("Setting default helm repository url")
if err := m.helmRepositoryURLToDB32(); err != nil {
return err
}
@ -42,6 +38,7 @@ func (m *Migrator) migrateDBVersionToDB32() error {
}
func (m *Migrator) updateRegistriesToDB32() error {
migrateLog.Info("- updating registries")
registries, err := m.registryService.Registries()
if err != nil {
return err
@ -84,6 +81,7 @@ func (m *Migrator) updateRegistriesToDB32() error {
}
func (m *Migrator) updateDockerhubToDB32() error {
migrateLog.Info("- updating dockerhub")
dockerhub, err := m.dockerhubService.DockerHub()
if err == errors.ErrObjectNotFound {
return nil
@ -172,6 +170,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
}
func (m *Migrator) updateVolumeResourceControlToDB32() error {
migrateLog.Info("- updating resource controls")
endpoints, err := m.endpointService.Endpoints()
if err != nil {
return fmt.Errorf("failed fetching environments: %w", err)
@ -264,6 +263,7 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interf
}
func (m *Migrator) kubeconfigExpiryToDB32() error {
migrateLog.Info("- updating kubeconfig expiry")
settings, err := m.settingsService.Settings()
if err != nil {
return err
@ -273,6 +273,7 @@ func (m *Migrator) kubeconfigExpiryToDB32() error {
}
func (m *Migrator) helmRepositoryURLToDB32() error {
migrateLog.Info("- setting default helm repository URL")
settings, err := m.settingsService.Settings()
if err != nil {
return err

7
api/datastore/migrator/migrate_dbversion32.go
View file

@ -1,8 +1,11 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
)
func (m *Migrator) migrateDBVersionToDB33() error {
migrateLog.Info("- updating settings")
if err := m.migrateSettingsToDB33(); err != nil {
return err
}
@ -16,7 +19,7 @@ func (m *Migrator) migrateSettingsToDB33() error {
return err
}
migrateLog.Info("Setting default kubectl shell image")
migrateLog.Info("- setting default kubectl shell image")
settings.KubectlShellImage = portainer.DefaultKubectlShellImage
return m.settingsService.UpdateSettings(settings)
}

6
api/datastore/migrator/migrate_dbversion33.go
View file

@ -1,9 +1,11 @@
package migrator
import "github.com/portainer/portainer/api/dataservices"
import (
"github.com/portainer/portainer/api/dataservices"
)
func (m *Migrator) migrateDBVersionToDB34() error {
migrateLog.Info("Migrating stacks")
migrateLog.Info("- updating stacks")
err := MigrateStackEntryPoint(m.stackService)
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion34.go
View file

@ -3,7 +3,7 @@ package migrator
func (m *Migrator) migrateDBVersionToDB35() error {
// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
// calling it again will now fix the issue as the function has been repaired.
migrateLog.Info("Updating dockerhub registries")
migrateLog.Info("- updating dockerhub registries")
err := m.updateDockerhubToDB32()
if err != nil {
return err