feat(version): migrate version to semver [EE-3756] (#7693)

redisigned version bucket and migration code

api/datastore/migrator/migrate_ce.go

@@ -4,143 +4,124 @@ import (
 	"reflect"
 	"runtime"
 
+	"github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
 
-	"github.com/pkg/errors"
+	"github.com/Masterminds/semver"
 	"github.com/rs/zerolog/log"
 )
 
-type migration struct {
-	dbversion int
-	migrate   func() error
-}
-
 func migrationError(err error, context string) error {
 	return errors.Wrap(err, "failed in "+context)
 }
 
-func newMigration(dbversion int, migrate func() error) migration {
-	return migration{
-		dbversion: dbversion,
-		migrate:   migrate,
-	}
-}
-
-func dbTooOldError() error {
-	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
-}
-
 func GetFunctionName(i interface{}) string {
 	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
 }
 
 // Migrate checks the database version and migrate the existing data to the most recent data model.
 func (m *Migrator) Migrate() error {
-	// set DB to updating status
-	err := m.versionService.StoreIsUpdating(true)
+	version, err := m.versionService.Version()
 	if err != nil {
-		return migrationError(err, "StoreIsUpdating")
+		return migrationError(err, "get version service")
 	}
 
-	migrations := []migration{
-		// Portainer < 1.21.0
-		newMigration(17, dbTooOldError),
-
-		// Portainer 1.21.0
-		newMigration(18, m.updateUsersToDBVersion18),
-		newMigration(18, m.updateEndpointsToDBVersion18),
-		newMigration(18, m.updateEndpointGroupsToDBVersion18),
-		newMigration(18, m.updateRegistriesToDBVersion18),
-
-		// 1.22.0
-		newMigration(19, m.updateSettingsToDBVersion19),
-
-		// 1.22.1
-		newMigration(20, m.updateUsersToDBVersion20),
-		newMigration(20, m.updateSettingsToDBVersion20),
-		newMigration(20, m.updateSchedulesToDBVersion20),
-
-		// Portainer 1.23.0
-		// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-		newMigration(22, m.updateResourceControlsToDBVersion22),
-		newMigration(22, m.updateUsersAndRolesToDBVersion22),
-
-		// Portainer 1.24.0
-		newMigration(23, m.updateTagsToDBVersion23),
-		newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
-
-		// Portainer 1.24.1
-		newMigration(24, m.updateSettingsToDB24),
-
-		// Portainer 2.0.0
-		newMigration(25, m.updateSettingsToDB25),
-		newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
-
-		// Portainer 2.1.0
-		newMigration(26, m.updateEndpointSettingsToDB25),
-
-		// Portainer 2.2.0
-		newMigration(27, m.updateStackResourceControlToDB27),
-
-		// Portainer 2.6.0
-		newMigration(30, m.migrateDBVersionToDB30),
-
-		// Portainer 2.9.0
-		newMigration(32, m.migrateDBVersionToDB32),
-
-		// Portainer 2.9.1, 2.9.2
-		newMigration(33, m.migrateDBVersionToDB33),
-
-		// Portainer 2.10
-		newMigration(34, m.migrateDBVersionToDB34),
-
-		// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
-		newMigration(35, m.migrateDBVersionToDB35),
-
-		newMigration(36, m.migrateDBVersionToDB36),
-
-		// Portainer 2.13
-		newMigration(40, m.migrateDBVersionToDB40),
-
-		// Portainer 2.14
-		newMigration(50, m.migrateDBVersionToDB50),
-
-		// Portainer 2.15
-		newMigration(60, m.migrateDBVersionToDB60),
-
-		// Portainer 2.16
-		newMigration(70, m.migrateDBVersionToDB70),
-
-		// Portainer 2.16.1
-		newMigration(71, m.migrateDBVersionToDB71),
+	schemaVersion, err := semver.NewVersion(version.SchemaVersion)
+	if err != nil {
+		return migrationError(err, "invalid db schema version")
 	}
 
-	var lastDbVersion int
-	for _, migration := range migrations {
-		if m.currentDBVersion < migration.dbversion {
+	newMigratorCount := 0
+	versionUpdateRequired := false
+	if schemaVersion.Equal(semver.MustParse(portainer.APIVersion)) {
+		// detect and run migrations when the versions are the same.
+		// e.g. development builds
+		latestMigrations := m.latestMigrations()
+		if latestMigrations.version.Equal(schemaVersion) &&
+			version.MigratorCount != len(latestMigrations.migrationFuncs) {
 
-			// Print the next line only when the version changes
-			if migration.dbversion > lastDbVersion {
-				log.Info().Int("to_version", migration.dbversion).Msg("migrating DB")
-			}
-
-			err := migration.migrate()
+			versionUpdateRequired = true
+			err := runMigrations(latestMigrations.migrationFuncs)
 			if err != nil {
-				return migrationError(err, GetFunctionName(migration.migrate))
+				return err
 			}
+			newMigratorCount = len(latestMigrations.migrationFuncs)
+		}
+	} else {
+		// regular path when major/minor/patch versions differ
+		for _, migration := range m.migrations {
+			if schemaVersion.LessThan(migration.version) {
+				versionUpdateRequired = true
+				log.Info().Msgf("migrating data to %s", migration.version.String())
+				err := runMigrations(migration.migrationFuncs)
+				if err != nil {
+					return err
+				}
+			}
+
+			newMigratorCount = len(migration.migrationFuncs)
 		}
-		lastDbVersion = migration.dbversion
 	}
 
-	log.Info().Int("version", portainer.DBVersion).Msg("setting DB version")
+	if versionUpdateRequired || newMigratorCount != version.MigratorCount {
+		err := m.Always()
+		if err != nil {
+			return migrationError(err, "Always migrations returned error")
+		}
 
-	err = m.versionService.StoreDBVersion(portainer.DBVersion)
-	if err != nil {
-		return migrationError(err, "StoreDBVersion")
+		version.SchemaVersion = portainer.APIVersion
+		version.MigratorCount = newMigratorCount
+
+		err = m.versionService.UpdateVersion(version)
+		if err != nil {
+			return migrationError(err, "StoreDBVersion")
+		}
+
+		log.Info().Msgf("db migrated to %s", portainer.APIVersion)
 	}
 
-	log.Info().Int("version", portainer.DBVersion).Msg("updated DB version")
-
-	// reset DB updating status
-	return m.versionService.StoreIsUpdating(false)
+	return nil
+}
+
+func runMigrations(migrationFuncs []func() error) error {
+	for _, migrationFunc := range migrationFuncs {
+		err := migrationFunc()
+		if err != nil {
+			return migrationError(err, GetFunctionName(migrationFunc))
+		}
+	}
+	return nil
+}
+
+func (m *Migrator) NeedsMigration() bool {
+	// we need to migrate if anything changes with the version in the DB vs what our software version is.
+	// If the version matches, then it's all down to the number of migration funcs we have for the current version
+	// i.e. the MigratorCount
+
+	// In this particular instance we should log a fatal error
+	if m.CurrentDBEdition() != portainer.PortainerCE {
+		log.Fatal().Msg("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+		return false
+	}
+
+	if m.CurrentSemanticDBVersion().LessThan(semver.MustParse(portainer.APIVersion)) {
+		return true
+	}
+
+	// Check if we have any migrations for the current version
+	latestMigrations := m.latestMigrations()
+	if latestMigrations.version.Equal(semver.MustParse(portainer.APIVersion)) {
+		if m.currentDBVersion.MigratorCount != len(latestMigrations.migrationFuncs) {
+			return true
+		}
+	} else {
+		// One remaining possibility if we get here.  If our migrator count > 0 and we have no migration funcs
+		// for the current version (i.e. they were deleted during development).  Then we we need to migrate.
+		// This is to reset the migrator count back to 0
+		if m.currentDBVersion.MigratorCount > 0 {
+			return true
+		}
+	}
+
+	return false
 }

api/datastore/migrator/migrate_dbversion33.go

@@ -12,7 +12,7 @@ func (m *Migrator) migrateDBVersionToDB34() error {
 	return MigrateStackEntryPoint(m.stackService)
 }
 
-// MigrateStackEntryPoint exported for testing (blah.)
+// MigrateStackEntryPoint exported for testing
 func MigrateStackEntryPoint(stackService dataservices.StackService) error {
 	stacks, err := stackService.Stacks()
 	if err != nil {

api/datastore/migrator/migrate_dbversion34.go

@@ -1,12 +1,7 @@
 package migrator
 
-import "github.com/rs/zerolog/log"
-
 func (m *Migrator) migrateDBVersionToDB35() error {
 	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
 	// calling it again will now fix the issue as the function has been repaired.
-
-	log.Info().Msg("updating dockerhub registries")
-
 	return m.updateDockerhubToDB32()
 }

api/datastore/migrator/migrate_dbversion60.go

@@ -19,12 +19,13 @@ func (m *Migrator) addGpuInputFieldDB60() error {
 	}
 
 	for _, endpoint := range endpoints {
-		endpoint.Gpus = []portainer.Pair{}
-		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
-		if err != nil {
-			return err
+		if endpoint.Gpus == nil {
+			endpoint.Gpus = []portainer.Pair{}
+			err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
 		}
-
 	}
 
 	return nil

api/datastore/migrator/migrate_dbversion70.go

@@ -7,7 +7,7 @@ import (
 )
 
 func (m *Migrator) migrateDBVersionToDB70() error {
-	log.Info().Msg("- add IngressAvailabilityPerNamespace field")
+	log.Info().Msg("add IngressAvailabilityPerNamespace field")
 	if err := m.updateIngressFieldsForEnvDB70(); err != nil {
 		return err
 	}

api/datastore/migrator/migrator.go

@@ -1,7 +1,13 @@
 package migrator
 
 import (
+	"errors"
+
+	"github.com/Masterminds/semver"
+	"github.com/rs/zerolog/log"
+
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/models"
 	"github.com/portainer/portainer/api/dataservices/dockerhub"
 	"github.com/portainer/portainer/api/dataservices/endpoint"
 	"github.com/portainer/portainer/api/dataservices/endpointgroup"
@@ -25,7 +31,9 @@ import (
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		currentDBVersion        int
+		currentDBVersion *models.Version
+		migrations       []Migrations
+
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
@@ -49,7 +57,7 @@ type (
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
 	MigratorParameters struct {
-		DatabaseVersion         int
+		CurrentDBVersion        *models.Version
 		EndpointGroupService    *endpointgroup.Service
 		EndpointService         *endpoint.Service
 		EndpointRelationService *endpointrelation.Service
@@ -74,8 +82,8 @@ type (
 
 // NewMigrator creates a new Migrator.
 func NewMigrator(parameters *MigratorParameters) *Migrator {
-	return &Migrator{
-		currentDBVersion:        parameters.DatabaseVersion,
+	migrator := &Migrator{
+		currentDBVersion:        parameters.CurrentDBVersion,
 		endpointGroupService:    parameters.EndpointGroupService,
 		endpointService:         parameters.EndpointService,
 		endpointRelationService: parameters.EndpointRelationService,
@@ -96,9 +104,112 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		authorizationService:    parameters.AuthorizationService,
 		dockerhubService:        parameters.DockerhubService,
 	}
+
+	migrator.initMigrations()
+	return migrator
 }
 
-// Version exposes version of database
-func (migrator *Migrator) Version() int {
-	return migrator.currentDBVersion
+func (m *Migrator) CurrentDBVersion() string {
+	return m.currentDBVersion.SchemaVersion
+}
+
+func (m *Migrator) CurrentDBEdition() portainer.SoftwareEdition {
+	return portainer.SoftwareEdition(m.currentDBVersion.Edition)
+}
+
+func (m *Migrator) CurrentSemanticDBVersion() *semver.Version {
+	v, err := semver.NewVersion(m.currentDBVersion.SchemaVersion)
+	if err != nil {
+		log.Fatal().Stack().Err(err).Msg("failed to parse current version")
+	}
+
+	return v
+}
+
+func (m *Migrator) addMigrations(v string, funcs ...func() error) {
+	m.migrations = append(m.migrations, Migrations{
+		version:        semver.MustParse(v),
+		migrationFuncs: funcs,
+	})
+}
+
+func (m *Migrator) latestMigrations() Migrations {
+	return m.migrations[len(m.migrations)-1]
+}
+
+// !NOTE: Migration funtions should ideally be idempotent.
+// !      Which simply means the function can run over the same data many times but only transform it once.
+// !      In practice this really just means an extra check or two to ensure we're not destroying valid data.
+// !      This is not a hard rule though.  Understand the limitations.  A migration function may only run over
+// !      the data more than once if a new migration function is added and the version of your database schema is
+// !      the same.  e.g. two developers working on the same version add two different functions for different things.
+// !      This increases the migration funcs count and so they all run again.
+
+type Migrations struct {
+	version        *semver.Version
+	migrationFuncs MigrationFuncs
+}
+
+type MigrationFuncs []func() error
+
+func (m *Migrator) initMigrations() {
+	// !IMPORTANT: Do not be tempted to alter the order of these migrations.
+	// !           Even though one of them looks out of order. Caused by history related
+	// !           to maintaining two versions and releasing at different times
+
+	m.addMigrations("1.0.0", dbTooOldError) // default version found after migration
+
+	m.addMigrations("1.21",
+		m.updateUsersToDBVersion18,
+		m.updateEndpointsToDBVersion18,
+		m.updateEndpointGroupsToDBVersion18,
+		m.updateRegistriesToDBVersion18)
+
+	m.addMigrations("1.22", m.updateSettingsToDBVersion19)
+
+	m.addMigrations("1.22.1",
+		m.updateUsersToDBVersion20,
+		m.updateSettingsToDBVersion20,
+		m.updateSchedulesToDBVersion20)
+
+	m.addMigrations("1.23",
+		m.updateResourceControlsToDBVersion22,
+		m.updateUsersAndRolesToDBVersion22)
+
+	m.addMigrations("1.24",
+		m.updateTagsToDBVersion23,
+		m.updateEndpointsAndEndpointGroupsToDBVersion23)
+
+	m.addMigrations("1.24.1", m.updateSettingsToDB24)
+
+	m.addMigrations("2.0",
+		m.updateSettingsToDB25,
+		m.updateStacksToDB24)
+
+	m.addMigrations("2.1", m.updateEndpointSettingsToDB25)
+	m.addMigrations("2.2", m.updateStackResourceControlToDB27)
+	m.addMigrations("2.6", m.migrateDBVersionToDB30)
+	m.addMigrations("2.9", m.migrateDBVersionToDB32)
+	m.addMigrations("2.9.2", m.migrateDBVersionToDB33)
+	m.addMigrations("2.10.0", m.migrateDBVersionToDB34)
+	m.addMigrations("2.9.3", m.migrateDBVersionToDB35)
+	m.addMigrations("2.12", m.migrateDBVersionToDB36)
+	m.addMigrations("2.13", m.migrateDBVersionToDB40)
+	m.addMigrations("2.14", m.migrateDBVersionToDB50)
+	m.addMigrations("2.15", m.migrateDBVersionToDB60)
+	m.addMigrations("2.16", m.migrateDBVersionToDB70)
+	m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
+
+	// Add new migrations below...
+	// One function per migration, each versions migration funcs in the same file.
+}
+
+// Always is always run at the end of migrations
+func (m *Migrator) Always() error {
+	// currently nothing to be done in CE... yet
+	return nil
+}
+
+func dbTooOldError() error {
+	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support")
 }