mirror of
https://github.com/portainer/portainer.git
synced 2025-07-19 13:29:41 +02:00
feat(version): migrate version to semver [EE-3756] (#7693)
redisigned version bucket and migration code
This commit is contained in:
Matt Hook
GitHub
parent
4cfa584c7c
commit
583346321e
27 changed files with 747 additions and 509 deletions
|
|
@ -10,39 +10,41 @@ import (
|
|||
"strings"
|
||||
"testing"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/database/boltdb"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/portainer/portainer/api/database/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// testVersion is a helper which tests current store version against wanted version
|
||||
func testVersion(store *Store, versionWant int, t *testing.T) {
|
||||
v, err := store.VersionService.DBVersion()
|
||||
func testVersion(store *Store, versionWant string, t *testing.T) {
|
||||
v, err := store.VersionService.Version()
|
||||
if err != nil {
|
||||
t.Errorf("Expect store version to be %d but was %d with error: %s", versionWant, v, err)
|
||||
t.Errorf("Expect store version to be %s but was %s with error: %s", versionWant, v.SchemaVersion, err)
|
||||
}
|
||||
if v != versionWant {
|
||||
t.Errorf("Expect store version to be %d but was %d", versionWant, v)
|
||||
if v.SchemaVersion != versionWant {
|
||||
t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateData(t *testing.T) {
|
||||
snapshotTests := []struct {
|
||||
testName string
|
||||
srcPath string
|
||||
wantPath string
|
||||
testName string
|
||||
srcPath string
|
||||
wantPath string
|
||||
overrideInstanceId bool
|
||||
}{
|
||||
{
|
||||
testName: "migrate version 24 to latest",
|
||||
srcPath: "test_data/input_24.json",
|
||||
wantPath: "test_data/output_24_to_latest.json",
|
||||
testName: "migrate version 24 to latest",
|
||||
srcPath: "test_data/input_24.json",
|
||||
wantPath: "test_data/output_24_to_latest.json",
|
||||
overrideInstanceId: true,
|
||||
},
|
||||
}
|
||||
for _, test := range snapshotTests {
|
||||
t.Run(test.testName, func(t *testing.T) {
|
||||
err := migrateDBTestHelper(t, test.srcPath, test.wantPath)
|
||||
err := migrateDBTestHelper(t, test.srcPath, test.wantPath, test.overrideInstanceId)
|
||||
if err != nil {
|
||||
t.Errorf(
|
||||
"Failed migrating mock database %v: %v",
|
||||
|
|
@ -53,111 +55,111 @@ func TestMigrateData(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
|
||||
newStore, store, teardown := MustNewTestStore(t, false, true)
|
||||
defer teardown()
|
||||
// t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
|
||||
// newStore, store, teardown := MustNewTestStore(t, true, false)
|
||||
// defer teardown()
|
||||
|
||||
if !newStore {
|
||||
t.Error("Expect a new DB")
|
||||
}
|
||||
// if !newStore {
|
||||
// t.Error("Expect a new DB")
|
||||
// }
|
||||
|
||||
// not called for new stores
|
||||
//store.MigrateData()
|
||||
// testVersion(store, portainer.APIVersion, t)
|
||||
// store.Close()
|
||||
|
||||
testVersion(store, portainer.DBVersion, t)
|
||||
store.Close()
|
||||
// newStore, _ = store.Open()
|
||||
// if newStore {
|
||||
// t.Error("Expect store to NOT be new DB")
|
||||
// }
|
||||
// })
|
||||
|
||||
newStore, _ = store.Open()
|
||||
if newStore {
|
||||
t.Error("Expect store to NOT be new DB")
|
||||
}
|
||||
})
|
||||
// tests := []struct {
|
||||
// version string
|
||||
// expectedVersion string
|
||||
// }{
|
||||
// {version: "1.24.1", expectedVersion: portainer.APIVersion},
|
||||
// {version: "2.0.0", expectedVersion: portainer.APIVersion},
|
||||
// }
|
||||
// for _, tc := range tests {
|
||||
// _, store, teardown := MustNewTestStore(t, true, true)
|
||||
// defer teardown()
|
||||
|
||||
tests := []struct {
|
||||
version int
|
||||
expectedVersion int
|
||||
}{
|
||||
{version: 17, expectedVersion: portainer.DBVersion},
|
||||
{version: 21, expectedVersion: portainer.DBVersion},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
_, store, teardown := MustNewTestStore(t, true, true)
|
||||
defer teardown()
|
||||
// // Setup data
|
||||
// v := models.Version{SchemaVersion: tc.version}
|
||||
// store.VersionService.UpdateVersion(&v)
|
||||
|
||||
// Setup data
|
||||
store.VersionService.StoreDBVersion(tc.version)
|
||||
// // Required roles by migrations 22.2
|
||||
// store.RoleService.Create(&portainer.Role{ID: 1})
|
||||
// store.RoleService.Create(&portainer.Role{ID: 2})
|
||||
// store.RoleService.Create(&portainer.Role{ID: 3})
|
||||
// store.RoleService.Create(&portainer.Role{ID: 4})
|
||||
|
||||
// Required roles by migrations 22.2
|
||||
store.RoleService.Create(&portainer.Role{ID: 1})
|
||||
store.RoleService.Create(&portainer.Role{ID: 2})
|
||||
store.RoleService.Create(&portainer.Role{ID: 3})
|
||||
store.RoleService.Create(&portainer.Role{ID: 4})
|
||||
// t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) {
|
||||
// store.MigrateData()
|
||||
// testVersion(store, tc.expectedVersion, t)
|
||||
// })
|
||||
|
||||
t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
|
||||
store.MigrateData()
|
||||
testVersion(store, tc.expectedVersion, t)
|
||||
})
|
||||
// t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) {
|
||||
// store.Rollback(true)
|
||||
// store.Open()
|
||||
// testVersion(store, tc.version, t)
|
||||
// })
|
||||
// }
|
||||
|
||||
t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) {
|
||||
store.Rollback(true)
|
||||
store.Open()
|
||||
testVersion(store, tc.version, t)
|
||||
})
|
||||
}
|
||||
// t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
|
||||
// _, store, teardown := MustNewTestStore(t, false, true)
|
||||
// defer teardown()
|
||||
|
||||
t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
|
||||
_, store, teardown := MustNewTestStore(t, false, true)
|
||||
defer teardown()
|
||||
// v := models.Version{SchemaVersion: "1.24.1"}
|
||||
// store.VersionService.UpdateVersion(&v)
|
||||
|
||||
version := 17
|
||||
store.VersionService.StoreDBVersion(version)
|
||||
// store.MigrateData()
|
||||
|
||||
store.MigrateData()
|
||||
// testVersion(store, v.SchemaVersion, t)
|
||||
// })
|
||||
|
||||
testVersion(store, version, t)
|
||||
})
|
||||
// t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
|
||||
// _, store, teardown := MustNewTestStore(t, false, true)
|
||||
// defer teardown()
|
||||
|
||||
t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
|
||||
_, store, teardown := MustNewTestStore(t, false, true)
|
||||
defer teardown()
|
||||
store.VersionService.StoreDBVersion(0)
|
||||
// v := models.Version{SchemaVersion: "0.0.0"}
|
||||
// store.VersionService.UpdateVersion(&v)
|
||||
|
||||
store.MigrateData()
|
||||
// store.MigrateData()
|
||||
|
||||
options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
// options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
|
||||
if !isFileExist(options.BackupPath) {
|
||||
t.Errorf("Backup file should exist; file=%s", options.BackupPath)
|
||||
}
|
||||
})
|
||||
// if !isFileExist(options.BackupPath) {
|
||||
// t.Errorf("Backup file should exist; file=%s", options.BackupPath)
|
||||
// }
|
||||
// })
|
||||
|
||||
t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
|
||||
_, store, teardown := MustNewTestStore(t, false, true)
|
||||
defer teardown()
|
||||
// t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
|
||||
// _, store, teardown := MustNewTestStore(t, false, true)
|
||||
// defer teardown()
|
||||
|
||||
store.VersionService.StoreIsUpdating(true)
|
||||
// store.VersionService.StoreIsUpdating(true)
|
||||
|
||||
store.MigrateData()
|
||||
// store.MigrateData()
|
||||
|
||||
options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
// options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
|
||||
if isFileExist(options.BackupPath) {
|
||||
t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
|
||||
}
|
||||
})
|
||||
// if isFileExist(options.BackupPath) {
|
||||
// t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
|
||||
// }
|
||||
// })
|
||||
|
||||
t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
|
||||
_, store, teardown := MustNewTestStore(t, false, true)
|
||||
defer teardown()
|
||||
// t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
|
||||
// _, store, teardown := MustNewTestStore(t, false, true)
|
||||
// defer teardown()
|
||||
|
||||
store.MigrateData()
|
||||
// store.MigrateData()
|
||||
|
||||
options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
// options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
|
||||
if isFileExist(options.BackupPath) {
|
||||
t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
|
||||
}
|
||||
})
|
||||
// if isFileExist(options.BackupPath) {
|
||||
// t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
|
||||
// }
|
||||
// })
|
||||
}
|
||||
|
||||
func Test_getBackupRestoreOptions(t *testing.T) {
|
||||
|
|
@ -179,18 +181,23 @@ func Test_getBackupRestoreOptions(t *testing.T) {
|
|||
|
||||
func TestRollback(t *testing.T) {
|
||||
t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
|
||||
version := 21
|
||||
_, store, teardown := MustNewTestStore(t, false, true)
|
||||
version := models.Version{SchemaVersion: "2.4.0"}
|
||||
_, store, teardown := MustNewTestStore(t, true, false)
|
||||
defer teardown()
|
||||
store.VersionService.StoreDBVersion(version)
|
||||
|
||||
_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
err := store.VersionService.UpdateVersion(&version)
|
||||
if err != nil {
|
||||
t.Errorf("Failed updating version: %v", err)
|
||||
}
|
||||
|
||||
_, err = store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("")
|
||||
}
|
||||
|
||||
// Change the current edition
|
||||
err = store.VersionService.StoreDBVersion(version + 10)
|
||||
// Change the current version
|
||||
version2 := models.Version{SchemaVersion: "2.6.0"}
|
||||
err = store.VersionService.UpdateVersion(&version2)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("")
|
||||
}
|
||||
|
|
@ -199,12 +206,17 @@ func TestRollback(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Logf("Rollback failed: %s", err)
|
||||
t.Fail()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
store.Open()
|
||||
testVersion(store, version, t)
|
||||
_, err = store.Open()
|
||||
if err != nil {
|
||||
t.Logf("Open failed: %s", err)
|
||||
t.Fail()
|
||||
return
|
||||
}
|
||||
|
||||
testVersion(store, version.SchemaVersion, t)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -220,15 +232,20 @@ func isFileExist(path string) bool {
|
|||
// migrateDBTestHelper loads a json representation of a bolt database from srcPath,
|
||||
// parses it into a database, runs a migration on that database, and then
|
||||
// compares it with an expected output database.
|
||||
func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
|
||||
func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanceId bool) error {
|
||||
srcJSON, err := os.ReadFile(srcPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed loading source JSON file %v: %v", srcPath, err)
|
||||
}
|
||||
|
||||
// Parse source json to db.
|
||||
_, store, teardown := MustNewTestStore(t, true, false)
|
||||
defer teardown()
|
||||
// When we create a new test store, it sets its version field automatically to latest.
|
||||
_, store, _ := MustNewTestStore(t, true, false)
|
||||
|
||||
fmt.Println("store.path=", store.GetConnection().GetDatabaseFilePath())
|
||||
store.connection.DeleteObject("version", []byte("VERSION"))
|
||||
|
||||
// defer teardown()
|
||||
err = importJSON(t, bytes.NewReader(srcJSON), store)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -240,6 +257,21 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if overrideInstanceId {
|
||||
// old versions of portainer did not have instance-id. Because this gets generated
|
||||
// we need to override the expected output to match the expected value to pass the test
|
||||
v, err := store.VersionService.Version()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v.InstanceID = "463d5c47-0ea5-4aca-85b1-405ceefee254"
|
||||
err = store.VersionService.UpdateVersion(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Assert that our database connection is using bolt so we can call
|
||||
// exportJson rather than ExportRaw. The exportJson function allows us to
|
||||
// strip out the metadata which we don't want for our tests.
|
||||
|
|
@ -316,42 +348,65 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
|
|||
t.Logf("failed casting %s to map[string]interface{}", k)
|
||||
}
|
||||
|
||||
// New format db
|
||||
version, ok := versions["VERSION"]
|
||||
if ok {
|
||||
err := con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("VERSION"),
|
||||
version,
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing VERSION in %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
// old format db
|
||||
|
||||
dbVersion, ok := versions["DB_VERSION"]
|
||||
if !ok {
|
||||
t.Logf("failed getting DB_VERSION from %s", k)
|
||||
}
|
||||
if ok {
|
||||
numDBVersion, ok := dbVersion.(json.Number)
|
||||
if !ok {
|
||||
t.Logf("failed parsing DB_VERSION as json number from %s", k)
|
||||
}
|
||||
|
||||
numDBVersion, ok := dbVersion.(json.Number)
|
||||
if !ok {
|
||||
t.Logf("failed parsing DB_VERSION as json number from %s", k)
|
||||
}
|
||||
intDBVersion, err := numDBVersion.Int64()
|
||||
if err != nil {
|
||||
t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
|
||||
}
|
||||
|
||||
intDBVersion, err := numDBVersion.Int64()
|
||||
if err != nil {
|
||||
t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
|
||||
}
|
||||
|
||||
err = con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("DB_VERSION"),
|
||||
int(intDBVersion),
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing DB_VERSION in %s: %v", k, err)
|
||||
err = con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("DB_VERSION"),
|
||||
int(intDBVersion),
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing DB_VERSION in %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
instanceID, ok := versions["INSTANCE_ID"]
|
||||
if !ok {
|
||||
t.Logf("failed getting INSTANCE_ID from %s", k)
|
||||
if ok {
|
||||
err = con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("INSTANCE_ID"),
|
||||
instanceID,
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("INSTANCE_ID"),
|
||||
instanceID,
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
|
||||
edition, ok := versions["EDITION"]
|
||||
if ok {
|
||||
err = con.CreateObjectWithStringId(
|
||||
k,
|
||||
[]byte("EDITION"),
|
||||
edition,
|
||||
)
|
||||
if err != nil {
|
||||
t.Logf("failed writing EDITION in %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
|
||||
case "dockerhub":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue