Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-04 21:35:23 +02:00
Code Issues Releases Activity

fix(database): db migration improvements EE-2688 (#6662)

Browse source
This commit is contained in:
Prabhat Khera 2022-04-14 16:25:13 +12:00 • committed by GitHub
parent 85ad4e334a
commit e4241207cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 4276 additions and 258 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/datastore/backup.go
View file

@ -69,6 +69,11 @@ func getBackupRestoreOptions(backupDir string) *BackupOptions {
}
}
// Backup current database with default options
func (store *Store) Backup() (string, error) {
return store.backupWithOptions(nil)
}
func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
if options == nil {
options = &BackupOptions{}

8
api/datastore/backup_test.go
View file

@ -10,7 +10,7 @@ import (
)
func TestCreateBackupFolders(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
connection := store.GetConnection()
@ -27,7 +27,7 @@ func TestCreateBackupFolders(t *testing.T) {
}
func TestStoreCreation(t *testing.T) {
_, store, teardown := MustNewTestStore(true)
_, store, teardown := MustNewTestStore(true, true)
defer teardown()
if store == nil {
@ -40,7 +40,7 @@ func TestStoreCreation(t *testing.T) {
}
func TestBackup(t *testing.T) {
_, store, teardown := MustNewTestStore(true)
_, store, teardown := MustNewTestStore(true, true)
connection := store.GetConnection()
defer teardown()
@ -67,7 +67,7 @@ func TestBackup(t *testing.T) {
}
func TestRemoveWithOptions(t *testing.T) {
_, store, teardown := MustNewTestStore(true)
_, store, teardown := MustNewTestStore(true, true)
defer teardown()
t.Run("successfully removes file if existent", func(t *testing.T) {

2
api/datastore/datastore_test.go
View file

@ -27,7 +27,7 @@ const (
// TestStoreFull an eventually comprehensive set of tests for the Store.
// The idea is what we write to the store, we should read back.
func TestStoreFull(t *testing.T) {
_, store, teardown := MustNewTestStore(true)
_, store, teardown := MustNewTestStore(true, true)
defer teardown()
testCases := map[string]func(t *testing.T){

10
api/datastore/log/log.go
View file

@ -28,10 +28,20 @@ func (slog *ScopedLog) Debug(message string) {
slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Debugf(message string, vars ...interface{}) {
message = fmt.Sprintf(message, vars...)
slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Info(message string) {
slog.print(INFO, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Infof(message string, vars ...interface{}) {
message = fmt.Sprintf(message, vars...)
slog.print(INFO, fmt.Sprintf("[message: %s]", message))
}
func (slog *ScopedLog) Error(message string, err error) {
slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
}

29
api/datastore/migrate_data.go
View file

@ -9,6 +9,7 @@ import (
plog "github.com/portainer/portainer/api/datastore/log"
"github.com/portainer/portainer/api/datastore/migrator"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/sirupsen/logrus"
werrors "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
@ -24,6 +25,12 @@ func (store *Store) MigrateData() error {
return err
}
// Backup Database
backupPath, err := store.Backup()
if err != nil {
return werrors.Wrap(err, "while backing up db before migration")
}
migratorParams := &migrator.MigratorParameters{
DatabaseVersion: version,
EndpointGroupService: store.EndpointGroupService,
@ -46,7 +53,27 @@ func (store *Store) MigrateData() error {
AuthorizationService: authorization.NewService(store),
}
return store.connectionMigrateData(migratorParams)
// restore on error
err = store.connectionMigrateData(migratorParams)
if err != nil {
logrus.Errorf("While DB migration %v. Restoring DB", err)
// Restore options
options := BackupOptions{
BackupPath: backupPath,
}
err := store.restoreWithOptions(&options)
if err != nil {
logrus.Fatalf(
"Failed restoring the backup. portainer database file needs to restored manually by "+
"replacing %s database file with recent backup %s. Error %v",
store.databasePath(),
options.BackupPath,
err,
)
}
}
return err
}
// FailSafeMigrate backup and restore DB if migration fail

293
api/datastore/migrate_data_test.go
View file

@ -1,13 +1,19 @@
package datastore
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
)
// testVersion is a helper which tests current store version against wanted version
@ -22,8 +28,32 @@ func testVersion(store *Store, versionWant int, t *testing.T) {
}
func TestMigrateData(t *testing.T) {
snapshotTests := []struct {
testName string
srcPath string
wantPath string
}{
{
testName: "migrate version 24 to 35",
srcPath: "test_data/input_24.json",
wantPath: "test_data/output_35.json",
},
}
for _, test := range snapshotTests {
t.Run(test.testName, func(t *testing.T) {
err := migrateDBTestHelper(t, test.srcPath, test.wantPath)
if err != nil {
t.Errorf(
"Failed migrating mock database %v: %v",
test.srcPath,
err,
)
}
})
}
t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
newStore, store, teardown := MustNewTestStore(false)
newStore, store, teardown := MustNewTestStore(false, true)
defer teardown()
if !newStore {
@ -50,7 +80,7 @@ func TestMigrateData(t *testing.T) {
{version: 21, expectedVersion: portainer.DBVersion},
}
for _, tc := range tests {
_, store, teardown := MustNewTestStore(true)
_, store, teardown := MustNewTestStore(true, true)
defer teardown()
// Setup data
@ -75,7 +105,7 @@ func TestMigrateData(t *testing.T) {
}
t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
version := 17
@ -87,7 +117,7 @@ func TestMigrateData(t *testing.T) {
})
t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
store.VersionService.StoreDBVersion(0)
@ -101,7 +131,7 @@ func TestMigrateData(t *testing.T) {
})
t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
store.VersionService.StoreIsUpdating(true)
@ -116,7 +146,7 @@ func TestMigrateData(t *testing.T) {
})
t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
store.MigrateData()
@ -131,7 +161,7 @@ func TestMigrateData(t *testing.T) {
}
func Test_getBackupRestoreOptions(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
options := getBackupRestoreOptions(store.commonBackupDir())
@ -150,7 +180,7 @@ func Test_getBackupRestoreOptions(t *testing.T) {
func TestRollback(t *testing.T) {
t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
version := 21
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
store.VersionService.StoreDBVersion(version)
@ -185,3 +215,250 @@ func isFileExist(path string) bool {
}
return len(matches) > 0
}
// migrateDBTestHelper loads a json representation of a bolt database from srcPath,
// parses it into a database, runs a migration on that database, and then
// compares it with an expected output database.
func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
srcJSON, err := os.ReadFile(srcPath)
if err != nil {
t.Fatalf("failed loading source JSON file %v: %v", srcPath, err)
}
// Parse source json to db.
_, store, teardown := MustNewTestStore(true, false)
defer teardown()
err = importJSON(t, bytes.NewReader(srcJSON), store)
if err != nil {
return err
}
// Run the actual migrations on our input database.
err = store.MigrateData()
if err != nil {
return err
}
// Assert that our database connection is using bolt so we can call
// exportJson rather than ExportRaw. The exportJson function allows us to
// strip out the metadata which we don't want for our tests.
// TODO: update connection interface in CE to allow us to use ExportRaw and pass meta false
err = store.connection.Close()
if err != nil {
t.Fatalf("err closing bolt connection: %v", err)
}
con, ok := store.connection.(*boltdb.DbConnection)
if !ok {
t.Fatalf("backing database is not using boltdb, but the migrations test requires it")
}
// Convert database back to json.
databasePath := con.GetDatabaseFilePath()
if _, err := os.Stat(databasePath); err != nil {
return fmt.Errorf("stat on %s failed: %s", databasePath, err)
}
gotJSON, err := con.ExportJson(databasePath, false)
if err != nil {
t.Logf(
"failed re-exporting database %s to JSON: %v",
databasePath,
err,
)
}
wantJSON, err := os.ReadFile(wantPath)
if err != nil {
t.Fatalf("failed loading want JSON file %v: %v", wantPath, err)
}
// Compare the result we got with the one we wanted.
if diff := cmp.Diff(wantJSON, gotJSON); diff != "" {
gotPath := filepath.Join(os.TempDir(), "portainer-migrator-test-fail.json")
os.WriteFile(
gotPath,
gotJSON,
0600,
)
t.Errorf(
"migrate data from %s to %s failed\nwrote migrated input to %s\nmismatch (-want +got):\n%s",
srcPath,
wantPath,
gotPath,
diff,
)
}
return nil
}
// importJSON reads input JSON and commits it to a portainer datastore.Store.
// Errors are logged with the testing package.
func importJSON(t *testing.T, r io.Reader, store *Store) error {
objects := make(map[string]interface{})
// Parse json into map of objects.
d := json.NewDecoder(r)
d.UseNumber()
err := d.Decode(&objects)
if err != nil {
return err
}
// Get database connection from store.
con := store.connection
for k, v := range objects {
switch k {
case "version":
versions, ok := v.(map[string]interface{})
if !ok {
t.Logf("failed casting %s to map[string]interface{}", k)
}
dbVersion, ok := versions["DB_VERSION"]
if !ok {
t.Logf("failed getting DB_VERSION from %s", k)
}
numDBVersion, ok := dbVersion.(json.Number)
if !ok {
t.Logf("failed parsing DB_VERSION as json number from %s", k)
}
intDBVersion, err := numDBVersion.Int64()
if err != nil {
t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
}
err = con.CreateObjectWithStringId(
k,
[]byte("DB_VERSION"),
int(intDBVersion),
)
if err != nil {
t.Logf("failed writing DB_VERSION in %s: %v", k, err)
}
instanceID, ok := versions["INSTANCE_ID"]
if !ok {
t.Logf("failed getting INSTANCE_ID from %s", k)
}
err = con.CreateObjectWithStringId(
k,
[]byte("INSTANCE_ID"),
instanceID,
)
if err != nil {
t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
}
case "dockerhub":
obj, ok := v.([]interface{})
if !ok {
t.Logf("failed to cast %s to []interface{}", k)
}
err := con.CreateObjectWithStringId(
k,
[]byte("DOCKERHUB"),
obj[0],
)
if err != nil {
t.Logf("failed writing DOCKERHUB in %s: %v", k, err)
}
case "ssl":
obj, ok := v.(map[string]interface{})
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
}
err := con.CreateObjectWithStringId(
k,
[]byte("SSL"),
obj,
)
if err != nil {
t.Logf("failed writing SSL in %s: %v", k, err)
}
case "settings":
obj, ok := v.(map[string]interface{})
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
}
err := con.CreateObjectWithStringId(
k,
[]byte("SETTINGS"),
obj,
)
if err != nil {
t.Logf("failed writing SETTINGS in %s: %v", k, err)
}
case "tunnel_server":
obj, ok := v.(map[string]interface{})
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
}
err := con.CreateObjectWithStringId(
k,
[]byte("INFO"),
obj,
)
if err != nil {
t.Logf("failed writing INFO in %s: %v", k, err)
}
case "templates":
continue
default:
objlist, ok := v.([]interface{})
if !ok {
t.Logf("failed to cast %s to []interface{}", k)
}
for _, obj := range objlist {
value, ok := obj.(map[string]interface{})
if !ok {
t.Logf("failed to cast %v to map[string]interface{}", obj)
} else {
var ok bool
var id interface{}
switch k {
case "endpoint_relations":
// TODO: need to make into an int, then do that weird
// stringification
id, ok = value["EndpointID"]
default:
id, ok = value["Id"]
}
if !ok {
// endpoint_relations: EndpointID
t.Logf("missing Id field: %s", k)
id = "error"
}
n, ok := id.(json.Number)
if !ok {
t.Logf("failed to cast %v to json.Number in %s", id, k)
} else {
key, err := n.Int64()
if err != nil {
t.Logf("failed to cast %v to int in %s", n, k)
} else {
err := con.CreateObjectWithId(
k,
int(key),
value,
)
if err != nil {
t.Logf("failed writing %v in %s: %v", key, k, err)
}
}
}
}
}
}
}
return nil
}

2
api/datastore/migrate_dbversion29_test.go
View file

@ -33,7 +33,7 @@ func setup(store *Store) error {
}
func TestMigrateSettings(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
err := setup(store)

2
api/datastore/migrate_dbversion33_test.go
View file

@ -10,7 +10,7 @@ import (
)
func TestMigrateStackEntryPoint(t *testing.T) {
_, store, teardown := MustNewTestStore(false)
_, store, teardown := MustNewTestStore(false, true)
defer teardown()
stackService := store.Stack()

258
api/datastore/migrator/migrate_ce.go
View file

@ -1,16 +1,38 @@
package migrator
import (
"fmt"
"errors"
"reflect"
"runtime"
werrors "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
)
type migration struct {
dbversion int
migrate func() error
}
func migrationError(err error, context string) error {
return werrors.Wrap(err, "failed in "+context)
}
func newMigration(dbversion int, migrate func() error) migration {
return migration{
dbversion: dbversion,
migrate: migrate,
}
}
func dbTooOldError() error {
return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
}
func GetFunctionName(i interface{}) string {
return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}
// Migrate checks the database version and migrate the existing data to the most recent data model.
func (m *Migrator) Migrate() error {
// set DB to updating status
@ -19,181 +41,87 @@ func (m *Migrator) Migrate() error {
return migrationError(err, "StoreIsUpdating")
}
if m.currentDBVersion < 17 {
return migrationError(err, "migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
migrations := []migration{
// Portainer < 1.21.0
newMigration(17, dbTooOldError),
// Portainer 1.21.0
newMigration(18, m.updateUsersToDBVersion18),
newMigration(18, m.updateEndpointsToDBVersion18),
newMigration(18, m.updateEndpointGroupsToDBVersion18),
newMigration(18, m.updateRegistriesToDBVersion18),
// 1.22.0
newMigration(19, m.updateSettingsToDBVersion19),
// 1.22.1
newMigration(20, m.updateUsersToDBVersion20),
newMigration(20, m.updateSettingsToDBVersion20),
newMigration(20, m.updateSchedulesToDBVersion20),
// Portainer 1.23.0
// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
newMigration(22, m.updateResourceControlsToDBVersion22),
newMigration(22, m.updateUsersAndRolesToDBVersion22),
// Portainer 1.24.0
newMigration(23, m.updateTagsToDBVersion23),
newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
// Portainer 1.24.1
newMigration(24, m.updateSettingsToDB24),
// Portainer 2.0.0
newMigration(25, m.updateSettingsToDB25),
newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
// Portainer 2.1.0
newMigration(26, m.updateEndpointSettingsToDB25),
// Portainer 2.2.0
newMigration(27, m.updateStackResourceControlToDB27),
// Portainer 2.6.0
newMigration(30, m.migrateDBVersionToDB30),
// Portainer 2.9.0
newMigration(32, m.migrateDBVersionToDB32),
// Portainer 2.9.1, 2.9.2
newMigration(33, m.migrateDBVersionToDB33),
// Portainer 2.10
newMigration(34, m.migrateDBVersionToDB34),
// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
newMigration(35, m.migrateDBVersionToDB35),
newMigration(36, m.migrateDBVersionToDB36),
}
// Portainer 1.21.0
if m.currentDBVersion < 18 {
err := m.updateUsersToDBVersion18()
if err != nil {
return migrationError(err, "updateUsersToDBVersion18")
}
var lastDbVersion int
for _, migration := range migrations {
if m.currentDBVersion < migration.dbversion {
err = m.updateEndpointsToDBVersion18()
if err != nil {
return migrationError(err, "updateEndpointsToDBVersion18")
}
// Print the next line only when the version changes
if migration.dbversion > lastDbVersion {
migrateLog.Infof("Migrating DB to version %d", migration.dbversion)
}
err = m.updateEndpointGroupsToDBVersion18()
if err != nil {
return migrationError(err, "updateEndpointGroupsToDBVersion18")
}
err = m.updateRegistriesToDBVersion18()
if err != nil {
return migrationError(err, "updateRegistriesToDBVersion18")
err := migration.migrate()
if err != nil {
return migrationError(err, GetFunctionName(migration.migrate))
}
}
lastDbVersion = migration.dbversion
}
// Portainer 1.22.0
if m.currentDBVersion < 19 {
err := m.updateSettingsToDBVersion19()
if err != nil {
return migrationError(err, "updateSettingsToDBVersion19")
}
}
// Portainer 1.22.1
if m.currentDBVersion < 20 {
err := m.updateUsersToDBVersion20()
if err != nil {
return migrationError(err, "updateUsersToDBVersion20")
}
err = m.updateSettingsToDBVersion20()
if err != nil {
return migrationError(err, "updateSettingsToDBVersion20")
}
err = m.updateSchedulesToDBVersion20()
if err != nil {
return migrationError(err, "updateSchedulesToDBVersion20")
}
}
// Portainer 1.23.0
// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
if m.currentDBVersion < 22 {
err := m.updateResourceControlsToDBVersion22()
if err != nil {
return migrationError(err, "updateResourceControlsToDBVersion22")
}
err = m.updateUsersAndRolesToDBVersion22()
if err != nil {
return migrationError(err, "updateUsersAndRolesToDBVersion22")
}
}
// Portainer 1.24.0
if m.currentDBVersion < 23 {
migrateLog.Info("Migrating to DB 23")
err := m.updateTagsToDBVersion23()
if err != nil {
return migrationError(err, "updateTagsToDBVersion23")
}
err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
if err != nil {
return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
}
}
// Portainer 1.24.1
if m.currentDBVersion < 24 {
migrateLog.Info("Migrating to DB 24")
err := m.updateSettingsToDB24()
if err != nil {
return migrationError(err, "updateSettingsToDB24")
}
}
// Portainer 2.0.0
if m.currentDBVersion < 25 {
migrateLog.Info("Migrating to DB 25")
err := m.updateSettingsToDB25()
if err != nil {
return migrationError(err, "updateSettingsToDB25")
}
err = m.updateStacksToDB24()
if err != nil {
return migrationError(err, "updateStacksToDB24")
}
}
// Portainer 2.1.0
if m.currentDBVersion < 26 {
migrateLog.Info("Migrating to DB 26")
err := m.updateEndpointSettingsToDB25()
if err != nil {
return migrationError(err, "updateEndpointSettingsToDB25")
}
}
// Portainer 2.2.0
if m.currentDBVersion < 27 {
migrateLog.Info("Migrating to DB 27")
err := m.updateStackResourceControlToDB27()
if err != nil {
return migrationError(err, "updateStackResourceControlToDB27")
}
}
// Portainer 2.6.0
if m.currentDBVersion < 30 {
migrateLog.Info("Migrating to DB 30")
err := m.migrateDBVersionToDB30()
if err != nil {
return migrationError(err, "migrateDBVersionToDB30")
}
}
// Portainer 2.9.0
if m.currentDBVersion < 32 {
err := m.migrateDBVersionToDB32()
if err != nil {
return migrationError(err, "migrateDBVersionToDB32")
}
}
// Portainer 2.9.1, 2.9.2
if m.currentDBVersion < 33 {
migrateLog.Info("Migrating to DB 33")
err := m.migrateDBVersionToDB33()
if err != nil {
return migrationError(err, "migrateDBVersionToDB33")
}
}
// Portainer 2.10
if m.currentDBVersion < 34 {
migrateLog.Info("Migrating to DB 34")
if err := m.migrateDBVersionToDB34(); err != nil {
return migrationError(err, "migrateDBVersionToDB34")
}
}
// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
if m.currentDBVersion < 35 {
migrateLog.Info("Migrating to DB 35")
if err := m.migrateDBVersionToDB35(); err != nil {
return migrationError(err, "migrateDBVersionToDB35")
}
}
if m.currentDBVersion < 36 {
migrateLog.Info("Migrating to DB 36")
if err := m.migrateDBVersionToDB36(); err != nil {
return migrationError(err, "migrateDBVersionToDB36")
}
}
migrateLog.Infof("Setting DB version to %d", portainer.DBVersion)
err = m.versionService.StoreDBVersion(portainer.DBVersion)
if err != nil {
return migrationError(err, "StoreDBVersion")
}
migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
migrateLog.Infof("Updated DB version to %d", portainer.DBVersion)
// reset DB updating status
return m.versionService.StoreIsUpdating(false)

4
api/datastore/migrator/migrate_dbversion17.go
View file

@ -5,6 +5,7 @@ import (
)
func (m *Migrator) updateUsersToDBVersion18() error {
migrateLog.Info("- updating users")
legacyUsers, err := m.userService.Users()
if err != nil {
return err
@ -39,6 +40,7 @@ func (m *Migrator) updateUsersToDBVersion18() error {
}
func (m *Migrator) updateEndpointsToDBVersion18() error {
migrateLog.Info("- updating endpoints")
legacyEndpoints, err := m.endpointService.Endpoints()
if err != nil {
return err
@ -69,6 +71,7 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
}
func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
migrateLog.Info("- updating endpoint groups")
legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
if err != nil {
return err
@ -99,6 +102,7 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
}
func (m *Migrator) updateRegistriesToDBVersion18() error {
migrateLog.Info("- updating registries")
legacyRegistries, err := m.registryService.Registries()
if err != nil {
return err

1
api/datastore/migrator/migrate_dbversion18.go
View file

@ -3,6 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateSettingsToDBVersion19() error {
migrateLog.Info("- updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion19.go
View file

@ -7,6 +7,7 @@ import (
const scheduleScriptExecutionJobType = 1
func (m *Migrator) updateUsersToDBVersion20() error {
migrateLog.Info("- updating user authentication")
return m.authorizationService.UpdateUsersAuthorizations()
}
@ -22,6 +23,7 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
}
func (m *Migrator) updateSchedulesToDBVersion20() error {
migrateLog.Info("- updating schedules")
legacySchedules, err := m.scheduleService.Schedules()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion20.go
View file

@ -6,6 +6,7 @@ import (
)
func (m *Migrator) updateResourceControlsToDBVersion22() error {
migrateLog.Info("- updating resource controls")
legacyResourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err
@ -24,6 +25,7 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
}
func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
migrateLog.Info("- updating users and roles")
legacyUsers, err := m.userService.Users()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion22.go
View file

@ -3,7 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateTagsToDBVersion23() error {
migrateLog.Info("Updating tags")
migrateLog.Info("- Updating tags")
tags, err := m.tagService.Tags()
if err != nil {
return err
@ -21,7 +21,7 @@ func (m *Migrator) updateTagsToDBVersion23() error {
}
func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
migrateLog.Info("Updating endpoints and endpoint groups")
migrateLog.Info("- updating endpoints and endpoint groups")
tags, err := m.tagService.Tags()
if err != nil {
return err

4
api/datastore/migrator/migrate_dbversion23.go
View file

@ -3,7 +3,7 @@ package migrator
import portainer "github.com/portainer/portainer/api"
func (m *Migrator) updateSettingsToDB24() error {
migrateLog.Info("Updating Settings")
migrateLog.Info("- updating Settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {
@ -18,7 +18,7 @@ func (m *Migrator) updateSettingsToDB24() error {
}
func (m *Migrator) updateStacksToDB24() error {
migrateLog.Info("Updating stacks")
migrateLog.Info("- updating stacks")
stacks, err := m.stackService.Stacks()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion24.go
View file

@ -5,7 +5,7 @@ import (
)
func (m *Migrator) updateSettingsToDB25() error {
migrateLog.Info("Updating settings")
migrateLog.Info("- updating settings")
legacySettings, err := m.settingsService.Settings()
if err != nil {

2
api/datastore/migrator/migrate_dbversion25.go
View file

@ -5,7 +5,7 @@ import (
)
func (m *Migrator) updateEndpointSettingsToDB25() error {
migrateLog.Info("Updating endpoint settings")
migrateLog.Info("- updating endpoint settings")
settings, err := m.settingsService.Settings()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion26.go
View file

@ -7,7 +7,7 @@ import (
)
func (m *Migrator) updateStackResourceControlToDB27() error {
migrateLog.Info("Updating stack resource controls")
migrateLog.Info("- updating stack resource controls")
resourceControls, err := m.resourceControlService.ResourceControls()
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion29.go
View file

@ -1,7 +1,7 @@
package migrator
func (m *Migrator) migrateDBVersionToDB30() error {
migrateLog.Info("Updating legacy settings")
migrateLog.Info("- updating legacy settings")
if err := m.MigrateSettingsToDB30(); err != nil {
return err
}

13
api/datastore/migrator/migrate_dbversion31.go
View file

@ -2,38 +2,34 @@ package migrator
import (
"fmt"
"github.com/portainer/portainer/api/dataservices/errors"
"log"
"github.com/portainer/portainer/api/dataservices/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/endpointutils"
snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
)
func (m *Migrator) migrateDBVersionToDB32() error {
migrateLog.Info("Updating registries")
err := m.updateRegistriesToDB32()
if err != nil {
return err
}
migrateLog.Info("Updating dockerhub")
err = m.updateDockerhubToDB32()
if err != nil {
return err
}
migrateLog.Info("Updating resource controls")
if err := m.updateVolumeResourceControlToDB32(); err != nil {
return err
}
migrateLog.Info("Updating kubeconfig expiry")
if err := m.kubeconfigExpiryToDB32(); err != nil {
return err
}
migrateLog.Info("Setting default helm repository url")
if err := m.helmRepositoryURLToDB32(); err != nil {
return err
}
@ -42,6 +38,7 @@ func (m *Migrator) migrateDBVersionToDB32() error {
}
func (m *Migrator) updateRegistriesToDB32() error {
migrateLog.Info("- updating registries")
registries, err := m.registryService.Registries()
if err != nil {
return err
@ -84,6 +81,7 @@ func (m *Migrator) updateRegistriesToDB32() error {
}
func (m *Migrator) updateDockerhubToDB32() error {
migrateLog.Info("- updating dockerhub")
dockerhub, err := m.dockerhubService.DockerHub()
if err == errors.ErrObjectNotFound {
return nil
@ -172,6 +170,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
}
func (m *Migrator) updateVolumeResourceControlToDB32() error {
migrateLog.Info("- updating resource controls")
endpoints, err := m.endpointService.Endpoints()
if err != nil {
return fmt.Errorf("failed fetching environments: %w", err)
@ -264,6 +263,7 @@ func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interf
}
func (m *Migrator) kubeconfigExpiryToDB32() error {
migrateLog.Info("- updating kubeconfig expiry")
settings, err := m.settingsService.Settings()
if err != nil {
return err
@ -273,6 +273,7 @@ func (m *Migrator) kubeconfigExpiryToDB32() error {
}
func (m *Migrator) helmRepositoryURLToDB32() error {
migrateLog.Info("- setting default helm repository URL")
settings, err := m.settingsService.Settings()
if err != nil {
return err

7
api/datastore/migrator/migrate_dbversion32.go
View file

@ -1,8 +1,11 @@
package migrator
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
)
func (m *Migrator) migrateDBVersionToDB33() error {
migrateLog.Info("- updating settings")
if err := m.migrateSettingsToDB33(); err != nil {
return err
}
@ -16,7 +19,7 @@ func (m *Migrator) migrateSettingsToDB33() error {
return err
}
migrateLog.Info("Setting default kubectl shell image")
migrateLog.Info("- setting default kubectl shell image")
settings.KubectlShellImage = portainer.DefaultKubectlShellImage
return m.settingsService.UpdateSettings(settings)
}

6
api/datastore/migrator/migrate_dbversion33.go
View file

@ -1,9 +1,11 @@
package migrator
import "github.com/portainer/portainer/api/dataservices"
import (
"github.com/portainer/portainer/api/dataservices"
)
func (m *Migrator) migrateDBVersionToDB34() error {
migrateLog.Info("Migrating stacks")
migrateLog.Info("- updating stacks")
err := MigrateStackEntryPoint(m.stackService)
if err != nil {
return err

2
api/datastore/migrator/migrate_dbversion34.go
View file

@ -3,7 +3,7 @@ package migrator
func (m *Migrator) migrateDBVersionToDB35() error {
// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
// calling it again will now fix the issue as the function has been repaired.
migrateLog.Info("Updating dockerhub registries")
migrateLog.Info("- updating dockerhub registries")
err := m.updateDockerhubToDB32()
if err != nil {
return err

2804
api/datastore/test_data/input_24.json Normal file
View file

File diff suppressed because it is too large Load diff

802
api/datastore/test_data/output_35.json Normal file
View file

@ -0,0 +1,802 @@
{
"dockerhub": [
{
"Authentication": false,
"Username": ""
}
],
"endpoint_groups": [
{
"AuthorizedTeams": null,
"AuthorizedUsers": null,
"Description": "Unassigned endpoints",
"Id": 1,
"Labels": [],
"Name": "Unassigned",
"TagIds": [],
"Tags": null,
"TeamAccessPolicies": {},
"UserAccessPolicies": {}
}
],
"endpoint_relations": [
{
"EdgeStacks": {},
"EndpointID": 1
}
],
"endpoints": [
{
"AuthorizedTeams": null,
"AuthorizedUsers": null,
"AzureCredentials": {
"ApplicationID": "",
"AuthenticationKey": "",
"TenantID": ""
},
"ComposeSyntaxMaxVersion": "",
"EdgeCheckinInterval": 0,
"EdgeKey": "",
"GroupId": 1,
"Id": 1,
"IsEdgeDevice": false,
"Kubernetes": {
"Configuration": {
"IngressClasses": null,
"RestrictDefaultNamespace": false,
"StorageClasses": null,
"UseLoadBalancer": false,
"UseServerMetrics": false
},
"Snapshots": null
},
"LastCheckInDate": 0,
"Name": "local",
"PublicURL": "",
"QueryDate": 0,
"SecuritySettings": {
"allowBindMountsForRegularUsers": true,
"allowContainerCapabilitiesForRegularUsers": true,
"allowDeviceMappingForRegularUsers": true,
"allowHostNamespaceForRegularUsers": true,
"allowPrivilegedModeForRegularUsers": true,
"allowStackManagementForRegularUsers": true,
"allowSysctlSettingForRegularUsers": false,
"allowVolumeBrowserForRegularUsers": false,
"enableHostManagementFeatures": false
},
"Snapshots": [
{
"DockerSnapshotRaw": {
"Containers": null,
"Images": null,
"Info": null,
"Networks": null,
"Version": null,
"Volumes": null
},
"DockerVersion": "20.10.13",
"HealthyContainerCount": 0,
"ImageCount": 9,
"NodeCount": 0,
"RunningContainerCount": 5,
"ServiceCount": 0,
"StackCount": 2,
"StoppedContainerCount": 0,
"Swarm": false,
"Time": 1648610112,
"TotalCPU": 8,
"TotalMemory": 25098706944,
"UnhealthyContainerCount": 0,
"VolumeCount": 10
}
],
"Status": 1,
"TLSConfig": {
"TLS": false,
"TLSSkipVerify": false
},
"TagIds": [],
"Tags": null,
"TeamAccessPolicies": {},
"Type": 1,
"URL": "unix:///var/run/docker.sock",
"UserAccessPolicies": {},
"UserTrusted": false
}
],
"registries": [
{
"Authentication": true,
"AuthorizedTeams": null,
"AuthorizedUsers": null,
"BaseURL": "",
"Ecr": {
"Region": ""
},
"Gitlab": {
"InstanceURL": "",
"ProjectId": 0,
"ProjectPath": ""
},
"Id": 1,
"ManagementConfiguration": null,
"Name": "canister.io",
"Password": "MjWbx8A6YK7cw7",
"Quay": {
"OrganisationName": "",
"UseOrganisation": false
},
"RegistryAccesses": {
"1": {
"Namespaces": [],
"TeamAccessPolicies": {},
"UserAccessPolicies": {}
}
},
"TeamAccessPolicies": {},
"Type": 3,
"URL": "cloud.canister.io:5000",
"UserAccessPolicies": {},
"Username": "prabhatkhera"
}
],
"resource_control": [
{
"AdministratorsOnly": false,
"Id": 2,
"Public": true,
"ResourceId": "762gbwaj8r4gcsdy8ld1u4why",
"SubResourceIds": [],
"System": false,
"TeamAccesses": [],
"Type": 5,
"UserAccesses": []
},
{
"AdministratorsOnly": false,
"Id": 3,
"Public": true,
"ResourceId": "1_alpine",
"SubResourceIds": [],
"System": false,
"TeamAccesses": [],
"Type": 6,
"UserAccesses": []
},
{
"AdministratorsOnly": false,
"Id": 4,
"Public": true,
"ResourceId": "1_redis",
"SubResourceIds": [],
"System": false,
"TeamAccesses": [],
"Type": 6,
"UserAccesses": []
},
{
"AdministratorsOnly": false,
"Id": 5,
"Public": false,
"ResourceId": "1_nginx",
"SubResourceIds": [],
"System": false,
"TeamAccesses": [
{
"AccessLevel": 1,
"TeamId": 1
}
],
"Type": 6,
"UserAccesses": []
}
],
"roles": [
{
"Authorizations": {
"DockerAgentBrowseDelete": true,
"DockerAgentBrowseGet": true,
"DockerAgentBrowseList": true,
"DockerAgentBrowsePut": true,
"DockerAgentBrowseRename": true,
"DockerAgentHostInfo": true,
"DockerAgentList": true,
"DockerAgentPing": true,
"DockerAgentUndefined": true,
"DockerBuildCancel": true,
"DockerBuildPrune": true,
"DockerConfigCreate": true,
"DockerConfigDelete": true,
"DockerConfigInspect": true,
"DockerConfigList": true,
"DockerConfigUpdate": true,
"DockerContainerArchive": true,
"DockerContainerArchiveInfo": true,
"DockerContainerAttach": true,
"DockerContainerAttachWebsocket": true,
"DockerContainerChanges": true,
"DockerContainerCreate": true,
"DockerContainerDelete": true,
"DockerContainerExec": true,
"DockerContainerExport": true,
"DockerContainerInspect": true,
"DockerContainerKill": true,
"DockerContainerList": true,
"DockerContainerLogs": true,
"DockerContainerPause": true,
"DockerContainerPrune": true,
"DockerContainerPutContainerArchive": true,
"DockerContainerRename": true,
"DockerContainerResize": true,
"DockerContainerRestart": true,
"DockerContainerStart": true,
"DockerContainerStats": true,
"DockerContainerStop": true,
"DockerContainerTop": true,
"DockerContainerUnpause": true,
"DockerContainerUpdate": true,
"DockerContainerWait": true,
"DockerDistributionInspect": true,
"DockerEvents": true,
"DockerExecInspect": true,
"DockerExecResize": true,
"DockerExecStart": true,
"DockerImageBuild": true,
"DockerImageCommit": true,
"DockerImageCreate": true,
"DockerImageDelete": true,
"DockerImageGet": true,
"DockerImageGetAll": true,
"DockerImageHistory": true,
"DockerImageInspect": true,
"DockerImageList": true,
"DockerImageLoad": true,
"DockerImagePrune": true,
"DockerImagePush": true,
"DockerImageSearch": true,
"DockerImageTag": true,
"DockerInfo": true,
"DockerNetworkConnect": true,
"DockerNetworkCreate": true,
"DockerNetworkDelete": true,
"DockerNetworkDisconnect": true,
"DockerNetworkInspect": true,
"DockerNetworkList": true,
"DockerNetworkPrune": true,
"DockerNodeDelete": true,
"DockerNodeInspect": true,
"DockerNodeList": true,
"DockerNodeUpdate": true,
"DockerPing": true,
"DockerPluginCreate": true,
"DockerPluginDelete": true,
"DockerPluginDisable": true,
"DockerPluginEnable": true,
"DockerPluginInspect": true,
"DockerPluginList": true,
"DockerPluginPrivileges": true,
"DockerPluginPull": true,
"DockerPluginPush": true,
"DockerPluginSet": true,
"DockerPluginUpgrade": true,
"DockerSecretCreate": true,
"DockerSecretDelete": true,
"DockerSecretInspect": true,
"DockerSecretList": true,
"DockerSecretUpdate": true,
"DockerServiceCreate": true,
"DockerServiceDelete": true,
"DockerServiceInspect": true,
"DockerServiceList": true,
"DockerServiceLogs": true,
"DockerServiceUpdate": true,
"DockerSessionStart": true,
"DockerSwarmInit": true,
"DockerSwarmInspect": true,
"DockerSwarmJoin": true,
"DockerSwarmLeave": true,
"DockerSwarmUnlock": true,
"DockerSwarmUnlockKey": true,
"DockerSwarmUpdate": true,
"DockerSystem": true,
"DockerTaskInspect": true,
"DockerTaskList": true,
"DockerTaskLogs": true,
"DockerUndefined": true,
"DockerVersion": true,
"DockerVolumeCreate": true,
"DockerVolumeDelete": true,
"DockerVolumeInspect": true,
"DockerVolumeList": true,
"DockerVolumePrune": true,
"EndpointResourcesAccess": true,
"IntegrationStoridgeAdmin": true,
"PortainerResourceControlCreate": true,
"PortainerResourceControlUpdate": true,
"PortainerStackCreate": true,
"PortainerStackDelete": true,
"PortainerStackFile": true,
"PortainerStackInspect": true,
"PortainerStackList": true,
"PortainerStackMigrate": true,
"PortainerStackUpdate": true,
"PortainerWebhookCreate": true,
"PortainerWebhookDelete": true,
"PortainerWebhookList": true,
"PortainerWebsocketExec": true
},
"Description": "Full control of all resources in an endpoint",
"Id": 1,
"Name": "Endpoint administrator",
"Priority": 1
},
{
"Authorizations": {
"DockerAgentHostInfo": true,
"DockerAgentList": true,
"DockerAgentPing": true,
"DockerConfigInspect": true,
"DockerConfigList": true,
"DockerContainerArchiveInfo": true,
"DockerContainerChanges": true,
"DockerContainerInspect": true,
"DockerContainerList": true,
"DockerContainerLogs": true,
"DockerContainerStats": true,
"DockerContainerTop": true,
"DockerDistributionInspect": true,
"DockerEvents": true,
"DockerImageGet": true,
"DockerImageGetAll": true,
"DockerImageHistory": true,
"DockerImageInspect": true,
"DockerImageList": true,
"DockerImageSearch": true,
"DockerInfo": true,
"DockerNetworkInspect": true,
"DockerNetworkList": true,
"DockerNodeInspect": true,
"DockerNodeList": true,
"DockerPing": true,
"DockerPluginList": true,
"DockerSecretInspect": true,
"DockerSecretList": true,
"DockerServiceInspect": true,
"DockerServiceList": true,
"DockerServiceLogs": true,
"DockerSwarmInspect": true,
"DockerSystem": true,
"DockerTaskInspect": true,
"DockerTaskList": true,
"DockerTaskLogs": true,
"DockerVersion": true,
"DockerVolumeInspect": true,
"DockerVolumeList": true,
"EndpointResourcesAccess": true,
"PortainerStackFile": true,
"PortainerStackInspect": true,
"PortainerStackList": true,
"PortainerWebhookList": true
},
"Description": "Read-only access of all resources in an endpoint",
"Id": 2,
"Name": "Helpdesk",
"Priority": 2
},
{
"Authorizations": {
"DockerAgentHostInfo": true,
"DockerAgentList": true,
"DockerAgentPing": true,
"DockerAgentUndefined": true,
"DockerBuildCancel": true,
"DockerBuildPrune": true,
"DockerConfigCreate": true,
"DockerConfigDelete": true,
"DockerConfigInspect": true,
"DockerConfigList": true,
"DockerConfigUpdate": true,
"DockerContainerArchive": true,
"DockerContainerArchiveInfo": true,
"DockerContainerAttach": true,
"DockerContainerAttachWebsocket": true,
"DockerContainerChanges": true,
"DockerContainerCreate": true,
"DockerContainerDelete": true,
"DockerContainerExec": true,
"DockerContainerExport": true,
"DockerContainerInspect": true,
"DockerContainerKill": true,
"DockerContainerList": true,
"DockerContainerLogs": true,
"DockerContainerPause": true,
"DockerContainerPutContainerArchive": true,
"DockerContainerRename": true,
"DockerContainerResize": true,
"DockerContainerRestart": true,
"DockerContainerStart": true,
"DockerContainerStats": true,
"DockerContainerStop": true,
"DockerContainerTop": true,
"DockerContainerUnpause": true,
"DockerContainerUpdate": true,
"DockerContainerWait": true,
"DockerDistributionInspect": true,
"DockerEvents": true,
"DockerExecInspect": true,
"DockerExecResize": true,
"DockerExecStart": true,
"DockerImageBuild": true,
"DockerImageCommit": true,
"DockerImageCreate": true,
"DockerImageDelete": true,
"DockerImageGet": true,
"DockerImageGetAll": true,
"DockerImageHistory": true,
"DockerImageInspect": true,
"DockerImageList": true,
"DockerImageLoad": true,
"DockerImagePush": true,
"DockerImageSearch": true,
"DockerImageTag": true,
"DockerInfo": true,
"DockerNetworkConnect": true,
"DockerNetworkCreate": true,
"DockerNetworkDelete": true,
"DockerNetworkDisconnect": true,
"DockerNetworkInspect": true,
"DockerNetworkList": true,
"DockerNodeDelete": true,
"DockerNodeInspect": true,
"DockerNodeList": true,
"DockerNodeUpdate": true,
"DockerPing": true,
"DockerPluginCreate": true,
"DockerPluginDelete": true,
"DockerPluginDisable": true,
"DockerPluginEnable": true,
"DockerPluginInspect": true,
"DockerPluginList": true,
"DockerPluginPrivileges": true,
"DockerPluginPull": true,
"DockerPluginPush": true,
"DockerPluginSet": true,
"DockerPluginUpgrade": true,
"DockerSecretCreate": true,
"DockerSecretDelete": true,
"DockerSecretInspect": true,
"DockerSecretList": true,
"DockerSecretUpdate": true,
"DockerServiceCreate": true,
"DockerServiceDelete": true,
"DockerServiceInspect": true,
"DockerServiceList": true,
"DockerServiceLogs": true,
"DockerServiceUpdate": true,
"DockerSessionStart": true,
"DockerSwarmInit": true,
"DockerSwarmInspect": true,
"DockerSwarmJoin": true,
"DockerSwarmLeave": true,
"DockerSwarmUnlock": true,
"DockerSwarmUnlockKey": true,
"DockerSwarmUpdate": true,
"DockerSystem": true,
"DockerTaskInspect": true,
"DockerTaskList": true,
"DockerTaskLogs": true,
"DockerUndefined": true,
"DockerVersion": true,
"DockerVolumeCreate": true,
"DockerVolumeDelete": true,
"DockerVolumeInspect": true,
"DockerVolumeList": true,
"PortainerResourceControlUpdate": true,
"PortainerStackCreate": true,
"PortainerStackDelete": true,
"PortainerStackFile": true,
"PortainerStackInspect": true,
"PortainerStackList": true,
"PortainerStackMigrate": true,
"PortainerStackUpdate": true,
"PortainerWebhookCreate": true,
"PortainerWebhookList": true,
"PortainerWebsocketExec": true
},
"Description": "Full control of assigned resources in an endpoint",
"Id": 3,
"Name": "Standard user",
"Priority": 3
},
{
"Authorizations": {
"DockerAgentHostInfo": true,
"DockerAgentList": true,
"DockerAgentPing": true,
"DockerConfigInspect": true,
"DockerConfigList": true,
"DockerContainerArchiveInfo": true,
"DockerContainerChanges": true,
"DockerContainerInspect": true,
"DockerContainerList": true,
"DockerContainerLogs": true,
"DockerContainerStats": true,
"DockerContainerTop": true,
"DockerDistributionInspect": true,
"DockerEvents": true,
"DockerImageGet": true,
"DockerImageGetAll": true,
"DockerImageHistory": true,
"DockerImageInspect": true,
"DockerImageList": true,
"DockerImageSearch": true,
"DockerInfo": true,
"DockerNetworkInspect": true,
"DockerNetworkList": true,
"DockerNodeInspect": true,
"DockerNodeList": true,
"DockerPing": true,
"DockerPluginList": true,
"DockerSecretInspect": true,
"DockerSecretList": true,
"DockerServiceInspect": true,
"DockerServiceList": true,
"DockerServiceLogs": true,
"DockerSwarmInspect": true,
"DockerSystem": true,
"DockerTaskInspect": true,
"DockerTaskList": true,
"DockerTaskLogs": true,
"DockerVersion": true,
"DockerVolumeInspect": true,
"DockerVolumeList": true,
"PortainerStackFile": true,
"PortainerStackInspect": true,
"PortainerStackList": true,
"PortainerWebhookList": true
},
"Description": "Read-only access of assigned resources in an endpoint",
"Id": 4,
"Name": "Read-only user",
"Priority": 4
}
],
"schedules": [
{
"Created": 1648608136,
"CronExpression": "@every 5m",
"EdgeSchedule": null,
"EndpointSyncJob": null,
"Id": 1,
"JobType": 2,
"Name": "system_snapshot",
"Recurring": true,
"ScriptExecutionJob": null,
"SnapshotJob": {}
}
],
"settings": {
"AllowBindMountsForRegularUsers": true,
"AllowContainerCapabilitiesForRegularUsers": true,
"AllowDeviceMappingForRegularUsers": true,
"AllowHostNamespaceForRegularUsers": true,
"AllowPrivilegedModeForRegularUsers": true,
"AllowStackManagementForRegularUsers": true,
"AllowVolumeBrowserForRegularUsers": false,
"AuthenticationMethod": 1,
"BlackListedLabels": [],
"DisableTrustOnFirstConnect": false,
"DisplayDonationHeader": false,
"DisplayExternalContributors": false,
"EdgeAgentCheckinInterval": 5,
"EnableEdgeComputeFeatures": false,
"EnableHostManagementFeatures": false,
"EnableTelemetry": true,
"EnforceEdgeID": false,
"FeatureFlagSettings": null,
"HelmRepositoryURL": "https://charts.bitnami.com/bitnami",
"KubeconfigExpiry": "0",
"KubectlShellImage": "portainer/kubectl-shell",
"LDAPSettings": {
"AnonymousMode": true,
"AutoCreateUsers": true,
"GroupSearchSettings": [
{
"GroupAttribute": "",
"GroupBaseDN": "",
"GroupFilter": ""
}
],
"ReaderDN": "",
"SearchSettings": [
{
"BaseDN": "",
"Filter": "",
"UserNameAttribute": ""
}
],
"StartTLS": false,
"TLSConfig": {
"TLS": false,
"TLSSkipVerify": false
},
"URL": ""
},
"LogoURL": "",
"OAuthSettings": {
"AccessTokenURI": "",
"AuthorizationURI": "",
"ClientID": "",
"DefaultTeamID": 0,
"KubeSecretKey": null,
"LogoutURI": "",
"OAuthAutoCreateUsers": false,
"RedirectURI": "",
"ResourceURI": "",
"SSO": false,
"Scopes": "",
"UserIdentifier": ""
},
"SnapshotInterval": "5m",
"TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
"UserSessionTimeout": "8h",
"fdoConfiguration": {
"enabled": false,
"ownerPassword": "",
"ownerURL": "",
"ownerUsername": ""
},
"openAMTConfiguration": {
"certFileContent": "",
"certFileName": "",
"certFilePassword": "",
"domainName": "",
"enabled": false,
"mpsPassword": "",
"mpsServer": "",
"mpsToken": "",
"mpsUser": ""
}
},
"ssl": {
"certPath": "",
"httpEnabled": true,
"keyPath": "",
"selfSigned": false
},
"stacks": [
{
"AdditionalFiles": null,
"AutoUpdate": null,
"CreatedBy": "",
"CreationDate": 0,
"EndpointId": 1,
"EntryPoint": "docker/alpine37-compose.yml",
"Env": [],
"FromAppTemplate": false,
"GitConfig": null,
"Id": 2,
"IsComposeFormat": false,
"Name": "alpine",
"Namespace": "",
"ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/2",
"ResourceControl": null,
"Status": 1,
"SwarmId": "s3fd604zdba7z13tbq2x6lyue",
"Type": 1,
"UpdateDate": 0,
"UpdatedBy": ""
},
{
"AdditionalFiles": null,
"AutoUpdate": null,
"CreatedBy": "",
"CreationDate": 0,
"EndpointId": 1,
"EntryPoint": "docker-compose.yml",
"Env": [],
"FromAppTemplate": false,
"GitConfig": null,
"Id": 5,
"IsComposeFormat": false,
"Name": "redis",
"Namespace": "",
"ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/5",
"ResourceControl": null,
"Status": 1,
"SwarmId": "",
"Type": 2,
"UpdateDate": 0,
"UpdatedBy": ""
},
{
"AdditionalFiles": null,
"AutoUpdate": null,
"CreatedBy": "",
"CreationDate": 0,
"EndpointId": 1,
"EntryPoint": "docker-compose.yml",
"Env": [],
"FromAppTemplate": false,
"GitConfig": null,
"Id": 6,
"IsComposeFormat": false,
"Name": "nginx",
"Namespace": "",
"ProjectPath": "/home/prabhat/portainer/data/ce1.25/compose/6",
"ResourceControl": null,
"Status": 1,
"SwarmId": "",
"Type": 2,
"UpdateDate": 0,
"UpdatedBy": ""
}
],
"teams": [
{
"Id": 1,
"Name": "hello"
}
],
"tunnel_server": {
"PrivateKeySeed": "IvX6ZPRuWtLS5zyg"
},
"users": [
{
"EndpointAuthorizations": null,
"Id": 1,
"Password": "$2a$10$siRDprr/5uUFAU8iom3Sr./WXQkN2dhSNjAC471pkJaALkghS762a",
"PortainerAuthorizations": {
"PortainerDockerHubInspect": true,
"PortainerEndpointExtensionAdd": true,
"PortainerEndpointExtensionRemove": true,
"PortainerEndpointGroupList": true,
"PortainerEndpointInspect": true,
"PortainerEndpointList": true,
"PortainerExtensionList": true,
"PortainerMOTD": true,
"PortainerRegistryInspect": true,
"PortainerRegistryList": true,
"PortainerTeamList": true,
"PortainerTemplateInspect": true,
"PortainerTemplateList": true,
"PortainerUserInspect": true,
"PortainerUserList": true,
"PortainerUserMemberships": true
},
"Role": 1,
"Username": "admin"
},
{
"EndpointAuthorizations": null,
"Id": 2,
"Password": "$2a$10$WpCAW8mSt6FRRp1GkynbFOGSZnHR6E5j9cETZ8HiMlw06hVlDW/Li",
"PortainerAuthorizations": {
"PortainerDockerHubInspect": true,
"PortainerEndpointExtensionAdd": true,
"PortainerEndpointExtensionRemove": true,
"PortainerEndpointGroupList": true,
"PortainerEndpointInspect": true,
"PortainerEndpointList": true,
"PortainerExtensionList": true,
"PortainerMOTD": true,
"PortainerRegistryInspect": true,
"PortainerRegistryList": true,
"PortainerTeamList": true,
"PortainerTemplateInspect": true,
"PortainerTemplateList": true,
"PortainerUserInspect": true,
"PortainerUserList": true,
"PortainerUserMemberships": true
},
"Role": 1,
"Username": "prabhat"
}
],
"version": {
"DB_UPDATING": "false",
"DB_VERSION": "35",
"INSTANCE_ID": "null"
}
}

13
api/datastore/teststore.go
View file

@ -18,8 +18,8 @@ func (store *Store) GetConnection() portainer.Connection {
return store.connection
}
func MustNewTestStore(init bool) (bool, *Store, func()) {
newStore, store, teardown, err := NewTestStore(init)
func MustNewTestStore(init, secure bool) (bool, *Store, func()) {
newStore, store, teardown, err := NewTestStore(init, secure)
if err != nil {
if !errors.Is(err, errTempDir) {
teardown()
@ -30,7 +30,7 @@ func MustNewTestStore(init bool) (bool, *Store, func()) {
return newStore, store, teardown
}
func NewTestStore(init bool) (bool, *Store, func(), error) {
func NewTestStore(init, secure bool) (bool, *Store, func(), error) {
// Creates unique temp directory in a concurrency friendly manner.
storePath, err := ioutil.TempDir("", "test-store")
if err != nil {
@ -42,7 +42,12 @@ func NewTestStore(init bool) (bool, *Store, func(), error) {
return false, nil, nil, err
}
connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes"))
secretKey := []byte("apassphrasewhichneedstobe32bytes")
if !secure {
secretKey = nil
}
connection, err := database.NewDatabase("boltdb", storePath, secretKey)
if err != nil {
panic(err)
}

15
api/datastore/validate/validate.go Normal file
View file

@ -0,0 +1,15 @@
package validate
import (
"github.com/go-playground/validator/v10"
portainer "github.com/portainer/portainer/api"
)
var validate *validator.Validate
func ValidateLDAPSettings(ldp *portainer.LDAPSettings) error {
validate = validator.New()
registerValidationMethods(validate)
return validate.Struct(ldp)
}

61
api/datastore/validate/validate_test.go Normal file
View file

@ -0,0 +1,61 @@
package validate
import (
"testing"
portainer "github.com/portainer/portainer/api"
)
func TestValidateLDAPSettings(t *testing.T) {
tests := []struct {
name string
ldap portainer.LDAPSettings
wantErr bool
}{
{
name: "Empty LDAP Settings",
ldap: portainer.LDAPSettings{},
wantErr: true,
},
{
name: "With URL",
ldap: portainer.LDAPSettings{
AnonymousMode: true,
URL: "192.168.0.1:323",
},
wantErr: false,
},
{
name: "Validate URL and URLs",
ldap: portainer.LDAPSettings{
AnonymousMode: true,
URL: "192.168.0.1:323",
},
wantErr: false,
},
{
name: "validate client ldap",
ldap: portainer.LDAPSettings{
AnonymousMode: false,
ReaderDN: "CN=LDAP API Service Account",
Password: "Qu**dfUUU**",
URL: "aukdc15.pgc.co:389",
TLSConfig: portainer.TLSConfiguration{
TLS: false,
TLSSkipVerify: false,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateLDAPSettings(&tt.ldap)
if (err == nil) == tt.wantErr {
t.Errorf("No error expected but got %s", err)
}
})
}
}

17
api/datastore/validate/validationMethods.go Normal file
View file

@ -0,0 +1,17 @@
package validate
import (
"github.com/go-playground/validator/v10"
)
func registerValidationMethods(v *validator.Validate) {
v.RegisterValidation("validate_bool", ValidateBool)
}
/**
* Validation methods below are being used for custom validation
*/
func ValidateBool(fl validator.FieldLevel) bool {
_, ok := fl.Field().Interface().(bool)
return ok
}