feat(edge): EE-4570 allow pre-pull images with edge stack deployment (#8210)

Co-authored-by: Matt Hook <hookenz@gmail.com>

api/datastore/migrate_data.go

@@ -81,6 +81,7 @@ func (store *Store) newMigratorParameters(version *models.Version) *migrator.Mig
 		FileService:             store.fileService,
 		DockerhubService:        store.DockerHubService,
 		AuthorizationService:    authorization.NewService(store),
+		EdgeStackService:        store.EdgeStackService,
 	}
 }
 

api/datastore/migrate_legacyversion.go

@@ -40,6 +40,7 @@ var dbVerToSemVerMap = map[int]string{
 	60: "2.15",
 	61: "2.15.1",
 	70: "2.16",
+	80: "2.17",
 }
 
 func dbVersionToSemanticVersion(dbVersion int) string {

api/datastore/migrator/migrate_ce.go

@@ -36,29 +36,29 @@ func (m *Migrator) Migrate() error {
 	if schemaVersion.Equal(apiVersion) {
 		// detect and run migrations when the versions are the same.
 		// e.g. development builds
-		latestMigrations := m.latestMigrations()
-		if latestMigrations.version.Equal(schemaVersion) &&
-			version.MigratorCount != len(latestMigrations.migrationFuncs) {
-			err := runMigrations(latestMigrations.migrationFuncs)
+		latestMigrations := m.LatestMigrations()
+		if latestMigrations.Version.Equal(schemaVersion) &&
+			version.MigratorCount != len(latestMigrations.MigrationFuncs) {
+			err := runMigrations(latestMigrations.MigrationFuncs)
 			if err != nil {
 				return err
 			}
-			newMigratorCount = len(latestMigrations.migrationFuncs)
+			newMigratorCount = len(latestMigrations.MigrationFuncs)
 		}
 	} else {
 		// regular path when major/minor/patch versions differ
 		for _, migration := range m.migrations {
-			if schemaVersion.LessThan(migration.version) {
+			if schemaVersion.LessThan(migration.Version) {
 
-				log.Info().Msgf("migrating data to %s", migration.version.String())
-				err := runMigrations(migration.migrationFuncs)
+				log.Info().Msgf("migrating data to %s", migration.Version.String())
+				err := runMigrations(migration.MigrationFuncs)
 				if err != nil {
 					return err
 				}
 			}
 
-			if apiVersion.Equal(migration.version) {
-				newMigratorCount = len(migration.migrationFuncs)
+			if apiVersion.Equal(migration.Version) {
+				newMigratorCount = len(migration.MigrationFuncs)
 			}
 		}
 	}
@@ -107,9 +107,9 @@ func (m *Migrator) NeedsMigration() bool {
 	}
 
 	// Check if we have any migrations for the current version
-	latestMigrations := m.latestMigrations()
-	if latestMigrations.version.Equal(semver.MustParse(portainer.APIVersion)) {
-		if m.currentDBVersion.MigratorCount != len(latestMigrations.migrationFuncs) {
+	latestMigrations := m.LatestMigrations()
+	if latestMigrations.Version.Equal(semver.MustParse(portainer.APIVersion)) {
+		if m.currentDBVersion.MigratorCount != len(latestMigrations.MigrationFuncs) {
 			return true
 		}
 	} else {

api/datastore/migrator/migrate_dbversion80.go

@@ -0,0 +1,47 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+func (m *Migrator) migrateDBVersionToDB80() error {
+	return m.updateEdgeStackStatusForDB80()
+}
+
+func (m *Migrator) updateEdgeStackStatusForDB80() error {
+	log.Info().Msg("transfer type field to details field for edge stack status")
+
+	edgeStacks, err := m.edgeStackService.EdgeStacks()
+	if err != nil {
+		return err
+	}
+
+	for _, edgeStack := range edgeStacks {
+		for endpointId, status := range edgeStack.Status {
+			switch status.Type {
+			case portainer.EdgeStackStatusPending:
+				status.Details.Pending = true
+			case portainer.EdgeStackStatusOk:
+				status.Details.Ok = true
+			case portainer.EdgeStackStatusError:
+				status.Details.Error = true
+			case portainer.EdgeStackStatusAcknowledged:
+				status.Details.Acknowledged = true
+			case portainer.EdgeStackStatusRemove:
+				status.Details.Remove = true
+			case portainer.EdgeStackStatusRemoteUpdateSuccess:
+				status.Details.RemoteUpdateSuccess = true
+			}
+
+			edgeStack.Status[endpointId] = status
+		}
+
+		err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

api/datastore/migrator/migrator.go

@@ -3,6 +3,8 @@ package migrator
 import (
 	"errors"
 
+	"github.com/portainer/portainer/api/dataservices/edgestack"
+
 	"github.com/Masterminds/semver"
 	"github.com/rs/zerolog/log"
 
@@ -53,6 +55,7 @@ type (
 		fileService             portainer.FileService
 		authorizationService    *authorization.Service
 		dockerhubService        *dockerhub.Service
+		edgeStackService        *edgestack.Service
 	}
 
 	// MigratorParameters represents the required parameters to create a new Migrator instance.
@@ -77,6 +80,7 @@ type (
 		FileService             portainer.FileService
 		AuthorizationService    *authorization.Service
 		DockerhubService        *dockerhub.Service
+		EdgeStackService        *edgestack.Service
 	}
 )
 
@@ -103,6 +107,7 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
 		fileService:             parameters.FileService,
 		authorizationService:    parameters.AuthorizationService,
 		dockerhubService:        parameters.DockerhubService,
+		edgeStackService:        parameters.EdgeStackService,
 	}
 
 	migrator.initMigrations()
@@ -128,12 +133,12 @@ func (m *Migrator) CurrentSemanticDBVersion() *semver.Version {
 
 func (m *Migrator) addMigrations(v string, funcs ...func() error) {
 	m.migrations = append(m.migrations, Migrations{
-		version:        semver.MustParse(v),
-		migrationFuncs: funcs,
+		Version:        semver.MustParse(v),
+		MigrationFuncs: funcs,
 	})
 }
 
-func (m *Migrator) latestMigrations() Migrations {
+func (m *Migrator) LatestMigrations() Migrations {
 	return m.migrations[len(m.migrations)-1]
 }
 
@@ -146,8 +151,8 @@ func (m *Migrator) latestMigrations() Migrations {
 // !      This increases the migration funcs count and so they all run again.
 
 type Migrations struct {
-	version        *semver.Version
-	migrationFuncs MigrationFuncs
+	Version        *semver.Version
+	MigrationFuncs MigrationFuncs
 }
 
 type MigrationFuncs []func() error
@@ -199,6 +204,7 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.15", m.migrateDBVersionToDB60)
 	m.addMigrations("2.16", m.migrateDBVersionToDB70)
 	m.addMigrations("2.16.1", m.migrateDBVersionToDB71)
+	m.addMigrations("2.17", m.migrateDBVersionToDB80)
 
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.

api/datastore/test_data/output_24_to_latest.json

@@ -931,6 +931,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.17.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/http/handler/edgestacks/edgestack_status_update.go

@@ -27,7 +27,7 @@ func (payload *updateStatusPayload) Validate(r *http.Request) error {
 		return errors.New("Invalid EnvironmentID")
 	}
 
-	if *payload.Status == portainer.StatusError && govalidator.IsNull(payload.Error) {
+	if *payload.Status == portainer.EdgeStackStatusError && govalidator.IsNull(payload.Error) {
 		return errors.New("Error message is mandatory when status is error")
 	}
 
@@ -74,8 +74,24 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 	var stack portainer.EdgeStack
 
 	err = handler.DataStore.EdgeStack().UpdateEdgeStackFunc(portainer.EdgeStackID(stackID), func(edgeStack *portainer.EdgeStack) {
+		details := edgeStack.Status[payload.EndpointID].Details
+		details.Pending = false
+
+		switch *payload.Status {
+		case portainer.EdgeStackStatusOk:
+			details.Ok = true
+		case portainer.EdgeStackStatusError:
+			details.Error = true
+		case portainer.EdgeStackStatusAcknowledged:
+			details.Acknowledged = true
+		case portainer.EdgeStackStatusRemove:
+			details.Remove = true
+		case portainer.EdgeStackStatusImagesPulled:
+			details.ImagesPulled = true
+		}
+
 		edgeStack.Status[payload.EndpointID] = portainer.EdgeStackStatus{
-			Type:       *payload.Status,
+			Details:    details,
 			Error:      payload.Error,
 			EndpointID: payload.EndpointID,
 		}

api/http/handler/edgestacks/edgestack_test.go

@@ -149,7 +149,7 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
 		ID:   edgeStackID,
 		Name: "test-edge-stack-" + strconv.Itoa(int(edgeStackID)),
 		Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
-			endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpointID},
+			endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpointID},
 		},
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{edgeGroup.ID},
@@ -775,7 +775,7 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 	edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
 
 	// Update edge stack status
-	newStatus := portainer.StatusError
+	newStatus := portainer.EdgeStackStatusError
 	payload := updateStatusPayload{
 		Error:      "test-error",
 		Status:     &newStatus,
@@ -821,8 +821,8 @@ func TestUpdateStatusAndInspect(t *testing.T) {
 		t.Fatal("error decoding response:", err)
 	}
 
-	if data.Status[endpoint.ID].Type != *payload.Status {
-		t.Fatalf("expected EdgeStackStatusType %d, found %d", payload.Status, data.Status[endpoint.ID].Type)
+	if !data.Status[endpoint.ID].Details.Error {
+		t.Fatalf("expected EdgeStackStatusType %d, found %t", payload.Status, data.Status[endpoint.ID].Details.Error)
 	}
 
 	if data.Status[endpoint.ID].Error != payload.Error {
@@ -841,8 +841,8 @@ func TestUpdateStatusWithInvalidPayload(t *testing.T) {
 	edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
 
 	// Update edge stack status
-	statusError := portainer.StatusError
-	statusOk := portainer.StatusOk
+	statusError := portainer.EdgeStackStatusError
+	statusOk := portainer.EdgeStackStatusOk
 	cases := []struct {
 		Name                 string
 		Payload              updateStatusPayload

api/http/handler/edgestacks/edgestack_update.go

@@ -193,6 +193,9 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
 	}
 
+	stack.NumDeployments = len(relatedEndpointIds)
+	stack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
+
 	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
 	if err != nil {
 		return httperror.InternalServerError("Unable to persist the stack changes inside the database", err)

api/http/handler/endpointedge/endpointedge_status_inspect_test.go

@@ -324,7 +324,7 @@ func TestEdgeStackStatus(t *testing.T) {
 		ID:   edgeStackID,
 		Name: "test-edge-stack-17",
 		Status: map[portainer.EndpointID]portainer.EdgeStackStatus{
-			endpointID: {Type: portainer.StatusOk, Error: "", EndpointID: endpoint.ID},
+			endpointID: {Details: portainer.EdgeStackStatusDetails{Ok: true}, Error: "", EndpointID: endpoint.ID},
 		},
 		CreationDate:   time.Now().Unix(),
 		EdgeGroups:     []portainer.EdgeGroupID{1, 2},

api/internal/edge/edgestacks/service.go

@@ -92,6 +92,7 @@ func (service *Service) PersistEdgeStack(
 	stack.ManifestPath = manifestPath
 	stack.ProjectPath = projectPath
 	stack.EntryPoint = composePath
+	stack.NumDeployments = len(relatedEndpointIds)
 
 	err = service.updateEndpointRelations(stack.ID, relatedEndpointIds)
 	if err != nil {

api/portainer.go

@@ -283,6 +283,7 @@ type (
 		ProjectPath    string                         `json:"ProjectPath"`
 		EntryPoint     string                         `json:"EntryPoint"`
 		Version        int                            `json:"Version"`
+		NumDeployments int                            `json:"NumDeployments"`
 		ManifestPath   string
 		DeploymentType EdgeStackDeploymentType
 		// Uses the manifest's namespaces instead of the default one
@@ -297,11 +298,24 @@ type (
 	//EdgeStackID represents an edge stack id
 	EdgeStackID int
 
+	EdgeStackStatusDetails struct {
+		Pending             bool
+		Ok                  bool
+		Error               bool
+		Acknowledged        bool
+		Remove              bool
+		RemoteUpdateSuccess bool
+		ImagesPulled        bool
+	}
+
 	//EdgeStackStatus represents an edge stack status
 	EdgeStackStatus struct {
-		Type       EdgeStackStatusType `json:"Type"`
-		Error      string              `json:"Error"`
-		EndpointID EndpointID          `json:"EndpointID"`
+		Details    EdgeStackStatusDetails `json:"Details"`
+		Error      string                 `json:"Error"`
+		EndpointID EndpointID             `json:"EndpointID"`
+
+		// Deprecated
+		Type EdgeStackStatusType `json:"Type"`
 	}
 
 	//EdgeStackStatusType represents an edge stack status type
@@ -1558,13 +1572,20 @@ const (
 )
 
 const (
-	_ EdgeStackStatusType = iota
-	//StatusOk represents a successfully deployed edge stack
-	StatusOk
-	//StatusError represents an edge environment(endpoint) which failed to deploy its edge stack
-	StatusError
-	//StatusAcknowledged represents an acknowledged edge stack
-	StatusAcknowledged
+	// EdgeStackStatusPending represents a pending edge stack
+	EdgeStackStatusPending EdgeStackStatusType = iota
+	//EdgeStackStatusOk represents a successfully deployed edge stack
+	EdgeStackStatusOk
+	//EdgeStackStatusError represents an edge environment(endpoint) which failed to deploy its edge stack
+	EdgeStackStatusError
+	//EdgeStackStatusAcknowledged represents an acknowledged edge stack
+	EdgeStackStatusAcknowledged
+	//EdgeStackStatusRemove represents a removed edge stack (status isn't persisted)
+	EdgeStackStatusRemove
+	// StatusRemoteUpdateSuccess represents a successfully updated edge stack
+	EdgeStackStatusRemoteUpdateSuccess
+	// EdgeStackStatusImagesPulled represents a successfully images-pulling
+	EdgeStackStatusImagesPulled
 )
 
 const (