feat(edge/stacks): sync EE codechanges [EE-498] (#8580)

api/http/errors/tx.go

@@ -0,0 +1,20 @@
+package errors
+
+import (
+	"errors"
+
+	httperror "github.com/portainer/libhttp/error"
+)
+
+func TxResponse(err error, validResponse func() *httperror.HandlerError) *httperror.HandlerError {
+	if err != nil {
+		var handlerError *httperror.HandlerError
+		if errors.As(err, &handlerError) {
+			return handlerError
+		}
+
+		return httperror.InternalServerError("Unexpected error", err)
+	}
+
+	return validResponse()
+}

api/http/handler/edgestacks/edgestack_create.go

@@ -38,6 +38,8 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 		switch {
 		case httperrors.IsInvalidPayloadError(err):
 			return httperror.BadRequest("Invalid payload", err)
+		case httperrors.IsConflictError(err):
+			return httperror.NewError(http.StatusConflict, err.Error(), err)
 		default:
 			return httperror.InternalServerError("Unable to create Edge stack", err)
 		}

api/http/handler/edgestacks/edgestack_create_string.go

@@ -105,7 +105,6 @@ func (handler *Handler) storeFileContent(tx dataservices.DataStoreTx, stackFolde
 		}
 
 		return composePath, "", projectPath, nil
-
 	}
 
 	if deploymentType == portainer.EdgeStackDeploymentKubernetes {

api/http/handler/edgestacks/edgestack_create_test.go

@@ -144,7 +144,7 @@ func TestCreateWithInvalidPayload(t *testing.T) {
 				DeploymentType:   edgeStack.DeploymentType,
 			},
 			Method:             "string",
-			ExpectedStatusCode: 500,
+			ExpectedStatusCode: http.StatusConflict,
 		},
 		{
 			Name: "Empty EdgeStack Groups",

api/http/handler/edgestacks/edgestack_update.go

@@ -1,10 +1,10 @@
 package edgestacks
 
 import (
-	"errors"
 	"net/http"
 	"strconv"
 
+	"github.com/pkg/errors"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
@@ -12,7 +12,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/internal/edge"
-	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/internal/set"
 	"github.com/portainer/portainer/pkg/featureflags"
 
 	"github.com/rs/zerolog/log"
@@ -20,7 +20,7 @@ import (
 
 type updateEdgeStackPayload struct {
 	StackFileContent string
-	Version          *int
+	UpdateVersion    bool
 	EdgeGroups       []portainer.EdgeGroupID
 	DeploymentType   portainer.EdgeStackDeploymentType
 	// Uses the manifest's namespaces instead of the default one
@@ -104,72 +104,32 @@ func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID por
 		return nil, httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
 	}
 
-	endpointsToAdd := map[portainer.EndpointID]bool{}
-
+	groupsIds := stack.EdgeGroups
 	if payload.EdgeGroups != nil {
-		newRelated, err := edge.EdgeStackRelatedEndpoints(payload.EdgeGroups, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
+		newRelated, _, err := handler.handleChangeEdgeGroups(tx, stack.ID, payload.EdgeGroups, relatedEndpointIds, relationConfig)
 		if err != nil {
-			return nil, httperror.InternalServerError("Unable to retrieve edge stack related environments from database", err)
+			return nil, httperror.InternalServerError("Unable to handle edge groups change", err)
 		}
 
-		oldRelatedSet := endpointutils.EndpointSet(relatedEndpointIds)
-		newRelatedSet := endpointutils.EndpointSet(newRelated)
-
-		endpointsToRemove := map[portainer.EndpointID]bool{}
-		for endpointID := range oldRelatedSet {
-			if !newRelatedSet[endpointID] {
-				endpointsToRemove[endpointID] = true
-			}
-		}
-
-		for endpointID := range endpointsToRemove {
-			relation, err := tx.EndpointRelation().EndpointRelation(endpointID)
-			if err != nil {
-				return nil, httperror.InternalServerError("Unable to find environment relation in database", err)
-			}
-
-			delete(relation.EdgeStacks, stack.ID)
-
-			err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
-			if err != nil {
-				return nil, httperror.InternalServerError("Unable to persist environment relation in database", err)
-			}
-		}
-
-		for endpointID := range newRelatedSet {
-			if !oldRelatedSet[endpointID] {
-				endpointsToAdd[endpointID] = true
-			}
-		}
-
-		for endpointID := range endpointsToAdd {
-			relation, err := tx.EndpointRelation().EndpointRelation(endpointID)
-			if err != nil {
-				return nil, httperror.InternalServerError("Unable to find environment relation in database", err)
-			}
-
-			relation.EdgeStacks[stack.ID] = true
-
-			err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
-			if err != nil {
-				return nil, httperror.InternalServerError("Unable to persist environment relation in database", err)
-			}
-		}
-
-		stack.EdgeGroups = payload.EdgeGroups
+		groupsIds = payload.EdgeGroups
 		relatedEndpointIds = newRelated
+
 	}
 
-	if stack.DeploymentType != payload.DeploymentType {
+	entryPoint := stack.EntryPoint
+	manifestPath := stack.ManifestPath
+	deploymentType := stack.DeploymentType
+
+	if deploymentType != payload.DeploymentType {
 		// deployment type was changed - need to delete the old file
 		err = handler.FileService.RemoveDirectory(stack.ProjectPath)
 		if err != nil {
 			log.Warn().Err(err).Msg("Unable to clear old files")
 		}
 
-		stack.EntryPoint = ""
-		stack.ManifestPath = ""
-		stack.DeploymentType = payload.DeploymentType
+		entryPoint = ""
+		manifestPath = ""
+		deploymentType = payload.DeploymentType
 	}
 
 	stackFolder := strconv.Itoa(int(stack.ID))
@@ -183,52 +143,106 @@ func (handler *Handler) updateEdgeStack(tx dataservices.DataStoreTx, stackID por
 	}
 
 	if payload.DeploymentType == portainer.EdgeStackDeploymentCompose {
-		if stack.EntryPoint == "" {
-			stack.EntryPoint = filesystem.ComposeFileDefaultName
+		if entryPoint == "" {
+			entryPoint = filesystem.ComposeFileDefaultName
 		}
 
-		_, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
+		_, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, entryPoint, []byte(payload.StackFileContent))
 		if err != nil {
 			return nil, httperror.InternalServerError("Unable to persist updated Compose file on disk", err)
 		}
 
-		manifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, stack.ProjectPath, stack.EntryPoint, relatedEndpointIds)
+		tempManifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, stack.ProjectPath, entryPoint, relatedEndpointIds)
 		if err != nil {
 			return nil, httperror.InternalServerError("Unable to convert and persist updated Kubernetes manifest file on disk", err)
 		}
 
-		stack.ManifestPath = manifestPath
+		manifestPath = tempManifestPath
 	}
 
-	if payload.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
-		if stack.ManifestPath == "" {
-			stack.ManifestPath = filesystem.ManifestFileDefaultName
+	if deploymentType == portainer.EdgeStackDeploymentKubernetes {
+		if manifestPath == "" {
+			manifestPath = filesystem.ManifestFileDefaultName
 		}
 
-		stack.UseManifestNamespaces = payload.UseManifestNamespaces
-
-		_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
+		_, err = handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, manifestPath, []byte(payload.StackFileContent))
 		if err != nil {
 			return nil, httperror.InternalServerError("Unable to persist updated Kubernetes manifest file on disk", err)
 		}
 	}
 
-	versionUpdated := payload.Version != nil && *payload.Version != stack.Version
-	if versionUpdated {
-		stack.Version = *payload.Version
-		stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
-	}
+	err = tx.EdgeStack().UpdateEdgeStackFunc(stack.ID, func(edgeStack *portainer.EdgeStack) {
+		edgeStack.NumDeployments = len(relatedEndpointIds)
+		if payload.UpdateVersion {
+			edgeStack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
+			edgeStack.Version++
+		}
 
-	stack.NumDeployments = len(relatedEndpointIds)
+		edgeStack.UseManifestNamespaces = payload.UseManifestNamespaces
 
-	if versionUpdated {
-		stack.Status = make(map[portainer.EndpointID]portainer.EdgeStackStatus)
-	}
+		edgeStack.DeploymentType = deploymentType
+		edgeStack.EntryPoint = entryPoint
+		edgeStack.ManifestPath = manifestPath
 
-	err = tx.EdgeStack().UpdateEdgeStack(stack.ID, stack)
+		edgeStack.EdgeGroups = groupsIds
+	})
 	if err != nil {
 		return nil, httperror.InternalServerError("Unable to persist the stack changes inside the database", err)
 	}
 
 	return stack, nil
 }
+
+func (handler *Handler) handleChangeEdgeGroups(tx dataservices.DataStoreTx, edgeStackID portainer.EdgeStackID, newEdgeGroupsIDs []portainer.EdgeGroupID, oldRelatedEnvironmentIDs []portainer.EndpointID, relationConfig *edge.EndpointRelationsConfig) ([]portainer.EndpointID, set.Set[portainer.EndpointID], error) {
+	newRelatedEnvironmentIDs, err := edge.EdgeStackRelatedEndpoints(newEdgeGroupsIDs, relationConfig.Endpoints, relationConfig.EndpointGroups, relationConfig.EdgeGroups)
+	if err != nil {
+		return nil, nil, errors.WithMessage(err, "Unable to retrieve edge stack related environments from database")
+	}
+
+	oldRelatedSet := set.ToSet(oldRelatedEnvironmentIDs)
+	newRelatedSet := set.ToSet(newRelatedEnvironmentIDs)
+
+	endpointsToRemove := set.Set[portainer.EndpointID]{}
+	for endpointID := range oldRelatedSet {
+		if !newRelatedSet[endpointID] {
+			endpointsToRemove[endpointID] = true
+		}
+	}
+
+	for endpointID := range endpointsToRemove {
+		relation, err := tx.EndpointRelation().EndpointRelation(endpointID)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to find environment relation in database")
+		}
+
+		delete(relation.EdgeStacks, edgeStackID)
+
+		err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to persist environment relation in database")
+		}
+	}
+
+	endpointsToAdd := set.Set[portainer.EndpointID]{}
+	for endpointID := range newRelatedSet {
+		if !oldRelatedSet[endpointID] {
+			endpointsToAdd[endpointID] = true
+		}
+	}
+
+	for endpointID := range endpointsToAdd {
+		relation, err := tx.EndpointRelation().EndpointRelation(endpointID)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to find environment relation in database")
+		}
+
+		relation.EdgeStacks[edgeStackID] = true
+
+		err = tx.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
+		if err != nil {
+			return nil, nil, errors.WithMessage(err, "Unable to persist environment relation in database")
+		}
+	}
+
+	return newRelatedEnvironmentIDs, endpointsToAdd, nil
+}

api/http/handler/edgestacks/edgestack_update_test.go

@@ -54,10 +54,9 @@ func TestUpdateAndInspect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	newVersion := 238
 	payload := updateEdgeStackPayload{
 		StackFileContent: "update-test",
-		Version:          &newVersion,
+		UpdateVersion:    true,
 		EdgeGroups:       append(edgeStack.EdgeGroups, newEdgeGroup.ID),
 		DeploymentType:   portainer.EdgeStackDeploymentCompose,
 	}
@@ -101,7 +100,7 @@ func TestUpdateAndInspect(t *testing.T) {
 		t.Fatal("error decoding response:", err)
 	}
 
-	if data.Version != *payload.Version {
+	if payload.UpdateVersion && data.Version != edgeStack.Version+1 {
 		t.Fatalf("expected EdgeStackID %d, found %d", edgeStack.Version, data.Version)
 	}
 
@@ -132,7 +131,6 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
 
 	handler.DataStore.EdgeGroup().Create(&newEdgeGroup)
 
-	newVersion := 238
 	cases := []struct {
 		Name               string
 		Payload            updateEdgeStackPayload
@@ -142,7 +140,7 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
 			"Update with non-existing EdgeGroupID",
 			updateEdgeStackPayload{
 				StackFileContent: "error-test",
-				Version:          &newVersion,
+				UpdateVersion:    true,
 				EdgeGroups:       []portainer.EdgeGroupID{9999},
 				DeploymentType:   edgeStack.DeploymentType,
 			},
@@ -152,7 +150,7 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
 			"Update with invalid EdgeGroup (non-existing Endpoint)",
 			updateEdgeStackPayload{
 				StackFileContent: "error-test",
-				Version:          &newVersion,
+				UpdateVersion:    true,
 				EdgeGroups:       []portainer.EdgeGroupID{2},
 				DeploymentType:   edgeStack.DeploymentType,
 			},
@@ -162,7 +160,7 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
 			"Update DeploymentType from Docker to Kubernetes",
 			updateEdgeStackPayload{
 				StackFileContent: "error-test",
-				Version:          &newVersion,
+				UpdateVersion:    true,
 				EdgeGroups:       []portainer.EdgeGroupID{1},
 				DeploymentType:   portainer.EdgeStackDeploymentKubernetes,
 			},
@@ -200,7 +198,6 @@ func TestUpdateWithInvalidPayload(t *testing.T) {
 	endpoint := createEndpoint(t, handler.DataStore)
 	edgeStack := createEdgeStack(t, handler.DataStore, endpoint.ID)
 
-	newVersion := 238
 	cases := []struct {
 		Name               string
 		Payload            updateEdgeStackPayload
@@ -210,7 +207,7 @@ func TestUpdateWithInvalidPayload(t *testing.T) {
 			"Update with empty StackFileContent",
 			updateEdgeStackPayload{
 				StackFileContent: "",
-				Version:          &newVersion,
+				UpdateVersion:    true,
 				EdgeGroups:       edgeStack.EdgeGroups,
 				DeploymentType:   edgeStack.DeploymentType,
 			},
@@ -220,7 +217,7 @@ func TestUpdateWithInvalidPayload(t *testing.T) {
 			"Update with empty EdgeGroups",
 			updateEdgeStackPayload{
 				StackFileContent: "error-test",
-				Version:          &newVersion,
+				UpdateVersion:    true,
 				EdgeGroups:       []portainer.EdgeGroupID{},
 				DeploymentType:   edgeStack.DeploymentType,
 			},

api/http/handler/edgestacks/handler.go

@@ -25,6 +25,8 @@ type Handler struct {
 	KubernetesDeployer portainer.KubernetesDeployer
 }
 
+const contextKey = "edgeStack_item"
+
 // NewHandler creates a handler to manage environment(endpoint) group operations.
 func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, edgeStacksService *edgestackservice.Service) *Handler {
 	h := &Handler{