1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-24 15:59:41 +02:00

refactor(edge/updates): sync changes from EE [EE-4288] (#7726)

This commit is contained in:
Chaim Lev-Ari 2022-12-01 08:40:52 +02:00 committed by GitHub
parent 4fee359247
commit 82e9e2a895
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
80 changed files with 1099 additions and 1892 deletions

View file

@ -3,20 +3,16 @@ package edgestacks
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/asaskevich/govalidator"
"github.com/pkg/errors"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/internal/edge"
"github.com/asaskevich/govalidator"
"github.com/pkg/errors"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
)
type InvalidPayloadError struct {
@ -47,8 +43,14 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
if err != nil {
return httperror.BadRequest("Invalid query parameter: method", err)
}
dryrun, _ := request.RetrieveBooleanQueryParameter(r, "dryrun", true)
edgeStack, err := handler.createSwarmStack(method, r)
tokenData, err := security.RetrieveTokenData(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve user details from authentication token", err)
}
edgeStack, err := handler.createSwarmStack(method, dryrun, tokenData.ID, r)
if err != nil {
var payloadError *InvalidPayloadError
switch {
@ -62,14 +64,15 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
return response.JSON(w, edgeStack)
}
func (handler *Handler) createSwarmStack(method string, r *http.Request) (*portainer.EdgeStack, error) {
func (handler *Handler) createSwarmStack(method string, dryrun bool, userID portainer.UserID, r *http.Request) (*portainer.EdgeStack, error) {
switch method {
case "string":
return handler.createSwarmStackFromFileContent(r)
return handler.createSwarmStackFromFileContent(r, dryrun)
case "repository":
return handler.createSwarmStackFromGitRepository(r)
return handler.createSwarmStackFromGitRepository(r, dryrun, userID)
case "file":
return handler.createSwarmStackFromFileUpload(r)
return handler.createSwarmStackFromFileUpload(r, dryrun)
}
return nil, errors.New("Invalid value for query parameter: method. Value must be one of: string, repository or file")
}
@ -82,10 +85,13 @@ type swarmStackFromFileContentPayload struct {
// List of identifiers of EdgeGroups
EdgeGroups []portainer.EdgeGroupID `example:"1"`
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes'
// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
// kubernetes deploy type is enabled only for kubernetes environments(endpoints)
// nomad deploy type is enabled only for nomad environments(endpoints)
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
// List of Registries to use for this stack
Registries []portainer.RegistryID
}
func (payload *swarmStackFromFileContentPayload) Validate(r *http.Request) error {
@ -101,85 +107,69 @@ func (payload *swarmStackFromFileContentPayload) Validate(r *http.Request) error
return nil
}
func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*portainer.EdgeStack, error) {
func (handler *Handler) createSwarmStackFromFileContent(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) {
var payload swarmStackFromFileContentPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return nil, err
}
err = handler.validateUniqueName(payload.Name)
stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "failed to create Edge stack object")
}
stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
stack := &portainer.EdgeStack{
ID: portainer.EdgeStackID(stackID),
Name: payload.Name,
DeploymentType: payload.DeploymentType,
CreationDate: time.Now().Unix(),
EdgeGroups: payload.EdgeGroups,
Status: make(map[portainer.EndpointID]portainer.EdgeStackStatus),
Version: 1,
if dryrun {
return stack, nil
}
relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
if err != nil {
return nil, fmt.Errorf("unable to find environment relations in database: %w", err)
}
return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, []byte(payload.StackFileContent))
})
relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
if err != nil {
return nil, fmt.Errorf("unable to persist environment relation in database: %w", err)
}
}
stackFolder := strconv.Itoa(int(stack.ID))
if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
stack.EntryPoint = filesystem.ComposeFileDefaultName
func (handler *Handler) storeFileContent(stackFolder string, deploymentType portainer.EdgeStackDeploymentType, relatedEndpointIds []portainer.EndpointID, fileContent []byte) (composePath, manifestPath, projectPath string, err error) {
if deploymentType == portainer.EdgeStackDeploymentCompose {
composePath = filesystem.ComposeFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, composePath, fileContent)
if err != nil {
return nil, err
return "", "", "", err
}
stack.ProjectPath = projectPath
err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
manifestPath, err = handler.convertAndStoreKubeManifestIfNeeded(stackFolder, projectPath, composePath, relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
return "", "", "", fmt.Errorf("Failed creating and storing kube manifest: %w", err)
}
} else {
hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("unable to check for existence of docker endpoint: %w", err)
}
return composePath, manifestPath, projectPath, nil
if hasDockerEndpoint {
return nil, fmt.Errorf("edge stack with docker endpoint cannot be deployed with kubernetes config")
}
stack.ManifestPath = filesystem.ManifestFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
if err != nil {
return nil, err
}
stack.ProjectPath = projectPath
}
err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
hasDockerEndpoint, err := hasDockerEndpoint(handler.DataStore.Endpoint(), relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Unable to update endpoint relations: %w", err)
return "", "", "", fmt.Errorf("unable to check for existence of docker environment: %w", err)
}
err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
if err != nil {
return nil, err
if hasDockerEndpoint {
return "", "", "", fmt.Errorf("edge stack with docker environment cannot be deployed with kubernetes or nomad config")
}
return stack, nil
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
manifestPath = filesystem.ManifestFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, manifestPath, fileContent)
if err != nil {
return "", "", "", err
}
return "", manifestPath, projectPath, nil
}
return "", "", "", fmt.Errorf("invalid deployment type: %d", deploymentType)
}
type swarmStackFromGitRepositoryPayload struct {
@ -200,10 +190,13 @@ type swarmStackFromGitRepositoryPayload struct {
// List of identifiers of EdgeGroups
EdgeGroups []portainer.EdgeGroupID `example:"1"`
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes'
// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1"`
// kubernetes deploy type is enabled only for kubernetes environments(endpoints)
// nomad deploy type is enabled only for nomad environments(endpoints)
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
// List of Registries to use for this stack
Registries []portainer.RegistryID
}
func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error {
@ -217,7 +210,12 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
return &InvalidPayloadError{msg: "Invalid repository credentials. Password must be specified when authentication is enabled"}
}
if govalidator.IsNull(payload.FilePathInRepository) {
payload.FilePathInRepository = filesystem.ComposeFileDefaultName
switch payload.DeploymentType {
case portainer.EdgeStackDeploymentCompose:
payload.FilePathInRepository = filesystem.ComposeFileDefaultName
case portainer.EdgeStackDeploymentKubernetes:
payload.FilePathInRepository = filesystem.ManifestFileDefaultName
}
}
if payload.EdgeGroups == nil || len(payload.EdgeGroups) == 0 {
return &InvalidPayloadError{msg: "Edge Groups are mandatory for an Edge stack"}
@ -225,83 +223,50 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err
return nil
}
func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*portainer.EdgeStack, error) {
func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request, dryrun bool, userID portainer.UserID) (*portainer.EdgeStack, error) {
var payload swarmStackFromGitRepositoryPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return nil, err
}
err = handler.validateUniqueName(payload.Name)
stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "failed to create edge stack object")
}
stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
stack := &portainer.EdgeStack{
ID: portainer.EdgeStackID(stackID),
Name: payload.Name,
CreationDate: time.Now().Unix(),
EdgeGroups: payload.EdgeGroups,
Status: make(map[portainer.EndpointID]portainer.EdgeStackStatus),
DeploymentType: payload.DeploymentType,
Version: 1,
if dryrun {
return stack, nil
}
projectPath := handler.FileService.GetEdgeStackProjectPath(strconv.Itoa(int(stack.ID)))
stack.ProjectPath = projectPath
repositoryUsername := payload.RepositoryUsername
repositoryPassword := payload.RepositoryPassword
if !payload.RepositoryAuthentication {
repositoryUsername = ""
repositoryPassword = ""
repoConfig := gittypes.RepoConfig{
URL: payload.RepositoryURL,
ReferenceName: payload.RepositoryReferenceName,
}
relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
if err != nil {
return nil, fmt.Errorf("failed fetching relations config: %w", err)
}
relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
if err != nil {
return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
}
err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
if err != nil {
return nil, err
}
if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
stack.EntryPoint = payload.FilePathInRepository
err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
if payload.RepositoryAuthentication {
repoConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
}
} else {
stack.ManifestPath = payload.FilePathInRepository
}
err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Unable to update environment relations: %w", err)
}
err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
if err != nil {
return nil, err
}
return stack, nil
return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeManifestFromGitRepository(stackFolder, relatedEndpointIds, payload.DeploymentType, userID, repoConfig)
})
}
type swarmStackFromFileUploadPayload struct {
Name string
StackFileContent []byte
EdgeGroups []portainer.EdgeGroupID
DeploymentType portainer.EdgeStackDeploymentType
// Deployment type to deploy this stack
// Valid values are: 0 - 'compose', 1 - 'kubernetes', 2 - 'nomad'
// for compose stacks will use kompose to convert to kubernetes manifest for kubernetes environments(endpoints)
// kubernetes deploytype is enabled only for kubernetes environments(endpoints)
// nomad deploytype is enabled only for nomad environments(endpoints)
DeploymentType portainer.EdgeStackDeploymentType `example:"0" enums:"0,1,2"`
Registries []portainer.RegistryID
}
func (payload *swarmStackFromFileUploadPayload) Validate(r *http.Request) error {
@ -330,109 +295,67 @@ func (payload *swarmStackFromFileUploadPayload) Validate(r *http.Request) error
}
payload.DeploymentType = portainer.EdgeStackDeploymentType(deploymentType)
var registries []portainer.RegistryID
request.RetrieveMultiPartFormJSONValue(r, "Registries", &registries, false)
if err != nil {
return errors.New("Invalid registry type")
}
payload.Registries = registries
return nil
}
func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portainer.EdgeStack, error) {
func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request, dryrun bool) (*portainer.EdgeStack, error) {
payload := &swarmStackFromFileUploadPayload{}
err := payload.Validate(r)
if err != nil {
return nil, err
}
err = handler.validateUniqueName(payload.Name)
stack, err := handler.edgeStacksService.BuildEdgeStack(payload.Name, payload.DeploymentType, payload.EdgeGroups, payload.Registries)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "failed to create edge stack object")
}
stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
stack := &portainer.EdgeStack{
ID: portainer.EdgeStackID(stackID),
Name: payload.Name,
DeploymentType: payload.DeploymentType,
CreationDate: time.Now().Unix(),
EdgeGroups: payload.EdgeGroups,
Status: make(map[portainer.EndpointID]portainer.EdgeStackStatus),
Version: 1,
if dryrun {
return stack, nil
}
relationConfig, err := fetchEndpointRelationsConfig(handler.DataStore)
if err != nil {
return nil, fmt.Errorf("failed fetching relations config: %w", err)
}
relatedEndpointIds, err := edge.EdgeStackRelatedEndpoints(stack.EdgeGroups, relationConfig.endpoints, relationConfig.endpointGroups, relationConfig.edgeGroups)
if err != nil {
return nil, fmt.Errorf("unable to retrieve related endpoints: %w", err)
}
stackFolder := strconv.Itoa(int(stack.ID))
if stack.DeploymentType == portainer.EdgeStackDeploymentCompose {
stack.EntryPoint = filesystem.ComposeFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
if err != nil {
return nil, err
}
stack.ProjectPath = projectPath
err = handler.convertAndStoreKubeManifestIfNeeded(stack, relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Failed creating and storing kube manifest: %w", err)
}
} else {
stack.ManifestPath = filesystem.ManifestFileDefaultName
projectPath, err := handler.FileService.StoreEdgeStackFileFromBytes(stackFolder, stack.ManifestPath, []byte(payload.StackFileContent))
if err != nil {
return nil, err
}
stack.ProjectPath = projectPath
}
err = updateEndpointRelations(handler.DataStore.EndpointRelation(), stack.ID, relatedEndpointIds)
if err != nil {
return nil, fmt.Errorf("Unable to update environment relations: %w", err)
}
err = handler.DataStore.EdgeStack().Create(stack.ID, stack)
if err != nil {
return nil, err
}
return stack, nil
return handler.edgeStacksService.PersistEdgeStack(stack, func(stackFolder string, relatedEndpointIds []portainer.EndpointID) (composePath string, manifestPath string, projectPath string, err error) {
return handler.storeFileContent(stackFolder, payload.DeploymentType, relatedEndpointIds, payload.StackFileContent)
})
}
func (handler *Handler) validateUniqueName(name string) error {
edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
func (handler *Handler) storeManifestFromGitRepository(stackFolder string, relatedEndpointIds []portainer.EndpointID, deploymentType portainer.EdgeStackDeploymentType, currentUserID portainer.UserID, repositoryConfig gittypes.RepoConfig) (composePath, manifestPath, projectPath string, err error) {
projectPath = handler.FileService.GetEdgeStackProjectPath(stackFolder)
repositoryUsername := ""
repositoryPassword := ""
if repositoryConfig.Authentication != nil {
if repositoryConfig.Authentication.Password != "" {
repositoryUsername = repositoryConfig.Authentication.Username
repositoryPassword = repositoryConfig.Authentication.Password
}
}
err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword)
if err != nil {
return err
return "", "", "", err
}
for _, stack := range edgeStacks {
if strings.EqualFold(stack.Name, name) {
return errors.New("Edge stack name must be unique")
}
}
return nil
}
if deploymentType == portainer.EdgeStackDeploymentCompose {
composePath := repositoryConfig.ConfigFilePath
// updateEndpointRelations adds a relation between the Edge Stack to the related environments(endpoints)
func updateEndpointRelations(endpointRelationService dataservices.EndpointRelationService, edgeStackID portainer.EdgeStackID, relatedEndpointIds []portainer.EndpointID) error {
for _, endpointID := range relatedEndpointIds {
relation, err := endpointRelationService.EndpointRelation(endpointID)
manifestPath, err := handler.convertAndStoreKubeManifestIfNeeded(stackFolder, projectPath, composePath, relatedEndpointIds)
if err != nil {
return fmt.Errorf("unable to find environment relation in database: %w", err)
return "", "", "", fmt.Errorf("Failed creating and storing kube manifest: %w", err)
}
relation.EdgeStacks[edgeStackID] = true
err = endpointRelationService.UpdateEndpointRelation(endpointID, relation)
if err != nil {
return fmt.Errorf("unable to persist environment relation in database: %w", err)
}
return composePath, manifestPath, projectPath, nil
}
return nil
if deploymentType == portainer.EdgeStackDeploymentKubernetes {
return "", repositoryConfig.ConfigFilePath, projectPath, nil
}
return "", "", "", fmt.Errorf("unknown deployment type: %d", deploymentType)
}