diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 78ca69e17..5a9139dae 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -553,7 +553,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { } scheduler := scheduler.NewScheduler(shutdownCtx) - stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager) + stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer) stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService) return &http.Server{ diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index 4fb4d163e..28c4d8278 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -47,7 +47,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string { func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint) error { url, proxy, err := manager.fetchEndpointProxy(endpoint) if err != nil { - return errors.Wrap(err, "failed to fetch endpoint proxy") + return errors.Wrap(err, "failed to fetch environment proxy") } if proxy != nil { @@ -80,7 +80,7 @@ func (manager *ComposeStackManager) Down(ctx context.Context, stack *portainer.S } // NormalizeStackName returns a new stack name with unsupported characters replaced -func (w *ComposeStackManager) NormalizeStackName(name string) string { +func (manager *ComposeStackManager) NormalizeStackName(name string) string { r := regexp.MustCompile("[^a-z0-9]+") return r.ReplaceAllString(strings.ToLower(name), "") } diff --git a/api/exec/exectest/kubernetes_mocks.go b/api/exec/exectest/kubernetes_mocks.go index 2809df2c5..cabc4a716 100644 --- a/api/exec/exectest/kubernetes_mocks.go +++ b/api/exec/exectest/kubernetes_mocks.go @@ -1,8 +1,6 @@ package exectest import ( - "net/http" - portainer "github.com/portainer/portainer/api" ) @@ -12,7 +10,11 @@ func NewKubernetesDeployer() portainer.KubernetesDeployer { return &kubernetesMockDeployer{} } -func (deployer *kubernetesMockDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, data string, namespace string) (string, error) { +func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return "", nil +} + +func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { return "", nil } diff --git a/api/exec/kubernetes_deploy.go b/api/exec/kubernetes_deploy.go index af97af174..9759ef70c 100644 --- a/api/exec/kubernetes_deploy.go +++ b/api/exec/kubernetes_deploy.go @@ -3,7 +3,6 @@ package exec import ( "bytes" "fmt" - "net/http" "os/exec" "path" "runtime" @@ -13,7 +12,6 @@ import ( "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" "github.com/portainer/portainer/api/http/proxy/factory/kubernetes" - "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/kubernetes/cli" portainer "github.com/portainer/portainer/api" @@ -43,12 +41,7 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan } } -func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) { - tokenData, err := security.RetrieveTokenData(request) - if err != nil { - return "", err - } - +func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) { kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint) if err != nil { return "", err @@ -61,11 +54,16 @@ func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *po return "", err } - if tokenData.Role == portainer.AdministratorRole { + user, err := deployer.dataStore.User().User(userID) + if err != nil { + return "", errors.Wrap(err, "failed to fetch the user") + } + + if user.Role == portainer.AdministratorRole { return tokenManager.GetAdminServiceAccountToken(), nil } - token, err := tokenManager.GetUserServiceAccountToken(int(tokenData.ID), endpoint.ID) + token, err := tokenManager.GetUserServiceAccountToken(int(user.ID), endpoint.ID) if err != nil { return "", err } @@ -76,15 +74,31 @@ func (deployer *KubernetesDeployer) getToken(request *http.Request, endpoint *po return token, nil } -// Deploy will deploy a Kubernetes manifest inside an optional namespace in a Kubernetes environment(endpoint). -// Otherwise it will use kubectl to deploy the manifest. -func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) { +// Deploy upserts Kubernetes resources defined in manifest(s) +func (deployer *KubernetesDeployer) Deploy(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return deployer.command("apply", userID, endpoint, manifestFiles, namespace) +} + +// Remove deletes Kubernetes resources defined in manifest(s) +func (deployer *KubernetesDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + return deployer.command("delete", userID, endpoint, manifestFiles, namespace) +} + +func (deployer *KubernetesDeployer) command(operation string, userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) { + token, err := deployer.getToken(userID, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment) + if err != nil { + return "", errors.Wrap(err, "failed generating a user token") + } + command := path.Join(deployer.binaryPath, "kubectl") if runtime.GOOS == "windows" { command = path.Join(deployer.binaryPath, "kubectl.exe") } - args := make([]string, 0) + args := []string{"--token", token} + if namespace != "" { + args = append(args, "--namespace", namespace) + } if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment { url, proxy, err := deployer.getAgentURL(endpoint) @@ -97,21 +111,18 @@ func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *port args = append(args, "--insecure-skip-tls-verify") } - token, err := deployer.getToken(request, endpoint, endpoint.Type == portainer.KubernetesLocalEnvironment) - if err != nil { - return "", err + if operation == "delete" { + args = append(args, "--ignore-not-found=true") } - args = append(args, "--token", token) - if namespace != "" { - args = append(args, "--namespace", namespace) + args = append(args, operation) + for _, path := range manifestFiles { + args = append(args, "-f", strings.TrimSpace(path)) } - args = append(args, "apply", "-f", "-") var stderr bytes.Buffer cmd := exec.Command(command, args...) cmd.Stderr = &stderr - cmd.Stdin = strings.NewReader(stackConfig) output, err := cmd.Output() if err != nil { diff --git a/api/filesystem/write.go b/api/filesystem/write.go new file mode 100644 index 000000000..235511933 --- /dev/null +++ b/api/filesystem/write.go @@ -0,0 +1,23 @@ +package filesystem + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +func WriteToFile(dst string, content []byte) error { + if err := os.MkdirAll(filepath.Dir(dst), 0744); err != nil { + return errors.Wrapf(err, "failed to create filestructure for the path %q", dst) + } + + file, err := os.Create(dst) + if err != nil { + return errors.Wrapf(err, "failed to open a file %q", dst) + } + defer file.Close() + + _, err = file.Write(content) + return errors.Wrapf(err, "failed to write a file %q", dst) +} diff --git a/api/filesystem/write_test.go b/api/filesystem/write_test.go new file mode 100644 index 000000000..89223a20e --- /dev/null +++ b/api/filesystem/write_test.go @@ -0,0 +1,48 @@ +package filesystem + +import ( + "io/ioutil" + "path" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_WriteFile_CanStoreContentInANewFile(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dummy") + + content := []byte("content") + err := WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} + +func Test_WriteFile_CanOverwriteExistingFile(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dummy") + + err := WriteToFile(tmpFilePath, []byte("content")) + assert.NoError(t, err) + + content := []byte("new content") + err = WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} + +func Test_WriteFile_CanWriteANestedPath(t *testing.T) { + tmpDir := t.TempDir() + tmpFilePath := path.Join(tmpDir, "dir", "sub-dir", "dummy") + + content := []byte("content") + err := WriteToFile(tmpFilePath, content) + assert.NoError(t, err) + + fileContent, _ := ioutil.ReadFile(tmpFilePath) + assert.Equal(t, content, fileContent) +} diff --git a/api/http/handler/helm/helm_install.go b/api/http/handler/helm/helm_install.go index 57e17825f..ab196d8c5 100644 --- a/api/http/handler/helm/helm_install.go +++ b/api/http/handler/helm/helm_install.go @@ -2,6 +2,7 @@ package helm import ( "fmt" + "io/ioutil" "net/http" "os" "strings" @@ -182,6 +183,11 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, return errors.Wrap(err, "unable to find an endpoint on request context") } + tokenData, err := security.RetrieveTokenData(r) + if err != nil { + return errors.Wrap(err, "unable to retrieve user details from authentication token") + } + // extract list of yaml resources from helm manifest yamlResources, err := kubernetes.ExtractDocuments(manifest, nil) if err != nil { @@ -193,6 +199,19 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, for _, resource := range yamlResources { resource := resource // https://golang.org/doc/faq#closures_and_goroutines g.Go(func() error { + tmpfile, err := ioutil.TempFile("", "helm-manifest-*") + if err != nil { + return errors.Wrap(err, "failed to create a tmp helm manifest file") + } + defer func() { + tmpfile.Close() + os.Remove(tmpfile.Name()) + }() + + if _, err := tmpfile.Write(resource); err != nil { + return errors.Wrap(err, "failed to write a tmp helm manifest file") + } + // get resource namespace, fallback to provided namespace if not explicit on resource resourceNamespace, err := kubernetes.GetNamespace(resource) if err != nil { @@ -201,7 +220,8 @@ func (handler *Handler) updateHelmAppManifest(r *http.Request, manifest []byte, if resourceNamespace == "" { resourceNamespace = namespace } - _, err = handler.kubernetesDeployer.Deploy(r, endpoint, string(resource), resourceNamespace) + + _, err = handler.kubernetesDeployer.Deploy(tokenData.ID, endpoint, []string{tmpfile.Name()}, resourceNamespace) return err }) } diff --git a/api/http/handler/stacks/autoupdate.go b/api/http/handler/stacks/autoupdate.go index 867e71fc2..237b0b58a 100644 --- a/api/http/handler/stacks/autoupdate.go +++ b/api/http/handler/stacks/autoupdate.go @@ -17,10 +17,8 @@ func startAutoupdate(stackID portainer.StackID, interval string, scheduler *sche return "", &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err} } - jobID = scheduler.StartJobEvery(d, func() { - if err := stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService); err != nil { - log.Printf("[ERROR] [http,stacks] [message: failed redeploying] [err: %s]\n", err) - } + jobID = scheduler.StartJobEvery(d, func() error { + return stacks.RedeployWhenChanged(stackID, stackDeployer, datastore, gitService) }) return jobID, nil diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index fe6d30c97..7f3615500 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -46,7 +46,7 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } @@ -152,7 +152,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite payload.ComposeFile = filesystem.ComposeFileDefaultName } - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } @@ -208,11 +208,11 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } - commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} } - stack.GitConfig.ConfigHash = commitId + stack.GitConfig.ConfigHash = commitID config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) if configErr != nil { @@ -281,7 +281,7 @@ func (handler *Handler) createComposeStackFromFileUpload(w http.ResponseWriter, payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, false) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index f4133ddae..654bda92f 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -2,9 +2,8 @@ package stacks import ( "fmt" - "io/ioutil" "net/http" - "path/filepath" + "os" "strconv" "time" @@ -19,16 +18,19 @@ import ( "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/client" + "github.com/portainer/portainer/api/internal/stackutils" k "github.com/portainer/portainer/api/kubernetes" ) type kubernetesStringDeploymentPayload struct { + StackName string ComposeFormat bool Namespace string StackFileContent string } type kubernetesGitDeploymentPayload struct { + StackName string ComposeFormat bool Namespace string RepositoryURL string @@ -36,10 +38,13 @@ type kubernetesGitDeploymentPayload struct { RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string - FilePathInRepository string + ManifestFile string + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate } type kubernetesManifestURLDeploymentPayload struct { + StackName string Namespace string ComposeFormat bool ManifestURL string @@ -52,6 +57,9 @@ func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) erro if govalidator.IsNull(payload.Namespace) { return errors.New("Invalid namespace") } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -65,12 +73,18 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error { if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") } - if govalidator.IsNull(payload.FilePathInRepository) { - return errors.New("Invalid file path in repository") + if govalidator.IsNull(payload.ManifestFile) { + return errors.New("Invalid manifest file in repository") } if govalidator.IsNull(payload.RepositoryReferenceName) { payload.RepositoryReferenceName = defaultGitReferenceName } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err + } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -78,6 +92,9 @@ func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request) if govalidator.IsNull(payload.ManifestURL) || !govalidator.IsURL(payload.ManifestURL) { return errors.New("Invalid manifest URL") } + if govalidator.IsNull(payload.StackName) { + return errors.New("Invalid stack name") + } return nil } @@ -95,6 +112,13 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ @@ -102,6 +126,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit Type: portainer.KubernetesStack, EndpointID: endpoint.ID, EntryPoint: filesystem.ManifestFileDefaultName, + Name: payload.StackName, Namespace: payload.Namespace, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), @@ -124,11 +149,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - output, err := handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "content", + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "content", }) if err != nil { @@ -140,12 +165,11 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err} } - doCleanUp = false - resp := &createKubernetesStackResponse{ Output: output, } + doCleanUp = false return response.JSON(w, resp) } @@ -159,23 +183,44 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ ID: portainer.StackID(stackID), Type: portainer.KubernetesStack, EndpointID: endpoint.ID, - EntryPoint: payload.FilePathInRepository, + EntryPoint: payload.ManifestFile, GitConfig: &gittypes.RepoConfig{ URL: payload.RepositoryURL, ReferenceName: payload.RepositoryReferenceName, - ConfigFilePath: payload.FilePathInRepository, + ConfigFilePath: payload.ManifestFile, }, Namespace: payload.Namespace, + Name: payload.StackName, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), CreatedBy: user.Username, IsComposeFormat: payload.ComposeFormat, + AutoUpdate: payload.AutoUpdate, + AdditionalFiles: payload.AdditionalFiles, } if payload.RepositoryAuthentication { @@ -197,33 +242,48 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr } stack.GitConfig.ConfigHash = commitID - stackFileContent, err := handler.cloneManifestContentFromGitRepo(&payload, stack.ProjectPath) - if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to process manifest from Git repository", Err: err} + repositoryUsername := payload.RepositoryUsername + repositoryPassword := payload.RepositoryPassword + if !payload.RepositoryAuthentication { + repositoryUsername = "" + repositoryPassword = "" } - output, err := handler.deployKubernetesStack(r, endpoint, stackFileContent, payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "git", + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to clone git repository", Err: err} + } + + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "git", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err} } + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + + stack.AutoUpdate.JobID = jobID + } + err = handler.DataStore.Stack().CreateStack(stack) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err} } - doCleanUp = false - resp := &createKubernetesStackResponse{ Output: output, } + doCleanUp = false return response.JSON(w, resp) } @@ -237,6 +297,13 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to load user information from the database", Err: err} } + isUnique, err := handler.checkUniqueStackName(endpoint, payload.StackName, 0) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists} + } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ @@ -245,6 +312,7 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit EndpointID: endpoint.ID, EntryPoint: filesystem.ManifestFileDefaultName, Namespace: payload.Namespace, + Name: payload.StackName, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), CreatedBy: user.Username, @@ -267,11 +335,11 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - output, err := handler.deployKubernetesStack(r, endpoint, string(manifestContent), payload.ComposeFormat, payload.Namespace, k.KubeAppLabels{ - StackID: stackID, - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "url", + output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{ + StackID: stackID, + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "url", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err} @@ -291,42 +359,14 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit return response.JSON(w, resp) } -func (handler *Handler) deployKubernetesStack(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string, appLabels k.KubeAppLabels) (string, error) { +func (handler *Handler) deployKubernetesStack(userID portainer.UserID, endpoint *portainer.Endpoint, stack *portainer.Stack, appLabels k.KubeAppLabels) (string, error) { handler.stackCreationMutex.Lock() defer handler.stackCreationMutex.Unlock() - manifest := []byte(stackConfig) - if composeFormat { - convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(manifest) - if err != nil { - return "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest") - } - manifest = convertedConfig - } - - manifest, err := k.AddAppLabels(manifest, appLabels.ToMap()) + manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, handler.KubernetesDeployer, appLabels) if err != nil { - return "", errors.Wrap(err, "failed to add application labels") + return "", errors.Wrap(err, "failed to create temp kub deployment files") } - - return handler.KubernetesDeployer.Deploy(request, endpoint, string(manifest), namespace) -} - -func (handler *Handler) cloneManifestContentFromGitRepo(gitInfo *kubernetesGitDeploymentPayload, projectPath string) (string, error) { - repositoryUsername := gitInfo.RepositoryUsername - repositoryPassword := gitInfo.RepositoryPassword - if !gitInfo.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" - } - - err := handler.GitService.CloneRepository(projectPath, gitInfo.RepositoryURL, gitInfo.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - return "", err - } - content, err := ioutil.ReadFile(filepath.Join(projectPath, gitInfo.FilePathInRepository)) - if err != nil { - return "", err - } - return string(content), nil + defer os.RemoveAll(tempDir) + return handler.KubernetesDeployer.Deploy(userID, endpoint, manifestFilePaths, stack.Namespace) } diff --git a/api/http/handler/stacks/create_kubernetes_stack_test.go b/api/http/handler/stacks/create_kubernetes_stack_test.go deleted file mode 100644 index 2bcd35ab5..000000000 --- a/api/http/handler/stacks/create_kubernetes_stack_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package stacks - -import ( - "io/ioutil" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" -) - -type git struct { - content string -} - -func (g *git) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error { - return g.ClonePublicRepository(repositoryURL, referenceName, destination) -} -func (g *git) ClonePublicRepository(repositoryURL string, referenceName string, destination string) error { - return ioutil.WriteFile(path.Join(destination, "deployment.yml"), []byte(g.content), 0755) -} -func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName string, destination, username, password string) error { - return g.ClonePublicRepository(repositoryURL, referenceName, destination) -} - -func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - return "", nil -} - -func TestCloneAndConvertGitRepoFile(t *testing.T) { - dir, err := os.MkdirTemp("", "kube-create-stack") - assert.NoError(t, err, "failed to create a tmp dir") - defer os.RemoveAll(dir) - - content := `apiVersion: apps/v1 - kind: Deployment - metadata: - name: nginx-deployment - labels: - app: nginx - spec: - replicas: 3 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx:1.14.2 - ports: - - containerPort: 80` - - h := &Handler{ - GitService: &git{ - content: content, - }, - } - gitInfo := &kubernetesGitDeploymentPayload{ - FilePathInRepository: "deployment.yml", - } - fileContent, err := h.cloneManifestContentFromGitRepo(gitInfo, dir) - assert.NoError(t, err, "failed to clone or convert the file from Git repo") - assert.Equal(t, content, fileContent) -} diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index 7e615acc3..e898b5502 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -51,7 +51,8 @@ func (handler *Handler) createSwarmStackFromFileContent(w http.ResponseWriter, r payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) + if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } @@ -161,7 +162,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } @@ -218,11 +219,11 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } - commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} } - stack.GitConfig.ConfigHash = commitId + stack.GitConfig.ConfigHash = commitID config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) if configErr != nil { @@ -298,7 +299,8 @@ func (handler *Handler) createSwarmStackFromFileUpload(w http.ResponseWriter, r payload.Name = handler.SwarmStackManager.NormalizeStackName(payload.Name) - isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, payload.Name, 0, true) + if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} } diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index f5ba2c983..8795e0fe3 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -127,7 +127,7 @@ func (handler *Handler) userCanCreateStack(securityContext *security.RestrictedR return handler.userIsAdminOrEndpointAdmin(user, endpointID) } -func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) { +func (handler *Handler) checkUniqueStackName(endpoint *portainer.Endpoint, name string, stackID portainer.StackID) (bool, error) { stacks, err := handler.DataStore.Stack().Stacks() if err != nil { return false, err @@ -139,6 +139,15 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin } } + return true, nil +} + +func (handler *Handler) checkUniqueStackNameInDocker(endpoint *portainer.Endpoint, name string, stackID portainer.StackID, swarmMode bool) (bool, error) { + isUniqueStackName, err := handler.checkUniqueStackName(endpoint, name, stackID) + if err != nil { + return false, err + } + dockerClient, err := handler.DockerClientFactory.CreateClient(endpoint, "") if err != nil { return false, err @@ -171,7 +180,7 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin } } - return true, nil + return isUniqueStackName, nil } func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) { diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index cd5e2cd26..799f6f612 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -2,15 +2,20 @@ package stacks import ( "context" - "errors" + "fmt" + "io/ioutil" "net/http" + "os" + "path" "strconv" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/filesystem" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" @@ -34,12 +39,12 @@ import ( func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } externalStack, _ := request.RetrieveBooleanQueryParameter(r, "external", true) @@ -49,52 +54,52 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt id, err := strconv.Atoi(stackID) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(id)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } isOrphaned := portainer.EndpointID(endpointID) != stack.EndpointID if isOrphaned && !securityContext.IsAdmin { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to remove orphaned stack", errors.New("Permission denied to remove orphaned stack")} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to remove orphaned stack", Err: errors.New("Permission denied to remove orphaned stack")} } endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } if !isOrphaned { err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} } if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } } } @@ -104,26 +109,26 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) } - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } err = handler.DataStore.Stack().DeleteStack(portainer.StackID(id)) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the stack from the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the stack from the database", Err: err} } if resourceControl != nil { err = handler.DataStore.ResourceControl().DeleteResourceControl(resourceControl.ID) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the associated resource control from the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove the associated resource control from the database", Err: err} } } err = handler.FileService.RemoveDirectory(stack.ProjectPath) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove stack files from disk", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to remove stack files from disk", Err: err} } return response.Empty(w) @@ -132,31 +137,31 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWriter, stackName string, securityContext *security.RestrictedRequestContext) *httperror.HandlerError { endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if !securityContext.IsAdmin { - return &httperror.HandlerError{http.StatusUnauthorized, "Permission denied to delete the stack", httperrors.ErrUnauthorized} + return &httperror.HandlerError{StatusCode: http.StatusUnauthorized, Message: "Permission denied to delete the stack", Err: httperrors.ErrUnauthorized} } stack, err := handler.DataStore.Stack().StackByName(stackName) if err != nil && err != bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for stack existence inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for stack existence inside the database", Err: err} } if stack != nil { - return &httperror.HandlerError{http.StatusBadRequest, "A stack with this name exists inside the database. Cannot use external delete method", errors.New("A tag already exists with this name")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "A stack with this name exists inside the database. Cannot use external delete method", Err: errors.New("A tag already exists with this name")} } endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the environment associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } stack = &portainer.Stack{ @@ -164,18 +169,57 @@ func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWrit Type: portainer.DockerSwarmStack, } - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to delete stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to delete stack", Err: err} } return response.Empty(w) } -func (handler *Handler) deleteStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error { +func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.Stack, endpoint *portainer.Endpoint) error { if stack.Type == portainer.DockerSwarmStack { return handler.SwarmStackManager.Remove(stack, endpoint) } + if stack.Type == portainer.DockerComposeStack { + return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint) + } + if stack.Type == portainer.KubernetesStack { + var manifestFiles []string - return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint) + //if it is a compose format kub stack, create a temp dir and convert the manifest files into it + //then process the remove operation + if stack.IsComposeFormat { + fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + tmpDir, err := ioutil.TempDir("", "kub_delete") + if err != nil { + return errors.Wrap(err, "failed to create temp directory for deleting kub stack") + } + defer os.RemoveAll(tmpDir) + + for _, fileName := range fileNames { + manifestFilePath := path.Join(tmpDir, fileName) + manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName)) + if err != nil { + return errors.Wrap(err, "failed to read manifest file") + } + + manifestContent, err = handler.KubernetesDeployer.ConvertCompose(manifestContent) + if err != nil { + return errors.Wrap(err, "failed to convert docker compose file to a kube manifest") + } + + err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent)) + if err != nil { + return errors.Wrap(err, "failed to create temp manifest file") + } + manifestFiles = append(manifestFiles, manifestFilePath) + } + } else { + manifestFiles = stackutils.GetStackFilePaths(stack) + } + out, err := handler.KubernetesDeployer.Remove(userID, endpoint, manifestFiles, stack.Namespace) + return errors.WithMessagef(err, "failed to remove kubernetes resources: %q", out) + } + return fmt.Errorf("unsupported stack type: %v", stack.Type) } diff --git a/api/http/handler/stacks/stack_migrate.go b/api/http/handler/stacks/stack_migrate.go index c631f8fe3..d69666d90 100644 --- a/api/http/handler/stacks/stack_migrate.go +++ b/api/http/handler/stacks/stack_migrate.go @@ -50,52 +50,54 @@ func (payload *stackMigratePayload) Validate(r *http.Request) error { func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } var payload stackMigratePayload err = request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Migrating a kubernetes stack is not supported", Err: err} } endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 @@ -103,7 +105,7 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht // can use the optional EndpointID query parameter to associate a valid environment(endpoint) identifier to the stack. endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if endpointID != int(stack.EndpointID) { stack.EndpointID = portainer.EndpointID(endpointID) @@ -111,9 +113,9 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht targetEndpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(payload.EndpointID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } stack.EndpointID = portainer.EndpointID(payload.EndpointID) @@ -126,14 +128,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht stack.Name = payload.Name } - isUnique, err := handler.checkUniqueName(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "") + isUnique, err := handler.checkUniqueStackNameInDocker(targetEndpoint, stack.Name, stack.ID, stack.SwarmID != "") if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on environment '%s'", stack.Name, targetEndpoint.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + errorMessage := fmt.Sprintf("A stack with the name '%s' is already running on endpoint '%s'", stack.Name, targetEndpoint.Name) + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)} } migrationError := handler.migrateStack(r, stack, targetEndpoint) @@ -142,14 +144,14 @@ func (handler *Handler) stackMigrate(w http.ResponseWriter, r *http.Request) *ht } stack.Name = oldName - err = handler.deleteStack(stack, endpoint) + err = handler.deleteStack(securityContext.UserID, stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} } if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { @@ -175,7 +177,7 @@ func (handler *Handler) migrateComposeStack(r *http.Request, stack *portainer.St err := handler.deployComposeStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } return nil @@ -189,7 +191,7 @@ func (handler *Handler) migrateSwarmStack(r *http.Request, stack *portainer.Stac err := handler.deploySwarmStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } return nil diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index 3a6ca0285..7c1f20f97 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -33,59 +33,61 @@ import ( func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Starting a kubernetes stack is not supported", Err: err} } endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an environment with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } - isUnique, err := handler.checkUniqueName(endpoint, stack.Name, stack.ID, stack.SwarmID != "") + isUnique, err := handler.checkUniqueStackNameInDocker(endpoint, stack.Name, stack.ID, stack.SwarmID != "") if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", stack.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: errorMessage, Err: errors.New(errorMessage)} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } if stack.Status == portainer.StackStatusActive { - return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is already active", Err: errors.New("Stack is already active")} } if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { @@ -101,13 +103,13 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http err = handler.startStack(stack, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to start stack", Err: err} } stack.Status = portainer.StackStatusActive err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update stack status", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update stack status", Err: err} } if stack.GitConfig != nil && stack.GitConfig.Authentication != nil && stack.GitConfig.Authentication.Password != "" { diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 6aea8e375..0b9816ef4 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -46,6 +46,10 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} } + if stack.Type == portainer.KubernetesStack { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stopping a kubernetes stack is not supported", Err: err} + } + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { return &httperror.HandlerError{http.StatusNotFound, "Unable to find an environment with the specified identifier inside the database", err} @@ -58,19 +62,17 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access environment", err} } - if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { - resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} - } + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} - } - if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} - } + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + } + if !access { + return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} } if stack.Status == portainer.StackStatusInactive { diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index 45576fe61..a37ba54f5 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -1,10 +1,11 @@ package stacks import ( - "errors" "net/http" + "time" "github.com/asaskevich/govalidator" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" @@ -98,17 +99,22 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access environment", Err: err} } + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + user, err := handler.DataStore.User().User(securityContext.UserID) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Cannot find context user", Err: errors.Wrap(err, "failed to fetch the user")} + } + if stack.Type == portainer.DockerSwarmStack || stack.Type == portainer.DockerComposeStack { resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } - securityContext, err := security.RetrieveRestrictedRequestContext(r) - if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} - } - access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} @@ -127,6 +133,8 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * stack.GitConfig.ReferenceName = payload.RepositoryReferenceName stack.AutoUpdate = payload.AutoUpdate stack.Env = payload.Env + stack.UpdatedBy = user.Username + stack.UpdateDate = time.Now().Unix() stack.GitConfig.Authentication = nil if payload.RepositoryAuthentication { diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go index d5d87c4f0..300a33d07 100644 --- a/api/http/handler/stacks/stack_update_git_redeploy.go +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -2,10 +2,8 @@ package stacks import ( "fmt" - "io/ioutil" "log" "net/http" - "path/filepath" "time" "github.com/asaskevich/govalidator" @@ -216,15 +214,15 @@ func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, end if stack.Namespace == "" { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Invalid namespace", Err: errors.New("Namespace must not be empty when redeploying kubernetes stacks")} } - content, err := ioutil.ReadFile(filepath.Join(stack.ProjectPath, stack.GitConfig.ConfigFilePath)) + tokenData, err := security.RetrieveTokenData(r) if err != nil { - return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to read deployment.yml manifest file", Err: errors.Wrap(err, "failed to read manifest file")} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err} } - _, err = handler.deployKubernetesStack(r, endpoint, string(content), stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{ - StackID: int(stack.ID), - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "git", + _, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: tokenData.Username, + Kind: "git", }) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to redeploy Kubernetes stack", Err: errors.WithMessage(err, "failed to deploy kube application")} diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index 114552b68..e5d659a52 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -2,7 +2,10 @@ package stacks import ( "fmt" + "io/ioutil" "net/http" + "os" + "path" "strconv" "github.com/asaskevich/govalidator" @@ -10,7 +13,9 @@ import ( httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/http/security" k "github.com/portainer/portainer/api/kubernetes" ) @@ -23,6 +28,7 @@ type kubernetesGitStackUpdatePayload struct { RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string + AutoUpdate *portainer.StackAutoUpdate } func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error { @@ -36,12 +42,20 @@ func (payload *kubernetesGitStackUpdatePayload) Validate(r *http.Request) error if govalidator.IsNull(payload.RepositoryReferenceName) { payload.RepositoryReferenceName = defaultGitReferenceName } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err + } return nil } func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { if stack.GitConfig != nil { + //stop the autoupdate job if there is any + if stack.AutoUpdate != nil { + stopAutoupdate(stack.ID, stack.AutoUpdate.JobID, *handler.Scheduler) + } + var payload kubernetesGitStackUpdatePayload if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil { @@ -49,6 +63,8 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + if payload.RepositoryAuthentication { password := payload.RepositoryPassword if password == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { @@ -61,6 +77,15 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } else { stack.GitConfig.Authentication = nil } + + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService) + if e != nil { + return e + } + stack.AutoUpdate.JobID = jobID + } + return nil } @@ -71,11 +96,27 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } - _, err = handler.deployKubernetesStack(r, endpoint, payload.StackFileContent, stack.IsComposeFormat, stack.Namespace, k.KubeAppLabels{ - StackID: int(stack.ID), - Name: stack.Name, - Owner: stack.CreatedBy, - Kind: "content", + tokenData, err := security.RetrieveTokenData(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Failed to retrieve user token data", Err: err} + } + + tempFileDir, _ := ioutil.TempDir("", "kub_file_content") + defer os.RemoveAll(tempFileDir) + + if err := filesystem.WriteToFile(path.Join(tempFileDir, stack.EntryPoint), []byte(payload.StackFileContent)); err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to persist deployment file in a temp directory", Err: err} + } + + //use temp dir as the stack project path for deployment + //so if the deployment failed, the original file won't be over-written + stack.ProjectPath = tempFileDir + + _, err = handler.deployKubernetesStack(tokenData.ID, endpoint, stack, k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: stack.CreatedBy, + Kind: "content", }) if err != nil { @@ -83,7 +124,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. } stackFolder := strconv.Itoa(int(stack.ID)) - _, err = handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent)) + projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent)) if err != nil { fileType := "Manifest" if stack.IsComposeFormat { @@ -92,6 +133,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. errMsg := fmt.Sprintf("Unable to persist Kubernetes %s file on disk", fileType) return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: errMsg, Err: err} } + stack.ProjectPath = projectPath return nil } diff --git a/api/http/handler/stacks/webhook_invoke.go b/api/http/handler/stacks/webhook_invoke.go index 01c9d701b..5868dc79e 100644 --- a/api/http/handler/stacks/webhook_invoke.go +++ b/api/http/handler/stacks/webhook_invoke.go @@ -1,10 +1,10 @@ package stacks import ( - "log" "net/http" "github.com/gofrs/uuid" + "github.com/sirupsen/logrus" "github.com/portainer/libhttp/response" @@ -31,7 +31,10 @@ func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *h } if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { - log.Printf("[ERROR] %s\n", err) + if _, ok := err.(*stacks.StackAuthorMissingErr); ok { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: "Autoupdate for the stack isn't available", Err: err} + } + logrus.WithError(err).Error("failed to update the stack") return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err} } diff --git a/api/http/proxy/factory/agent.go b/api/http/proxy/factory/agent.go index 159934421..74b01d084 100644 --- a/api/http/proxy/factory/agent.go +++ b/api/http/proxy/factory/agent.go @@ -24,6 +24,7 @@ type ProxyServer struct { // NewAgentProxy creates a new instance of ProxyServer that wrap http requests with agent headers func (factory *ProxyFactory) NewAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) { urlString := endpoint.URL + if endpointutils.IsEdgeEndpoint((endpoint)) { tunnel, err := factory.reverseTunnelService.GetActiveTunnel(endpoint) if err != nil { diff --git a/api/internal/endpointutils/endpointutils.go b/api/internal/endpointutils/endpointutils.go index 67cde11ed..4c54c4196 100644 --- a/api/internal/endpointutils/endpointutils.go +++ b/api/internal/endpointutils/endpointutils.go @@ -11,7 +11,7 @@ func IsLocalEndpoint(endpoint *portainer.Endpoint) bool { return strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") || endpoint.Type == 5 } -// IsKubernetesEndpoint returns true if this is a kubernetes environment(endpoint) +// IsKubernetesEndpoint returns true if this is a kubernetes endpoint func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool { return endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || @@ -25,6 +25,7 @@ func IsDockerEndpoint(endpoint *portainer.Endpoint) bool { endpoint.Type == portainer.EdgeAgentOnDockerEnvironment } +// IsEdgeEndpoint returns true if this is an Edge endpoint func IsEdgeEndpoint(endpoint *portainer.Endpoint) bool { return endpoint.Type == portainer.EdgeAgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment } diff --git a/api/internal/stackutils/stackutils.go b/api/internal/stackutils/stackutils.go index 7e94bff17..4945eeaa1 100644 --- a/api/internal/stackutils/stackutils.go +++ b/api/internal/stackutils/stackutils.go @@ -2,9 +2,13 @@ package stackutils import ( "fmt" + "io/ioutil" "path" + "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" + k "github.com/portainer/portainer/api/kubernetes" ) // ResourceControlID returns the stack resource control id @@ -20,3 +24,39 @@ func GetStackFilePaths(stack *portainer.Stack) []string { } return filePaths } + +// CreateTempK8SDeploymentFiles reads manifest files from original stack project path +// then add app labels into the file contents and create temp files for deployment +// return temp file paths and temp dir +func CreateTempK8SDeploymentFiles(stack *portainer.Stack, kubeDeployer portainer.KubernetesDeployer, appLabels k.KubeAppLabels) ([]string, string, error) { + fileNames := append([]string{stack.EntryPoint}, stack.AdditionalFiles...) + var manifestFilePaths []string + tmpDir, err := ioutil.TempDir("", "kub_deployment") + if err != nil { + return nil, "", errors.Wrap(err, "failed to create temp kub deployment directory") + } + + for _, fileName := range fileNames { + manifestFilePath := path.Join(tmpDir, fileName) + manifestContent, err := ioutil.ReadFile(path.Join(stack.ProjectPath, fileName)) + if err != nil { + return nil, "", errors.Wrap(err, "failed to read manifest file") + } + if stack.IsComposeFormat { + manifestContent, err = kubeDeployer.ConvertCompose(manifestContent) + if err != nil { + return nil, "", errors.Wrap(err, "failed to convert docker compose file to a kube manifest") + } + } + manifestContent, err = k.AddAppLabels(manifestContent, appLabels.ToMap()) + if err != nil { + return nil, "", errors.Wrap(err, "failed to add application labels") + } + err = filesystem.WriteToFile(manifestFilePath, []byte(manifestContent)) + if err != nil { + return nil, "", errors.Wrap(err, "failed to create temp manifest file") + } + manifestFilePaths = append(manifestFilePaths, manifestFilePath) + } + return manifestFilePaths, tmpDir, nil +} diff --git a/api/kubernetes/yaml.go b/api/kubernetes/yaml.go index b90020641..89dbc8009 100644 --- a/api/kubernetes/yaml.go +++ b/api/kubernetes/yaml.go @@ -12,6 +12,7 @@ import ( ) const ( + labelPortainerAppStack = "io.portainer.kubernetes.application.stack" labelPortainerAppStackID = "io.portainer.kubernetes.application.stackid" labelPortainerAppName = "io.portainer.kubernetes.application.name" labelPortainerAppOwner = "io.portainer.kubernetes.application.owner" @@ -20,17 +21,18 @@ const ( // KubeAppLabels are labels applied to all resources deployed in a kubernetes stack type KubeAppLabels struct { - StackID int - Name string - Owner string - Kind string + StackID int + StackName string + Owner string + Kind string } // ToMap converts KubeAppLabels to a map[string]string func (kal *KubeAppLabels) ToMap() map[string]string { return map[string]string{ labelPortainerAppStackID: strconv.Itoa(kal.StackID), - labelPortainerAppName: kal.Name, + labelPortainerAppStack: kal.StackName, + labelPortainerAppName: kal.StackName, labelPortainerAppOwner: kal.Owner, labelPortainerAppKind: kal.Kind, } @@ -167,13 +169,6 @@ func addLabels(obj map[string]interface{}, appLabels map[string]string) { labels[k] = v } - // fallback to metadata name if name label not explicitly provided - if name, ok := labels[labelPortainerAppName]; !ok || name == "" { - if n, ok := metadata["name"]; ok { - labels[labelPortainerAppName] = n.(string) - } - } - metadata["labels"] = labels obj["metadata"] = metadata } diff --git a/api/kubernetes/yaml_test.go b/api/kubernetes/yaml_test.go index 167b97038..48ca95373 100644 --- a/api/kubernetes/yaml_test.go +++ b/api/kubernetes/yaml_test.go @@ -39,6 +39,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -86,6 +87,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -174,6 +176,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -194,6 +197,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: redis spec: @@ -216,6 +220,7 @@ items: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -297,6 +302,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -322,6 +328,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -340,6 +347,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: busybox spec: @@ -388,6 +396,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" name: web spec: @@ -402,10 +411,10 @@ spec: } labels := KubeAppLabels{ - StackID: 123, - Name: "best-name", - Owner: "best-owner", - Kind: "git", + StackID: 123, + StackName: "best-name", + Owner: "best-owner", + Kind: "git", } for _, tt := range tests { @@ -417,81 +426,6 @@ spec: } } -func Test_AddAppLabels_PickingName_WhenLabelNameIsEmpty(t *testing.T) { - labels := KubeAppLabels{ - StackID: 123, - Owner: "best-owner", - Kind: "git", - } - - input := `apiVersion: v1 -kind: Service -metadata: - name: web -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - expected := `apiVersion: v1 -kind: Service -metadata: - labels: - io.portainer.kubernetes.application.kind: git - io.portainer.kubernetes.application.name: web - io.portainer.kubernetes.application.owner: best-owner - io.portainer.kubernetes.application.stackid: "123" - name: web -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - result, err := AddAppLabels([]byte(input), labels.ToMap()) - assert.NoError(t, err) - assert.Equal(t, expected, string(result)) -} - -func Test_AddAppLabels_PickingName_WhenLabelAndMetadataNameAreEmpty(t *testing.T) { - labels := KubeAppLabels{ - StackID: 123, - Owner: "best-owner", - Kind: "git", - } - - input := `apiVersion: v1 -kind: Service -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - expected := `apiVersion: v1 -kind: Service -metadata: - labels: - io.portainer.kubernetes.application.kind: git - io.portainer.kubernetes.application.name: "" - io.portainer.kubernetes.application.owner: best-owner - io.portainer.kubernetes.application.stackid: "123" -spec: - ports: - - name: "5000" - port: 5000 - targetPort: 5000 -` - - result, err := AddAppLabels([]byte(input), labels.ToMap()) - assert.NoError(t, err) - assert.Equal(t, expected, string(result)) -} - func Test_AddAppLabels_HelmApp(t *testing.T) { labels := GetHelmAppLabels("best-name", "best-owner") @@ -658,10 +592,10 @@ spec: func Test_DocumentSeperator(t *testing.T) { labels := KubeAppLabels{ - StackID: 123, - Name: "best-name", - Owner: "best-owner", - Kind: "git", + StackID: 123, + StackName: "best-name", + Owner: "best-owner", + Kind: "git", } input := `apiVersion: v1 @@ -684,6 +618,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" --- apiVersion: v1 @@ -694,6 +629,7 @@ metadata: io.portainer.kubernetes.application.kind: git io.portainer.kubernetes.application.name: best-name io.portainer.kubernetes.application.owner: best-owner + io.portainer.kubernetes.application.stack: best-name io.portainer.kubernetes.application.stackid: "123" ` result, err := AddAppLabels([]byte(input), labels.ToMap()) diff --git a/api/portainer.go b/api/portainer.go index b91413c30..d9642c5e9 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -3,7 +3,6 @@ package portainer import ( "context" "io" - "net/http" "time" gittypes "github.com/portainer/portainer/api/git/types" @@ -1281,7 +1280,8 @@ type ( // KubernetesDeployer represents a service to deploy a manifest inside a Kubernetes environment(endpoint) KubernetesDeployer interface { - Deploy(request *http.Request, endpoint *Endpoint, data string, namespace string) (string, error) + Deploy(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error) + Remove(userID UserID, endpoint *Endpoint, manifestFiles []string, namespace string) (string, error) ConvertCompose(data []byte) ([]byte, error) } diff --git a/api/scheduler/scheduler.go b/api/scheduler/scheduler.go index 6568f753f..529574f0c 100644 --- a/api/scheduler/scheduler.go +++ b/api/scheduler/scheduler.go @@ -8,11 +8,12 @@ import ( "github.com/pkg/errors" "github.com/robfig/cron/v3" + "github.com/sirupsen/logrus" ) type Scheduler struct { - crontab *cron.Cron - shutdownCtx context.Context + crontab *cron.Cron + activeJobs map[cron.EntryID]context.CancelFunc } func NewScheduler(ctx context.Context) *Scheduler { @@ -20,7 +21,8 @@ func NewScheduler(ctx context.Context) *Scheduler { crontab.Start() s := &Scheduler{ - crontab: crontab, + crontab: crontab, + activeJobs: make(map[cron.EntryID]context.CancelFunc), } if ctx != nil { @@ -43,8 +45,10 @@ func (s *Scheduler) Shutdown() error { ctx := s.crontab.Stop() <-ctx.Done() - for _, j := range s.crontab.Entries() { - s.crontab.Remove(j.ID) + for _, job := range s.crontab.Entries() { + if cancel, ok := s.activeJobs[job.ID]; ok { + cancel() + } } err := ctx.Err() @@ -60,14 +64,36 @@ func (s *Scheduler) StopJob(jobID string) error { if err != nil { return errors.Wrapf(err, "failed convert jobID %q to int", jobID) } - s.crontab.Remove(cron.EntryID(id)) + entryID := cron.EntryID(id) + if cancel, ok := s.activeJobs[entryID]; ok { + cancel() + } return nil } // StartJobEvery schedules a new periodic job with a given duration. -// Returns job id that could be used to stop the given job -func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string { - entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job)) - return strconv.Itoa(int(entryId)) +// Returns job id that could be used to stop the given job. +// When job run returns an error, that job won't be run again. +func (s *Scheduler) StartJobEvery(duration time.Duration, job func() error) string { + ctx, cancel := context.WithCancel(context.Background()) + + j := cron.FuncJob(func() { + if err := job(); err != nil { + logrus.Debug("job returned an error") + cancel() + } + }) + + entryID := s.crontab.Schedule(cron.Every(duration), j) + + s.activeJobs[entryID] = cancel + + go func(entryID cron.EntryID) { + <-ctx.Done() + logrus.Debug("job cancelled, stopping") + s.crontab.Remove(entryID) + }(entryID) + + return strconv.Itoa(int(entryID)) } diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go index 6d21e49ec..2d66f2e6b 100644 --- a/api/scheduler/scheduler_test.go +++ b/api/scheduler/scheduler_test.go @@ -9,49 +9,92 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_CanStartAndTerminate(t *testing.T) { - s := NewScheduler(context.Background()) - s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) +var jobInterval = time.Second - err := s.Shutdown() - assert.NoError(t, err, "Shutdown should return no errors") - assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed") -} - -func Test_CanTerminateByCancellingContext(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - s := NewScheduler(ctx) - s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) - - cancel() - - for i := 0; i < 100; i++ { - if len(s.crontab.Entries()) == 0 { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect") -} - -func Test_StartAndStopJob(t *testing.T) { +func Test_ScheduledJobRuns(t *testing.T) { s := NewScheduler(context.Background()) defer s.Shutdown() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) - var jobOne string var workDone bool - jobOne = s.StartJobEvery(time.Second, func() { - assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job") + s.StartJobEvery(jobInterval, func() error { workDone = true - s.StopJob(jobOne) cancel() + return nil }) <-ctx.Done() assert.True(t, workDone, "value should been set in the job") - assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs") - +} + +func Test_JobCanBeStopped(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + + var workDone bool + jobID := s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + s.StopJob(jobID) + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") +} + +func Test_JobShouldStop_UponError(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + var acc int + s.StartJobEvery(jobInterval, func() error { + acc++ + return fmt.Errorf("failed") + }) + + <-time.After(3 * jobInterval) + assert.Equal(t, 1, acc, "job stop after the first run because it returns an error") +} + +func Test_CanTerminateAllJobs_ByShuttingDownScheduler(t *testing.T) { + s := NewScheduler(context.Background()) + + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + + var workDone bool + s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + + s.Shutdown() + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") +} + +func Test_CanTerminateAllJobs_ByCancellingParentContext(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*jobInterval) + s := NewScheduler(ctx) + + var workDone bool + s.StartJobEvery(jobInterval, func() error { + workDone = true + + cancel() + return nil + }) + + cancel() + + <-ctx.Done() + assert.False(t, workDone, "job shouldn't had a chance to run") } diff --git a/api/stacks/deploy.go b/api/stacks/deploy.go index 797a415e3..2aa48868c 100644 --- a/api/stacks/deploy.go +++ b/api/stacks/deploy.go @@ -1,15 +1,29 @@ package stacks import ( + "fmt" "strings" "time" "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/security" + log "github.com/sirupsen/logrus" ) +type StackAuthorMissingErr struct { + stackID int + authorName string +} + +func (e *StackAuthorMissingErr) Error() string { + return fmt.Sprintf("stack's %v author %s is missing", e.stackID, e.authorName) +} + func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + logger := log.WithFields(log.Fields{"stackID": stackID}) + logger.Debug("redeploying stack") + stack, err := datastore.Stack().Stack(stackID) if err != nil { return errors.WithMessagef(err, "failed to get the stack %v", stackID) @@ -19,6 +33,17 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return nil // do nothing if it isn't a git-based stack } + author := stack.UpdatedBy + if author == "" { + author = stack.CreatedBy + } + + user, err := datastore.User().UserByUsername(author) + if err != nil { + logger.WithFields(log.Fields{"author": author, "stack": stack.Name, "endpointID": stack.EndpointID}).Warn("cannot autoupdate a stack, stack author user is missing") + return &StackAuthorMissingErr{int(stack.ID), author} + } + username, password := "", "" if stack.GitConfig.Authentication != nil { username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password @@ -54,12 +79,7 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return errors.WithMessagef(err, "failed to find the environment %v associated to the stack %v", stack.EndpointID, stack.ID) } - author := stack.UpdatedBy - if author == "" { - author = stack.CreatedBy - } - - registries, err := getUserRegistries(datastore, author, endpoint.ID) + registries, err := getUserRegistries(datastore, user, endpoint.ID) if err != nil { return err } @@ -75,6 +95,12 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data if err != nil { return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) } + case portainer.KubernetesStack: + logger.Debugf("deploying a kube app") + err := deployer.DeployKubernetesStack(stack, endpoint, user) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a kubternetes app stack %v", stackID) + } default: return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) } @@ -88,24 +114,19 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data return nil } -func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) { +func getUserRegistries(datastore portainer.DataStore, user *portainer.User, endpointID portainer.EndpointID) ([]portainer.Registry, error) { registries, err := datastore.Registry().Registries() if err != nil { return nil, errors.WithMessage(err, "unable to retrieve registries from the database") } - user, err := datastore.User().UserByUsername(authorUsername) - if err != nil { - return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername) - } - if user.Role == portainer.AdministratorRole { return registries, nil } userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID) if err != nil { - return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername) + return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", user.Username) } filteredRegistries := make([]portainer.Registry, 0, len(registries)) diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go index dd0b3ff69..9067f9232 100644 --- a/api/stacks/deploy_test.go +++ b/api/stacks/deploy_test.go @@ -35,6 +35,10 @@ func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *port return nil } +func (s *noopDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error { + return nil +} + func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() @@ -48,7 +52,11 @@ func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() - err := store.Stack().CreateStack(&portainer.Stack{ID: 1}) + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ID: 1, CreatedBy: "admin"}) assert.NoError(t, err, "failed to create a test stack") err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) @@ -61,8 +69,13 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { tmpDir, _ := ioutil.TempDir("", "stack") - err := store.Stack().CreateStack(&portainer.Stack{ + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ ID: 1, + CreatedBy: "admin", ProjectPath: tmpDir, GitConfig: &gittypes.RepoConfig{ URL: "url", @@ -80,8 +93,13 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { store, teardown := bolt.MustNewTestStore(true) defer teardown() - err := store.Stack().CreateStack(&portainer.Stack{ - ID: 1, + admin := &portainer.User{ID: 1, Username: "admin"} + err := store.User().CreateUser(admin) + assert.NoError(t, err, "error creating an admin") + + err = store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + CreatedBy: "admin", GitConfig: &gittypes.RepoConfig{ URL: "url", ReferenceName: "ref", @@ -136,12 +154,12 @@ func Test_redeployWhenChanged(t *testing.T) { assert.NoError(t, err) }) - t.Run("can NOT deploy kube stack", func(t *testing.T) { + t.Run("can deploy kube app", func(t *testing.T) { stack.Type = portainer.KubernetesStack store.Stack().UpdateStack(stack.ID, &stack) err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) - assert.EqualError(t, err, "cannot update stack, type 3 is unsupported") + assert.NoError(t, err) }) } @@ -151,12 +169,12 @@ func Test_getUserRegistries(t *testing.T) { endpointID := 123 - admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} - err := store.User().CreateUser(&admin) + admin := &portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} + err := store.User().CreateUser(admin) assert.NoError(t, err, "error creating an admin") - user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} - err = store.User().CreateUser(&user) + user := &portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} + err = store.User().CreateUser(user) assert.NoError(t, err, "error creating a user") team := portainer.Team{ID: 1, Name: "team"} @@ -208,13 +226,13 @@ func Test_getUserRegistries(t *testing.T) { assert.NoError(t, err, "couldn't create a registry") t.Run("admin should has access to all registries", func(t *testing.T) { - registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID)) + registries, err := getUserRegistries(store, admin, portainer.EndpointID(endpointID)) assert.NoError(t, err) assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries) }) t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) { - registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID)) + registries, err := getUserRegistries(store, user, portainer.EndpointID(endpointID)) assert.NoError(t, err) assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries) }) diff --git a/api/stacks/deployer.go b/api/stacks/deployer.go index c594e48ec..714bae066 100644 --- a/api/stacks/deployer.go +++ b/api/stacks/deployer.go @@ -2,27 +2,36 @@ package stacks import ( "context" + "os" "sync" + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" + k "github.com/portainer/portainer/api/kubernetes" ) type StackDeployer interface { DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error + DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error } type stackDeployer struct { lock *sync.Mutex swarmStackManager portainer.SwarmStackManager composeStackManager portainer.ComposeStackManager + kubernetesDeployer portainer.KubernetesDeployer } -func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer { +// NewStackDeployer inits a stackDeployer struct with a SwarmStackManager, a ComposeStackManager and a KubernetesDeployer +func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager, kubernetesDeployer portainer.KubernetesDeployer) *stackDeployer { return &stackDeployer{ lock: &sync.Mutex{}, swarmStackManager: swarmStackManager, composeStackManager: composeStackManager, + kubernetesDeployer: kubernetesDeployer, } } @@ -45,3 +54,33 @@ func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *por return d.composeStackManager.Up(context.TODO(), stack, endpoint) } + +func (d *stackDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error { + d.lock.Lock() + defer d.lock.Unlock() + + appLabels := k.KubeAppLabels{ + StackID: int(stack.ID), + StackName: stack.Name, + Owner: user.Username, + } + + if stack.GitConfig == nil { + appLabels.Kind = "content" + } else { + appLabels.Kind = "git" + } + + manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, d.kubernetesDeployer, appLabels) + if err != nil { + return errors.Wrap(err, "failed to create temp kub deployment files") + } + defer os.RemoveAll(tempDir) + + _, err = d.kubernetesDeployer.Deploy(user.ID, endpoint, manifestFilePaths, stack.Namespace) + if err != nil { + return errors.Wrap(err, "failed to deploy kubernetes application") + } + + return nil +} diff --git a/api/stacks/scheduled.go b/api/stacks/scheduled.go index fb90ca22c..238939fee 100644 --- a/api/stacks/scheduled.go +++ b/api/stacks/scheduled.go @@ -1,7 +1,6 @@ package stacks import ( - "log" "time" "github.com/pkg/errors" @@ -19,10 +18,9 @@ func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDepl if err != nil { return errors.Wrap(err, "Unable to parse auto update interval") } - jobID := scheduler.StartJobEvery(d, func() { - if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil { - log.Printf("[ERROR] %s\n", err) - } + stackID := stack.ID // to be captured by the scheduled function + jobID := scheduler.StartJobEvery(d, func() error { + return RedeployWhenChanged(stackID, stackdeployer, datastore, gitService) }) stack.AutoUpdate.JobID = jobID diff --git a/app/config.js b/app/config.js index 0ff080aaf..1793e5006 100644 --- a/app/config.js +++ b/app/config.js @@ -56,7 +56,7 @@ angular.module('portainer').config([ closeButton: true, progressBar: true, tapToDismiss: false, - } + }; Terminal.applyAddon(fit); diff --git a/app/edge/components/group-form/groupForm.html b/app/edge/components/group-form/groupForm.html index 2d9c79dc4..0bbcfc1b8 100644 --- a/app/edge/components/group-form/groupForm.html +++ b/app/edge/components/group-form/groupForm.html @@ -65,7 +65,9 @@