mirror of
https://github.com/portainer/portainer.git
synced 2025-07-19 13:29:41 +02:00
refactor(stack): stack build process backend only [EE-4342] (#7750)
This commit is contained in:
Oscar Zhou
GitHub
parent
83a1ce9d2a
commit
e9de484c3e
65 changed files with 2270 additions and 942 deletions
|
|
@ -3,10 +3,6 @@ package stacks
|
|||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/asaskevich/govalidator"
|
||||
"github.com/pkg/errors"
|
||||
|
|
@ -15,11 +11,10 @@ import (
|
|||
"github.com/portainer/libhttp/request"
|
||||
"github.com/portainer/libhttp/response"
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/filesystem"
|
||||
gittypes "github.com/portainer/portainer/api/git/types"
|
||||
"github.com/portainer/portainer/api/http/client"
|
||||
"github.com/portainer/portainer/api/internal/stackutils"
|
||||
k "github.com/portainer/portainer/api/kubernetes"
|
||||
"github.com/portainer/portainer/api/stacks/deployments"
|
||||
"github.com/portainer/portainer/api/stacks/stackbuilders"
|
||||
"github.com/portainer/portainer/api/stacks/stackutils"
|
||||
)
|
||||
|
||||
type kubernetesStringDeploymentPayload struct {
|
||||
|
|
@ -29,6 +24,15 @@ type kubernetesStringDeploymentPayload struct {
|
|||
StackFileContent string
|
||||
}
|
||||
|
||||
func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload {
|
||||
return stackbuilders.StackPayload{
|
||||
StackName: name,
|
||||
Namespace: namespace,
|
||||
StackFileContent: fileContent,
|
||||
ComposeFormat: composeFormat,
|
||||
}
|
||||
}
|
||||
|
||||
type kubernetesGitDeploymentPayload struct {
|
||||
StackName string
|
||||
ComposeFormat bool
|
||||
|
|
@ -43,6 +47,24 @@ type kubernetesGitDeploymentPayload struct {
|
|||
AutoUpdate *portainer.StackAutoUpdate
|
||||
}
|
||||
|
||||
func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.StackAutoUpdate) stackbuilders.StackPayload {
|
||||
return stackbuilders.StackPayload{
|
||||
StackName: name,
|
||||
RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{
|
||||
URL: repoUrl,
|
||||
ReferenceName: repoReference,
|
||||
Authentication: repoAuthentication,
|
||||
Username: repoUsername,
|
||||
Password: repoPassword,
|
||||
},
|
||||
Namespace: namespace,
|
||||
ComposeFormat: composeFormat,
|
||||
ManifestFile: manifest,
|
||||
AdditionalFiles: additionalFiles,
|
||||
AutoUpdate: autoUpdate,
|
||||
}
|
||||
}
|
||||
|
||||
type kubernetesManifestURLDeploymentPayload struct {
|
||||
StackName string
|
||||
Namespace string
|
||||
|
|
@ -50,6 +72,15 @@ type kubernetesManifestURLDeploymentPayload struct {
|
|||
ManifestURL string
|
||||
}
|
||||
|
||||
func createStackPayloadFromK8sUrlPayload(name, namespace, manifestUrl string, composeFormat bool) stackbuilders.StackPayload {
|
||||
return stackbuilders.StackPayload{
|
||||
StackName: name,
|
||||
Namespace: namespace,
|
||||
ManifestURL: manifestUrl,
|
||||
ComposeFormat: composeFormat,
|
||||
}
|
||||
}
|
||||
|
||||
func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) error {
|
||||
if govalidator.IsNull(payload.StackFileContent) {
|
||||
return errors.New("Invalid stack file content")
|
||||
|
|
@ -70,7 +101,7 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
|
|||
if govalidator.IsNull(payload.ManifestFile) {
|
||||
return errors.New("Invalid manifest file in repository")
|
||||
}
|
||||
if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil {
|
||||
if err := stackutils.ValidateStackAutoUpdate(payload.AutoUpdate); err != nil {
|
||||
return err
|
||||
}
|
||||
if govalidator.IsNull(payload.StackName) {
|
||||
|
|
@ -93,12 +124,6 @@ type createKubernetesStackResponse struct {
|
|||
Output string `json:"Output"`
|
||||
}
|
||||
|
||||
// convert string to valid kubernetes label by replacing invalid characters with periods
|
||||
func sanitizeLabel(value string) string {
|
||||
re := regexp.MustCompile(`[^A-Za-z0-9\.\-\_]+`)
|
||||
return re.ReplaceAllString(value, ".")
|
||||
}
|
||||
|
||||
func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError {
|
||||
var payload kubernetesStringDeploymentPayload
|
||||
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
|
||||
|
|
@ -114,60 +139,27 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
|
|||
return httperror.InternalServerError("Unable to check for name collision", err)
|
||||
}
|
||||
if !isUnique {
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
|
||||
}
|
||||
|
||||
stackID := handler.DataStore.Stack().GetNextIdentifier()
|
||||
stack := &portainer.Stack{
|
||||
ID: portainer.StackID(stackID),
|
||||
Type: portainer.KubernetesStack,
|
||||
EndpointID: endpoint.ID,
|
||||
EntryPoint: filesystem.ManifestFileDefaultName,
|
||||
Name: payload.StackName,
|
||||
Namespace: payload.Namespace,
|
||||
Status: portainer.StackStatusActive,
|
||||
CreationDate: time.Now().Unix(),
|
||||
CreatedBy: sanitizeLabel(user.Username),
|
||||
IsComposeFormat: payload.ComposeFormat,
|
||||
}
|
||||
stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat)
|
||||
|
||||
stackFolder := strconv.Itoa(int(stack.ID))
|
||||
k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore,
|
||||
handler.FileService,
|
||||
handler.StackDeployer,
|
||||
handler.KubernetesDeployer,
|
||||
user)
|
||||
|
||||
projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, []byte(payload.StackFileContent))
|
||||
if err != nil {
|
||||
fileType := "Manifest"
|
||||
if stack.IsComposeFormat {
|
||||
fileType = "Compose"
|
||||
}
|
||||
errMsg := fmt.Sprintf("Unable to persist Kubernetes %s file on disk", fileType)
|
||||
return httperror.InternalServerError(errMsg, err)
|
||||
}
|
||||
stack.ProjectPath = projectPath
|
||||
|
||||
doCleanUp := true
|
||||
defer handler.cleanUp(stack, &doCleanUp)
|
||||
|
||||
output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
|
||||
StackID: stackID,
|
||||
StackName: stack.Name,
|
||||
Owner: sanitizeLabel(stack.CreatedBy),
|
||||
Kind: "content",
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to deploy Kubernetes stack", err)
|
||||
}
|
||||
|
||||
err = handler.DataStore.Stack().Create(stack)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to persist the Kubernetes stack inside the database", err)
|
||||
stackBuilderDirector := stackbuilders.NewStackBuilderDirector(k8sStackBuilder)
|
||||
_, httpErr := stackBuilderDirector.Build(&stackPayload, endpoint)
|
||||
if httpErr != nil {
|
||||
return httpErr
|
||||
}
|
||||
|
||||
resp := &createKubernetesStackResponse{
|
||||
Output: output,
|
||||
Output: k8sStackBuilder.GetResponse(),
|
||||
}
|
||||
|
||||
doCleanUp = false
|
||||
return response.JSON(w, resp)
|
||||
}
|
||||
|
||||
|
|
@ -186,7 +178,7 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
|
|||
return httperror.InternalServerError("Unable to check for name collision", err)
|
||||
}
|
||||
if !isUnique {
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
|
||||
}
|
||||
|
||||
//make sure the webhook ID is unique
|
||||
|
|
@ -196,92 +188,40 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
|
|||
return httperror.InternalServerError("Unable to check for webhook ID collision", err)
|
||||
}
|
||||
if !isUnique {
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists}
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: stackutils.ErrWebhookIDAlreadyExists}
|
||||
}
|
||||
}
|
||||
|
||||
stackID := handler.DataStore.Stack().GetNextIdentifier()
|
||||
stack := &portainer.Stack{
|
||||
ID: portainer.StackID(stackID),
|
||||
Type: portainer.KubernetesStack,
|
||||
EndpointID: endpoint.ID,
|
||||
EntryPoint: payload.ManifestFile,
|
||||
GitConfig: &gittypes.RepoConfig{
|
||||
URL: payload.RepositoryURL,
|
||||
ReferenceName: payload.RepositoryReferenceName,
|
||||
ConfigFilePath: payload.ManifestFile,
|
||||
},
|
||||
Namespace: payload.Namespace,
|
||||
Name: payload.StackName,
|
||||
Status: portainer.StackStatusActive,
|
||||
CreationDate: time.Now().Unix(),
|
||||
CreatedBy: user.Username,
|
||||
IsComposeFormat: payload.ComposeFormat,
|
||||
AutoUpdate: payload.AutoUpdate,
|
||||
AdditionalFiles: payload.AdditionalFiles,
|
||||
}
|
||||
stackPayload := createStackPayloadFromK8sGitPayload(payload.StackName,
|
||||
payload.RepositoryURL,
|
||||
payload.RepositoryReferenceName,
|
||||
payload.RepositoryUsername,
|
||||
payload.RepositoryPassword,
|
||||
payload.RepositoryAuthentication,
|
||||
payload.ComposeFormat,
|
||||
payload.Namespace,
|
||||
payload.ManifestFile,
|
||||
payload.AdditionalFiles,
|
||||
payload.AutoUpdate)
|
||||
|
||||
if payload.RepositoryAuthentication {
|
||||
stack.GitConfig.Authentication = &gittypes.GitAuthentication{
|
||||
Username: payload.RepositoryUsername,
|
||||
Password: payload.RepositoryPassword,
|
||||
}
|
||||
}
|
||||
k8sStackBuilder := stackbuilders.CreateKubernetesStackGitBuilder(handler.DataStore,
|
||||
handler.FileService,
|
||||
handler.GitService,
|
||||
handler.Scheduler,
|
||||
handler.StackDeployer,
|
||||
handler.KubernetesDeployer,
|
||||
user)
|
||||
|
||||
projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID)))
|
||||
stack.ProjectPath = projectPath
|
||||
|
||||
doCleanUp := true
|
||||
defer handler.cleanUp(stack, &doCleanUp)
|
||||
|
||||
commitID, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to fetch git repository id", err)
|
||||
}
|
||||
stack.GitConfig.ConfigHash = commitID
|
||||
|
||||
repositoryUsername := payload.RepositoryUsername
|
||||
repositoryPassword := payload.RepositoryPassword
|
||||
if !payload.RepositoryAuthentication {
|
||||
repositoryUsername = ""
|
||||
repositoryPassword = ""
|
||||
}
|
||||
|
||||
err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Failed to clone git repository", err)
|
||||
}
|
||||
|
||||
output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
|
||||
StackID: stackID,
|
||||
StackName: stack.Name,
|
||||
Owner: sanitizeLabel(stack.CreatedBy),
|
||||
Kind: "git",
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to deploy Kubernetes stack", err)
|
||||
}
|
||||
|
||||
if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
|
||||
jobID, e := startAutoupdate(stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
|
||||
stack.AutoUpdate.JobID = jobID
|
||||
}
|
||||
|
||||
err = handler.DataStore.Stack().Create(stack)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to persist the Kubernetes stack inside the database", err)
|
||||
stackBuilderDirector := stackbuilders.NewStackBuilderDirector(k8sStackBuilder)
|
||||
_, httpErr := stackBuilderDirector.Build(&stackPayload, endpoint)
|
||||
if httpErr != nil {
|
||||
return httpErr
|
||||
}
|
||||
|
||||
resp := &createKubernetesStackResponse{
|
||||
Output: output,
|
||||
Output: k8sStackBuilder.GetResponse(),
|
||||
}
|
||||
|
||||
doCleanUp = false
|
||||
return response.JSON(w, resp)
|
||||
}
|
||||
|
||||
|
|
@ -300,58 +240,28 @@ func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWrit
|
|||
return httperror.InternalServerError("Unable to check for name collision", err)
|
||||
}
|
||||
if !isUnique {
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: errStackAlreadyExists}
|
||||
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
|
||||
}
|
||||
|
||||
stackID := handler.DataStore.Stack().GetNextIdentifier()
|
||||
stack := &portainer.Stack{
|
||||
ID: portainer.StackID(stackID),
|
||||
Type: portainer.KubernetesStack,
|
||||
EndpointID: endpoint.ID,
|
||||
EntryPoint: filesystem.ManifestFileDefaultName,
|
||||
Namespace: payload.Namespace,
|
||||
Name: payload.StackName,
|
||||
Status: portainer.StackStatusActive,
|
||||
CreationDate: time.Now().Unix(),
|
||||
CreatedBy: user.Username,
|
||||
IsComposeFormat: payload.ComposeFormat,
|
||||
stackPayload := createStackPayloadFromK8sUrlPayload(payload.StackName,
|
||||
payload.Namespace,
|
||||
payload.ManifestURL,
|
||||
payload.ComposeFormat)
|
||||
|
||||
k8sStackBuilder := stackbuilders.CreateKubernetesStackUrlBuilder(handler.DataStore,
|
||||
handler.FileService,
|
||||
handler.StackDeployer,
|
||||
handler.KubernetesDeployer,
|
||||
user)
|
||||
|
||||
stackBuilderDirector := stackbuilders.NewStackBuilderDirector(k8sStackBuilder)
|
||||
_, httpErr := stackBuilderDirector.Build(&stackPayload, endpoint)
|
||||
if httpErr != nil {
|
||||
return httpErr
|
||||
}
|
||||
|
||||
var manifestContent []byte
|
||||
manifestContent, err = client.Get(payload.ManifestURL, 30)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to retrieve manifest from URL", err)
|
||||
}
|
||||
|
||||
stackFolder := strconv.Itoa(int(stack.ID))
|
||||
projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, manifestContent)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to persist Kubernetes manifest file on disk", err)
|
||||
}
|
||||
stack.ProjectPath = projectPath
|
||||
|
||||
doCleanUp := true
|
||||
defer handler.cleanUp(stack, &doCleanUp)
|
||||
|
||||
output, err := handler.deployKubernetesStack(user.ID, endpoint, stack, k.KubeAppLabels{
|
||||
StackID: stackID,
|
||||
StackName: stack.Name,
|
||||
Owner: stack.CreatedBy,
|
||||
Kind: "url",
|
||||
})
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to deploy Kubernetes stack", err)
|
||||
}
|
||||
|
||||
err = handler.DataStore.Stack().Create(stack)
|
||||
if err != nil {
|
||||
return httperror.InternalServerError("Unable to persist the Kubernetes stack inside the database", err)
|
||||
}
|
||||
|
||||
doCleanUp = false
|
||||
|
||||
resp := &createKubernetesStackResponse{
|
||||
Output: output,
|
||||
Output: k8sStackBuilder.GetResponse(),
|
||||
}
|
||||
|
||||
return response.JSON(w, resp)
|
||||
|
|
@ -361,10 +271,18 @@ func (handler *Handler) deployKubernetesStack(userID portainer.UserID, endpoint
|
|||
handler.stackCreationMutex.Lock()
|
||||
defer handler.stackCreationMutex.Unlock()
|
||||
|
||||
manifestFilePaths, tempDir, err := stackutils.CreateTempK8SDeploymentFiles(stack, handler.KubernetesDeployer, appLabels)
|
||||
user := &portainer.User{
|
||||
ID: userID,
|
||||
}
|
||||
k8sDeploymentConfig, err := deployments.CreateKubernetesStackDeploymentConfig(stack, handler.KubernetesDeployer, appLabels, user, endpoint)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to create temp kub deployment files")
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
return handler.KubernetesDeployer.Deploy(userID, endpoint, manifestFilePaths, stack.Namespace)
|
||||
|
||||
err = k8sDeploymentConfig.Deploy()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return k8sDeploymentConfig.GetResponse(), nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue