feat(templates): allow managing git based templates [EE-2600] (#7855)

Co-authored-by: itsconquest <william.conquest@portainer.io>
Co-authored-by: oscarzhou <oscar.zhou@portainer.io>
Co-authored-by: Chaim Lev-Ari <chiptus@users.noreply.github.com>

api/http/handler/customtemplates/customtemplate_create.go

@@ -3,7 +3,6 @@ package customtemplates
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"net/http"
 	"os"
 	"regexp"
@@ -18,6 +17,7 @@ import (
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 	"github.com/rs/zerolog/log"
 )
 
@@ -135,6 +135,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
 	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
+	// Platform validation is only for docker related stack (docker standalone and docker swarm)
 	if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
 		return errors.New("Invalid custom template type")
 	}
@@ -215,6 +216,8 @@ type customTemplateFromGitRepositoryPayload struct {
 	Variables []portainer.CustomTemplateVariableDefinition
 	// TLSSkipVerify skips SSL verification when cloning the Git repository
 	TLSSkipVerify bool `example:"false"`
+	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
+	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@@ -234,14 +237,11 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
 		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
 	}
 
-	if payload.Type == portainer.KubernetesStack {
-		return errors.New("Creating a Kubernetes custom template from git is not supported")
-	}
-
-	if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
+	// Platform validation is only for docker related stack (docker standalone and docker swarm)
+	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}
-	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
+	if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack {
 		return errors.New("Invalid custom template type")
 	}
 	if !isValidNote(payload.Note) {
@@ -260,35 +260,44 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 
 	customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
 	customTemplate := &portainer.CustomTemplate{
-		ID:          portainer.CustomTemplateID(customTemplateID),
-		Title:       payload.Title,
-		EntryPoint:  payload.ComposeFilePathInRepository,
-		Description: payload.Description,
-		Note:        payload.Note,
-		Platform:    payload.Platform,
-		Type:        payload.Type,
-		Logo:        payload.Logo,
-		Variables:   payload.Variables,
+		ID:              portainer.CustomTemplateID(customTemplateID),
+		Title:           payload.Title,
+		Description:     payload.Description,
+		Note:            payload.Note,
+		Platform:        payload.Platform,
+		Type:            payload.Type,
+		Logo:            payload.Logo,
+		Variables:       payload.Variables,
+		IsComposeFormat: payload.IsComposeFormat,
 	}
 
-	projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
+	getProjectPath := func() string {
+		return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
+	}
+	projectPath := getProjectPath()
 	customTemplate.ProjectPath = projectPath
 
-	repositoryUsername := payload.RepositoryUsername
-	repositoryPassword := payload.RepositoryPassword
-	if !payload.RepositoryAuthentication {
-		repositoryUsername = ""
-		repositoryPassword = ""
+	gitConfig := &gittypes.RepoConfig{
+		URL:            payload.RepositoryURL,
+		ReferenceName:  payload.RepositoryReferenceName,
+		ConfigFilePath: payload.ComposeFilePathInRepository,
 	}
 
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword, payload.TLSSkipVerify)
-	if err != nil {
-		if err == gittypes.ErrAuthenticationFailure {
-			return nil, fmt.Errorf("invalid git credential")
+	if payload.RepositoryAuthentication {
+		gitConfig.Authentication = &gittypes.GitAuthentication{
+			Username: payload.RepositoryUsername,
+			Password: payload.RepositoryPassword,
 		}
+	}
+
+	commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath)
+	if err != nil {
 		return nil, err
 	}
 
+	gitConfig.ConfigHash = commitHash
+	customTemplate.GitConfig = gitConfig
+
 	isValidProject := true
 	defer func() {
 		if !isValidProject {
@@ -298,7 +307,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 		}
 	}()
 
-	entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint)
+	entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath)
 
 	exists, err := handler.FileService.FileExists(entryPath)
 	if err != nil || !exists {
@@ -310,6 +319,9 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
 	}
 
 	if !exists {
+		if payload.Type == portainer.KubernetesStack {
+			return nil, errors.New("Invalid Manifest file, ensure that the Manifest file path is correct")
+		}
 		return nil, errors.New("Invalid Compose file, ensure that the Compose file path is correct")
 	}
 
@@ -369,6 +381,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
 
 	platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true)
 	templatePlatform := portainer.CustomTemplatePlatform(platform)
+	// Platform validation is only for docker related stack (docker standalone and docker swarm)
 	if templateType != portainer.KubernetesStack && templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
 	}

api/http/handler/customtemplates/customtemplate_file.go

@@ -40,7 +40,11 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques
 		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
 	}
 
-	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint)
+	entryPath := customTemplate.EntryPoint
+	if customTemplate.GitConfig != nil {
+		entryPath = customTemplate.GitConfig.ConfigFilePath
+	}
+	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath)
 	if err != nil {
 		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
 	}

api/http/handler/customtemplates/customtemplate_git_fetch.go

@@ -0,0 +1,124 @@
+package customtemplates
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"sync"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/stacks/stackutils"
+	"github.com/rs/zerolog/log"
+)
+
+// @id CustomTemplateGitFetch
+// @summary Fetch the latest config file content based on custom template's git repository configuration
+// @description Retrieve details about a template created from git repository method.
+// @description **Access policy**: authenticated
+// @tags custom_templates
+// @security ApiKeyAuth
+// @security jwt
+// @produce json
+// @param id path int true "Template identifier"
+// @success 200 {object} fileResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 404 "Custom template not found"
+// @failure 500 "Server error"
+// @router /custom_templates/{id}/git_fetch [put]
+func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return httperror.BadRequest("Invalid Custom template identifier route variable", err)
+	}
+
+	customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID))
+	if handler.DataStore.IsErrObjectNotFound(err) {
+		return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
+	} else if err != nil {
+		return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
+	}
+
+	if customTemplate.GitConfig == nil {
+		return httperror.BadRequest("Git configuration does not exist in this custom template", err)
+	}
+
+	// If multiple users are trying to fetch the same custom template simultaneously, a lock needs to be added
+	mu, ok := handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)]
+	if !ok {
+		mu = &sync.Mutex{}
+		handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] = mu
+	}
+	mu.Lock()
+	defer mu.Unlock()
+
+	// back up the current custom template folder
+	backupPath, err := backupCustomTemplate(customTemplate.ProjectPath)
+	if err != nil {
+		return httperror.InternalServerError("Failed to backup the custom template folder", err)
+	}
+
+	// remove backup custom template folder
+	defer cleanUpBackupCustomTemplate(backupPath)
+
+	commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string {
+		return customTemplate.ProjectPath
+	})
+	if err != nil {
+		log.Warn().Err(err).Msg("failed to download git repository")
+		rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
+		if err != nil {
+			return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr)
+		}
+		return httperror.InternalServerError("Failed to download git repository", err)
+	}
+
+	if customTemplate.GitConfig.ConfigHash != commitHash {
+		customTemplate.GitConfig.ConfigHash = commitHash
+
+		err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
+		}
+	}
+
+	fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.GitConfig.ConfigFilePath)
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
+	}
+
+	return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
+}
+
+func backupCustomTemplate(projectPath string) (string, error) {
+	stat, err := os.Stat(projectPath)
+	if err != nil {
+		return "", err
+	}
+
+	backupPath := fmt.Sprintf("%s-backup", projectPath)
+	err = os.Rename(projectPath, backupPath)
+	if err != nil {
+		return "", err
+	}
+
+	err = os.Mkdir(projectPath, stat.Mode())
+	if err != nil {
+		return backupPath, err
+	}
+	return backupPath, nil
+}
+
+func rollbackCustomTemplate(backupPath, projectPath string) error {
+	err := os.RemoveAll(projectPath)
+	if err != nil {
+		return err
+	}
+	return os.Rename(backupPath, projectPath)
+}
+
+func cleanUpBackupCustomTemplate(backupPath string) error {
+	return os.RemoveAll(backupPath)
+}

api/http/handler/customtemplates/customtemplate_git_fetch_test.go

@@ -0,0 +1,174 @@
+package customtemplates
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	gittypes "github.com/portainer/portainer/api/git/types"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/internal/authorization"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/stretchr/testify/assert"
+)
+
+var testFileContent string = "abcdefg"
+
+type TestGitService struct {
+	portainer.GitService
+	targetFilePath string
+}
+
+func (g *TestGitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string, tlsSkipVerify bool) error {
+	time.Sleep(100 * time.Millisecond)
+	return createTestFile(g.targetFilePath)
+}
+
+func (g *TestGitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
+	return "", nil
+}
+
+type TestFileService struct {
+	portainer.FileService
+}
+
+func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]byte, error) {
+	return os.ReadFile(filepath.Join(projectPath, configFilePath))
+}
+
+func createTestFile(targetPath string) error {
+	f, err := os.Create(targetPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = f.WriteString(testFileContent)
+	return err
+}
+
+func prepareTestFolder(projectPath, filename string) error {
+	err := os.MkdirAll(projectPath, fs.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	return createTestFile(filepath.Join(projectPath, filename))
+}
+
+func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect string) {
+	type response struct {
+		FileContent string
+	}
+
+	req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
+
+	rr := httptest.NewRecorder()
+	h.ServeHTTP(rr, req)
+
+	is.Equal(http.StatusOK, rr.Code)
+
+	body, err := io.ReadAll(rr.Body)
+	is.NoError(err, "ReadAll should not return error")
+
+	var resp response
+	err = json.Unmarshal(body, &resp)
+	is.NoError(err, "response should be list json")
+	is.Equal(resp.FileContent, expect)
+}
+
+func Test_customTemplateGitFetch(t *testing.T) {
+	is := assert.New(t)
+
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
+	defer teardown()
+
+	// create user(s)
+	user1 := &portainer.User{ID: 1, Username: "user-1", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
+	err := store.User().Create(user1)
+	is.NoError(err, "error creating user 1")
+
+	user2 := &portainer.User{ID: 2, Username: "user-2", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
+	err = store.User().Create(user2)
+	is.NoError(err, "error creating user 2")
+
+	dir, err := os.Getwd()
+	is.NoError(err, "error to get working directory")
+
+	template1 := &portainer.CustomTemplate{ID: 1, Title: "custom-template-1", ProjectPath: filepath.Join(dir, "fixtures/custom_template_1"), GitConfig: &gittypes.RepoConfig{ConfigFilePath: "test-config-path.txt"}}
+	err = store.CustomTemplateService.Create(template1)
+	is.NoError(err, "error creating custom template 1")
+
+	// prepare testing folder
+	err = prepareTestFolder(template1.ProjectPath, template1.GitConfig.ConfigFilePath)
+	is.NoError(err, "error creating testing folder")
+
+	defer os.RemoveAll(filepath.Join(dir, "fixtures"))
+
+	// setup services
+	jwtService, err := jwt.NewService("1h", store)
+	is.NoError(err, "Error initiating jwt service")
+	requestBouncer := security.NewRequestBouncer(store, jwtService, nil)
+
+	gitService := &TestGitService{
+		targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath),
+	}
+	fileService := &TestFileService{}
+
+	h := NewHandler(requestBouncer, store, fileService, gitService)
+
+	// generate two standard users' tokens
+	jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
+	jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
+
+	t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
+		singleAPIRequest(h, jwt1, is, "abcdefg")
+	})
+
+	t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) {
+		var wg sync.WaitGroup
+		wg.Add(5)
+		for i := 0; i < 5; i++ {
+			go func() {
+				singleAPIRequest(h, jwt1, is, "abcdefg")
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+	})
+
+	t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) {
+		var wg sync.WaitGroup
+		wg.Add(10)
+		for i := 0; i < 10; i++ {
+			go func(j int) {
+				if j%1 == 0 {
+					singleAPIRequest(h, jwt1, is, "abcdefg")
+				} else {
+					singleAPIRequest(h, jwt2, is, "abcdefg")
+				}
+				wg.Done()
+			}(i)
+		}
+		wg.Wait()
+	})
+
+	t.Run("can return the expected file content after a new commit is made", func(t *testing.T) {
+		singleAPIRequest(h, jwt1, is, "abcdefg")
+
+		testFileContent = "gfedcba"
+
+		singleAPIRequest(h, jwt2, is, "gfedcba")
+	})
+}

api/http/handler/customtemplates/customtemplate_update.go

@@ -10,8 +10,11 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	gittypes "github.com/portainer/portainer/api/git/types"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 )
 
 type customTemplateUpdatePayload struct {
@@ -29,18 +32,37 @@ type customTemplateUpdatePayload struct {
 	Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
 	// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
 	Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
+	// URL of a Git repository hosting the Stack file
+	RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
+	// Reference name of a Git repository hosting the Stack file
+	RepositoryReferenceName string `example:"refs/heads/master"`
+	// Use basic authentication to clone the Git repository
+	RepositoryAuthentication bool `example:"true"`
+	// Username used in basic authentication. Required when RepositoryAuthentication is true
+	// and RepositoryGitCredentialID is 0
+	RepositoryUsername string `example:"myGitUsername"`
+	// Password used in basic authentication. Required when RepositoryAuthentication is true
+	// and RepositoryGitCredentialID is 0
+	RepositoryPassword string `example:"myGitPassword"`
+	// GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication
+	// is true and RepositoryUsername/RepositoryPassword are not provided
+	RepositoryGitCredentialID int `example:"0"`
+	// Path to the Stack file inside the Git repository
+	ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// Content of stack file
 	FileContent string `validate:"required"`
 	// Definitions of variables in the stack file
 	Variables []portainer.CustomTemplateVariableDefinition
+	// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
+	IsComposeFormat bool `example:"false"`
 }
 
 func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Title) {
 		return errors.New("Invalid custom template title")
 	}
-	if govalidator.IsNull(payload.FileContent) {
-		return errors.New("Invalid file content")
+	if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) {
+		return errors.New("Either file content or git repository url need to be provided")
 	}
 	if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
 		return errors.New("Invalid custom template platform")
@@ -55,7 +77,19 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
 		return errors.New("Invalid note. <img> tag is not supported")
 	}
 
-	return validateVariablesDefinitions(payload.Variables)
+	if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
+		return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
+	}
+	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
+		payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
+	}
+
+	err := validateVariablesDefinitions(payload.Variables)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // @id CustomTemplateUpdate
@@ -115,12 +149,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 		return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
 	}
 
-	templateFolder := strconv.Itoa(customTemplateID)
-	_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
-	if err != nil {
-		return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
-	}
-
 	customTemplate.Title = payload.Title
 	customTemplate.Logo = payload.Logo
 	customTemplate.Description = payload.Description
@@ -128,6 +156,42 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
 	customTemplate.Platform = payload.Platform
 	customTemplate.Type = payload.Type
 	customTemplate.Variables = payload.Variables
+	customTemplate.IsComposeFormat = payload.IsComposeFormat
+
+	if payload.RepositoryURL != "" {
+		if !govalidator.IsURL(payload.RepositoryURL) {
+			return httperror.BadRequest("Invalid repository URL. Must correspond to a valid URL format", err)
+		}
+
+		gitConfig := &gittypes.RepoConfig{
+			URL:            payload.RepositoryURL,
+			ReferenceName:  payload.RepositoryReferenceName,
+			ConfigFilePath: payload.ComposeFilePathInRepository,
+		}
+
+		if payload.RepositoryAuthentication {
+			gitConfig.Authentication = &gittypes.GitAuthentication{
+				Username: payload.RepositoryUsername,
+				Password: payload.RepositoryPassword,
+			}
+		}
+
+		commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string {
+			return customTemplate.ProjectPath
+		})
+		if err != nil {
+			return httperror.InternalServerError(err.Error(), err)
+		}
+
+		gitConfig.ConfigHash = commitHash
+		customTemplate.GitConfig = gitConfig
+	} else {
+		templateFolder := strconv.Itoa(customTemplateID)
+		_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
+		if err != nil {
+			return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
+		}
+	}
 
 	err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
 	if err != nil {

api/http/handler/customtemplates/handler.go

@@ -2,6 +2,7 @@ package customtemplates
 
 import (
 	"net/http"
+	"sync"
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
@@ -13,15 +14,20 @@ import (
 // Handler is the HTTP handler used to handle environment(endpoint) group operations.
 type Handler struct {
 	*mux.Router
-	DataStore   dataservices.DataStore
-	FileService portainer.FileService
-	GitService  portainer.GitService
+	DataStore      dataservices.DataStore
+	FileService    portainer.FileService
+	GitService     portainer.GitService
+	gitFetchMutexs map[portainer.TemplateID]*sync.Mutex
 }
 
 // NewHandler creates a handler to manage environment(endpoint) group operations.
-func NewHandler(bouncer *security.RequestBouncer) *Handler {
+func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, fileService portainer.FileService, gitService portainer.GitService) *Handler {
 	h := &Handler{
-		Router: mux.NewRouter(),
+		Router:         mux.NewRouter(),
+		DataStore:      dataStore,
+		FileService:    fileService,
+		GitService:     gitService,
+		gitFetchMutexs: make(map[portainer.TemplateID]*sync.Mutex),
 	}
 	h.Handle("/custom_templates",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateCreate))).Methods(http.MethodPost)
@@ -35,6 +41,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut)
 	h.Handle("/custom_templates/{id}",
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete)
+	h.Handle("/custom_templates/{id}/git_fetch",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut)
 	return h
 }
 

api/http/handler/gitops/git_repo_file_preview.go

@@ -0,0 +1,89 @@
+package gitops
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	gittypes "github.com/portainer/portainer/api/git/types"
+)
+
+type fileResponse struct {
+	FileContent string
+}
+
+type repositoryFilePreviewPayload struct {
+	Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"`
+	Reference  string `json:"reference" example:"refs/heads/master"`
+	Username   string `json:"username" example:"myGitUsername"`
+	Password   string `json:"password" example:"myGitPassword"`
+	// Path to file whose content will be read
+	TargetFile string `json:"targetFile" example:"docker-compose.yml"`
+	// TLSSkipVerify skips SSL verification when cloning the Git repository
+	TLSSkipVerify bool `example:"false"`
+}
+
+func (payload *repositoryFilePreviewPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.Repository) || !govalidator.IsURL(payload.Repository) {
+		return errors.New("Invalid repository URL. Must correspond to a valid URL format")
+	}
+
+	if govalidator.IsNull(payload.Reference) {
+		payload.Reference = "refs/heads/main"
+	}
+
+	if govalidator.IsNull(payload.TargetFile) {
+		return errors.New("Invalid target filename.")
+	}
+
+	return nil
+}
+
+// @id GitOperationRepoFilePreview
+// @summary preview the content of target file in the git repository
+// @description Retrieve the compose file content based on git repository configuration
+// @description **Access policy**: authenticated
+// @tags gitops
+// @security ApiKeyAuth
+// @security jwt
+// @produce json
+// @param body body repositoryFilePreviewPayload true "Template details"
+// @success 200 {object} fileResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /gitops/repo/file/preview [post]
+func (handler *Handler) gitOperationRepoFilePreview(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload repositoryFilePreviewPayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	projectPath, err := handler.fileService.GetTemporaryPath()
+	if err != nil {
+		return httperror.InternalServerError("Unable to create temporary folder", err)
+	}
+
+	err = handler.gitService.CloneRepository(projectPath, payload.Repository, payload.Reference, payload.Username, payload.Password, payload.TLSSkipVerify)
+	if err != nil {
+		if err == gittypes.ErrAuthenticationFailure {
+			return httperror.BadRequest("Invalid git credential", err)
+		}
+
+		newErr := fmt.Errorf("unable to clone git repository: %w", err)
+		return httperror.InternalServerError(newErr.Error(), newErr)
+	}
+
+	defer handler.fileService.RemoveDirectory(projectPath)
+
+	fileContent, err := handler.fileService.GetFileContent(projectPath, payload.TargetFile)
+	if err != nil {
+		return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
+	}
+
+	return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
+}

api/http/handler/gitops/handler.go

@@ -0,0 +1,33 @@
+package gitops
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// Handler is the HTTP handler used to handle git repo operation
+type Handler struct {
+	*mux.Router
+	dataStore   dataservices.DataStore
+	gitService  portainer.GitService
+	fileService portainer.FileService
+}
+
+func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService) *Handler {
+	h := &Handler{
+		Router:      mux.NewRouter(),
+		dataStore:   dataStore,
+		gitService:  gitService,
+		fileService: fileService,
+	}
+
+	h.Handle("/gitops/repo/file/preview",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.gitOperationRepoFilePreview))).Methods(http.MethodPost)
+
+	return h
+}

api/http/handler/handler.go

@@ -17,6 +17,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
+	"github.com/portainer/portainer/api/http/handler/gitops"
 	"github.com/portainer/portainer/api/http/handler/helm"
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/fdo"
 	"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
@@ -56,6 +57,7 @@ type Handler struct {
 	EndpointHandler        *endpoints.Handler
 	EndpointHelmHandler    *helm.Handler
 	EndpointProxyHandler   *endpointproxy.Handler
+	GitOperationHandler    *gitops.Handler
 	HelmTemplatesHandler   *helm.Handler
 	KubernetesHandler      *kubernetes.Handler
 	FileHandler            *file.Handler
@@ -121,6 +123,8 @@ type Handler struct {
 // @tag.description Manage Docker environments(endpoints)
 // @tag.name endpoint_groups
 // @tag.description Manage environment(endpoint) groups
+// @tag.name gitops
+// @tag.description Operate git repository
 // @tag.name kubernetes
 // @tag.description Manage Kubernetes cluster
 // @tag.name motd
@@ -203,6 +207,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		default:
 			http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r)
 		}
+	case strings.HasPrefix(r.URL.Path, "/api/gitops"):
+		http.StripPrefix("/api", h.GitOperationHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/ldap"):
 		http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/motd"):

api/http/handler/stacks/create_kubernetes_stack.go

@@ -23,14 +23,17 @@ type kubernetesStringDeploymentPayload struct {
 	ComposeFormat    bool
 	Namespace        string
 	StackFileContent string
+	// Whether the stack is from a app template
+	FromAppTemplate bool `example:"false"`
 }
 
-func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload {
+func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat, fromAppTemplate bool) stackbuilders.StackPayload {
 	return stackbuilders.StackPayload{
 		StackName:        name,
 		Namespace:        namespace,
 		StackFileContent: fileContent,
 		ComposeFormat:    composeFormat,
+		FromAppTemplate:  fromAppTemplate,
 	}
 }
 
@@ -146,7 +149,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
 		return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
 	}
 
-	stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat)
+	stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate)
 
 	k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore,
 		handler.FileService,