feat(edge/templates): introduce edge app templates [EE-6209] (#10480)

api/datastore/init.go

@@ -53,7 +53,7 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			},
 			SnapshotInterval:         portainer.DefaultSnapshotInterval,
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-			TemplatesURL:             portainer.DefaultTemplatesURL,
+			TemplatesURL:             "",
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
 			UserSessionTimeout:       portainer.DefaultUserSessionTimeout,
 			KubeconfigExpiry:         portainer.DefaultKubeconfigExpiry,

api/datastore/migrator/migrate_dbversion100.go

@@ -10,8 +10,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-func (m *Migrator) migrateDockerDesktopExtentionSetting() error {
-	log.Info().Msg("updating docker desktop extention flag in settings")
+func (m *Migrator) migrateDockerDesktopExtensionSetting() error {
+	log.Info().Msg("updating docker desktop extension flag in settings")
 
 	isDDExtension := false
 	if _, ok := os.LookupEnv("DOCKER_EXTENSION"); ok {

api/datastore/migrator/migrate_dbversion110.go

@@ -0,0 +1,25 @@
+package migrator
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
+)
+
+// updateAppTemplatesVersionForDB110 changes the templates URL to be empty if it was never changed
+// from the default value (version 2.0 URL)
+func (migrator *Migrator) updateAppTemplatesVersionForDB110() error {
+	log.Info().Msg("updating app templates url to v3.0")
+
+	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
+
+	settings, err := migrator.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if settings.TemplatesURL == version2URL || settings.TemplatesURL == portainer.DefaultTemplatesURL {
+		settings.TemplatesURL = ""
+	}
+
+	return migrator.settingsService.UpdateSettings(settings)
+}

api/datastore/migrator/migrate_dbversion24.go

@@ -14,8 +14,10 @@ func (m *Migrator) updateSettingsToDB25() error {
 		return err
 	}
 
+	// to keep the same migration functionality as before 2.20.0, we need to set the templates URL to v2
+	version2URL := "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
 	if legacySettings.TemplatesURL == "" {
-		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
+		legacySettings.TemplatesURL = version2URL
 	}
 
 	legacySettings.UserSessionTimeout = portainer.DefaultUserSessionTimeout

api/datastore/migrator/migrator.go

@@ -225,10 +225,14 @@ func (m *Migrator) initMigrations() {
 	m.addMigrations("2.18", m.migrateDBVersionToDB90)
 	m.addMigrations("2.19",
 		m.convertSeedToPrivateKeyForDB100,
-		m.migrateDockerDesktopExtentionSetting,
+		m.migrateDockerDesktopExtensionSetting,
 		m.updateEdgeStackStatusForDB100,
 	)
 
+	m.addMigrations("2.20",
+		m.updateAppTemplatesVersionForDB110,
+	)
+
 	// Add new migrations below...
 	// One function per migration, each versions migration funcs in the same file.
 }

api/datastore/test_data/output_24_to_latest.json

@@ -645,7 +645,7 @@
     },
     "ShowKomposeBuildOption": false,
     "SnapshotInterval": "5m",
-    "TemplatesURL": "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json",
+    "TemplatesURL": "",
     "TrustOnFirstConnect": false,
     "UserSessionTimeout": "8h",
     "fdoConfiguration": {
@@ -936,6 +936,6 @@
     }
   ],
   "version": {
-    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":0,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
+    "VERSION": "{\"SchemaVersion\":\"2.20.0\",\"MigratorCount\":1,\"Edition\":1,\"InstanceID\":\"463d5c47-0ea5-4aca-85b1-405ceefee254\"}"
   }
 }

api/http/handler/edgestacks/edgestack_create_git.go

@@ -31,7 +31,7 @@ type edgeStackFromGitRepositoryPayload struct {
 	// Path to the Stack file inside the Git repository
 	FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
 	// List of identifiers of EdgeGroups
-	EdgeGroups []portainer.EdgeGroupID `example:"1"`
+	EdgeGroups []portainer.EdgeGroupID `example:"1" validate:"required"`
 	// Deployment type to deploy this stack
 	// Valid values are: 0 - 'compose', 1 - 'kubernetes'
 	// compose is enabled only for docker environments
@@ -85,7 +85,6 @@ func (payload *edgeStackFromGitRepositoryPayload) Validate(r *http.Request) erro
 // @security ApiKeyAuth
 // @security jwt
 // @produce json
-// @param method query string true "Creation Method" Enums(file,string,repository)
 // @param body body edgeStackFromGitRepositoryPayload true "stack config"
 // @param dryrun query string false "if true, will not create an edge stack, but just will check the settings and return a non-persisted edge stack object"
 // @success 200 {object} portainer.EdgeStack

api/http/handler/edgetemplates/edgetemplate_list.go

@@ -2,6 +2,7 @@ package edgetemplates
 
 import (
 	"net/http"
+	"slices"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/client"
@@ -51,10 +52,16 @@ func (handler *Handler) edgeTemplateList(w http.ResponseWriter, r *http.Request)
 		return httperror.InternalServerError("Unable to parse template file", err)
 	}
 
+	// We only support version 3 of the template format
+	// this is only a temporary fix until we have custom edge templates
+	if templateFile.Version != "3" {
+		return httperror.InternalServerError("Unsupported template version", nil)
+	}
+
 	filteredTemplates := make([]portainer.Template, 0)
 
 	for _, template := range templateFile.Templates {
-		if template.Type == portainer.EdgeStackTemplate {
+		if slices.Contains(template.Categories, "edge") && slices.Contains([]portainer.TemplateType{portainer.ComposeStackTemplate, portainer.SwarmStackTemplate}, template.Type) {
 			filteredTemplates = append(filteredTemplates, template)
 		}
 	}

api/http/handler/templates/handler.go

@@ -26,8 +26,10 @@ func NewHandler(bouncer security.BouncerService) *Handler {
 	}
 
 	h.Handle("/templates",
-		bouncer.RestrictedAccess(httperror.LoggerHandler(h.templateList))).Methods(http.MethodGet)
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.templateList))).Methods(http.MethodGet)
+	h.Handle("/templates/{id}/file",
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.templateFile))).Methods(http.MethodPost)
 	h.Handle("/templates/file",
-		bouncer.RestrictedAccess(httperror.LoggerHandler(h.templateFile))).Methods(http.MethodPost)
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.templateFileOld))).Methods(http.MethodPost)
 	return h
 }

api/http/handler/templates/template_file.go

@@ -1,72 +1,22 @@
 package templates
 
 import (
-	"errors"
 	"net/http"
+	"slices"
 
 	portainer "github.com/portainer/portainer/api"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 	"github.com/portainer/portainer/pkg/libhttp/response"
 
-	"github.com/asaskevich/govalidator"
 	"github.com/rs/zerolog/log"
-	"github.com/segmentio/encoding/json"
 )
 
-type filePayload struct {
-	// URL of a git repository where the file is stored
-	RepositoryURL string `example:"https://github.com/portainer/portainer-compose" validate:"required"`
-	// Path to the file inside the git repository
-	ComposeFilePathInRepository string `example:"./subfolder/docker-compose.yml" validate:"required"`
-}
-
 type fileResponse struct {
 	// The requested file content
 	FileContent string `example:"version:2"`
 }
 
-func (payload *filePayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.RepositoryURL) {
-		return errors.New("Invalid repository url")
-	}
-
-	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
-		return errors.New("Invalid file path")
-	}
-
-	return nil
-}
-
-func (handler *Handler) ifRequestedTemplateExists(payload *filePayload) *httperror.HandlerError {
-	settings, err := handler.DataStore.Settings().Settings()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve settings from the database", err)
-	}
-
-	resp, err := http.Get(settings.TemplatesURL)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve templates via the network", err)
-	}
-	defer resp.Body.Close()
-
-	var templates struct {
-		Templates []portainer.Template
-	}
-	err = json.NewDecoder(resp.Body).Decode(&templates)
-	if err != nil {
-		return httperror.InternalServerError("Unable to parse template file", err)
-	}
-
-	for _, t := range templates.Templates {
-		if t.Repository.URL == payload.RepositoryURL && t.Repository.StackFile == payload.ComposeFilePathInRepository {
-			return nil
-		}
-	}
-
-	return httperror.InternalServerError("Invalid template", errors.New("requested template does not exist"))
-}
-
 // @id TemplateFile
 // @summary Get a template's file
 // @description Get a template's file
@@ -76,21 +26,42 @@ func (handler *Handler) ifRequestedTemplateExists(payload *filePayload) *httperr
 // @security jwt
 // @accept json
 // @produce json
-// @param body body filePayload true "File details"
+// @param id path int true "Template identifier"
 // @success 200 {object} fileResponse "Success"
 // @failure 400 "Invalid request"
 // @failure 500 "Server error"
-// @router /templates/file [post]
+// @router /templates/{id}/file [post]
 func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload filePayload
-
-	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	id, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
-		return httperror.BadRequest("Invalid request payload", err)
+		return httperror.BadRequest("Invalid template identifier", err)
 	}
 
-	if err := handler.ifRequestedTemplateExists(&payload); err != nil {
-		return err
+	templatesResponse, httpErr := handler.fetchTemplates()
+	if httpErr != nil {
+		return httpErr
+	}
+
+	templateIdx := slices.IndexFunc(templatesResponse.Templates, func(template portainer.Template) bool {
+		return template.ID == portainer.TemplateID(id)
+	})
+
+	if templateIdx == -1 {
+		return httperror.NotFound("Unable to find a template with the specified identifier", nil)
+	}
+
+	template := templatesResponse.Templates[templateIdx]
+
+	if template.Type == portainer.ContainerTemplate {
+		return httperror.BadRequest("Invalid template type", nil)
+	}
+
+	if template.StackFile != "" {
+		return response.JSON(w, fileResponse{FileContent: template.StackFile})
+	}
+
+	if template.Repository.StackFile == "" || template.Repository.URL == "" {
+		return httperror.BadRequest("Invalid template configuration", nil)
 	}
 
 	projectPath, err := handler.FileService.GetTemporaryPath()
@@ -100,12 +71,12 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht
 
 	defer handler.cleanUp(projectPath)
 
-	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "", false)
+	err = handler.GitService.CloneRepository(projectPath, template.Repository.URL, "", "", "", false)
 	if err != nil {
 		return httperror.InternalServerError("Unable to clone git repository", err)
 	}
 
-	fileContent, err := handler.FileService.GetFileContent(projectPath, payload.ComposeFilePathInRepository)
+	fileContent, err := handler.FileService.GetFileContent(projectPath, template.Repository.StackFile)
 	if err != nil {
 		return httperror.InternalServerError("Failed loading file content", err)
 	}

api/http/handler/templates/template_file_old.go

@@ -0,0 +1,95 @@
+package templates
+
+import (
+	"errors"
+	"net/http"
+
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/request"
+	"github.com/portainer/portainer/pkg/libhttp/response"
+	"github.com/rs/zerolog/log"
+
+	"github.com/asaskevich/govalidator"
+)
+
+type filePayload struct {
+	// URL of a git repository where the file is stored
+	RepositoryURL string `example:"https://github.com/portainer/portainer-compose" validate:"required"`
+	// Path to the file inside the git repository
+	ComposeFilePathInRepository string `example:"./subfolder/docker-compose.yml" validate:"required"`
+}
+
+func (payload *filePayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.RepositoryURL) {
+		return errors.New("Invalid repository url")
+	}
+
+	if govalidator.IsNull(payload.ComposeFilePathInRepository) {
+		return errors.New("Invalid file path")
+	}
+
+	return nil
+}
+
+func (handler *Handler) ifRequestedTemplateExists(payload *filePayload) *httperror.HandlerError {
+	response, httpErr := handler.fetchTemplates()
+	if httpErr != nil {
+		return httpErr
+	}
+
+	for _, t := range response.Templates {
+		if t.Repository.URL == payload.RepositoryURL && t.Repository.StackFile == payload.ComposeFilePathInRepository {
+			return nil
+		}
+	}
+	return httperror.InternalServerError("Invalid template", errors.New("requested template does not exist"))
+}
+
+// @id TemplateFileOld
+// @summary Get a template's file
+// @deprecated
+// @description Get a template's file
+// @description **Access policy**: authenticated
+// @tags templates
+// @security ApiKeyAuth
+// @security jwt
+// @accept json
+// @produce json
+// @param body body filePayload true "File details"
+// @success 200 {object} fileResponse "Success"
+// @failure 400 "Invalid request"
+// @failure 500 "Server error"
+// @router /templates/file [post]
+func (handler *Handler) templateFileOld(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	log.Warn().Msg("This api is deprecated. Please use /templates/{id}/file instead")
+
+	var payload filePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return httperror.BadRequest("Invalid request payload", err)
+	}
+
+	if err := handler.ifRequestedTemplateExists(&payload); err != nil {
+		return err
+	}
+
+	projectPath, err := handler.FileService.GetTemporaryPath()
+	if err != nil {
+		return httperror.InternalServerError("Unable to create temporary folder", err)
+	}
+
+	defer handler.cleanUp(projectPath)
+
+	err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "", false)
+	if err != nil {
+		return httperror.InternalServerError("Unable to clone git repository", err)
+	}
+
+	fileContent, err := handler.FileService.GetFileContent(projectPath, payload.ComposeFilePathInRepository)
+	if err != nil {
+		return httperror.InternalServerError("Failed loading file content", err)
+	}
+
+	return response.JSON(w, fileResponse{FileContent: string(fileContent)})
+
+}

api/http/handler/templates/template_list.go

@@ -1,19 +1,12 @@
 package templates
 
 import (
-	"io"
 	"net/http"
 
-	portainer "github.com/portainer/portainer/api"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/portainer/portainer/pkg/libhttp/response"
 )
 
-// introduced for swagger
-type listResponse struct {
-	Version   string
-	Templates []portainer.Template
-}
-
 // @id TemplateList
 // @summary List available templates
 // @description List available templates.
@@ -26,22 +19,10 @@ type listResponse struct {
 // @failure 500 "Server error"
 // @router /templates [get]
 func (handler *Handler) templateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	settings, err := handler.DataStore.Settings().Settings()
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve settings from the database", err)
+	templates, httpErr := handler.fetchTemplates()
+	if httpErr != nil {
+		return httpErr
 	}
 
-	resp, err := http.Get(settings.TemplatesURL)
-	if err != nil {
-		return httperror.InternalServerError("Unable to retrieve templates via the network", err)
-	}
-	defer resp.Body.Close()
-
-	w.Header().Set("Content-Type", "application/json")
-	_, err = io.Copy(w, resp.Body)
-	if err != nil {
-		return httperror.InternalServerError("Unable to write templates from templates URL", err)
-	}
-
-	return nil
+	return response.JSON(w, templates)
 }

api/http/handler/templates/utils_fetch_templates.go

@@ -0,0 +1,41 @@
+package templates
+
+import (
+	"net/http"
+
+	portainer "github.com/portainer/portainer/api"
+	httperror "github.com/portainer/portainer/pkg/libhttp/error"
+	"github.com/segmentio/encoding/json"
+)
+
+type listResponse struct {
+	Version   string               `json:"version"`
+	Templates []portainer.Template `json:"templates"`
+}
+
+func (handler *Handler) fetchTemplates() (*listResponse, *httperror.HandlerError) {
+	settings, err := handler.DataStore.Settings().Settings()
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
+	}
+
+	templatesURL := settings.TemplatesURL
+	if templatesURL == "" {
+		templatesURL = portainer.DefaultTemplatesURL
+	}
+
+	resp, err := http.Get(templatesURL)
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to retrieve templates via the network", err)
+	}
+	defer resp.Body.Close()
+
+	var body *listResponse
+	err = json.NewDecoder(resp.Body).Decode(&body)
+	if err != nil {
+		return nil, httperror.InternalServerError("Unable to parse template file", err)
+	}
+
+	return body, nil
+
+}

api/portainer.go

@@ -1153,7 +1153,7 @@ type (
 	Template struct {
 		// Mandatory container/stack fields
 		// Template Identifier
-		ID TemplateID `json:"Id" example:"1"`
+		ID TemplateID `json:"id" example:"1"`
 		// Template type. Valid values are: 1 (container), 2 (Swarm stack), 3 (Compose stack), 4 (Compose edge stack)
 		Type TemplateType `json:"type" example:"1"`
 		// Title of the template
@@ -1614,7 +1614,7 @@ const (
 	// DefaultEdgeAgentCheckinIntervalInSeconds represents the default interval (in seconds) used by Edge agents to checkin with the Portainer instance
 	DefaultEdgeAgentCheckinIntervalInSeconds = 5
 	// DefaultTemplatesURL represents the URL to the official templates supported by Portainer
-	DefaultTemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json"
+	DefaultTemplatesURL = "https://raw.githubusercontent.com/portainer/templates/v3.0/templates.json"
 	// DefaultHelmrepositoryURL represents the URL to the official templates supported by Bitnami
 	DefaultHelmRepositoryURL = "https://charts.bitnami.com/bitnami"
 	// DefaultUserSessionTimeout represents the default timeout after which the user session is cleared
@@ -1829,8 +1829,6 @@ const (
 	SwarmStackTemplate
 	// ComposeStackTemplate represents a template used to deploy a Compose stack
 	ComposeStackTemplate
-	// EdgeStackTemplate represents a template used to deploy an Edge stack
-	EdgeStackTemplate
 )
 
 const (