feat(templates): introduce templates management (#2017)

api/http/client/client.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -61,6 +62,27 @@ func (client *HTTPClient) ExecuteAzureAuthenticationRequest(credentials *portain
 	return &token, nil
 }
 
+// Get executes a simple HTTP GET to the specified URL and returns
+// the content of the response body.
+func Get(url string) ([]byte, error) {
+	client := &http.Client{
+		Timeout: time.Second * 3,
+	}
+
+	response, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return body, nil
+}
+
 // ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment
 // using the specified host and optional TLS configuration.
 // It uses a new Http.Client for each operation.

api/http/handler/settings/settings_public.go

@@ -10,7 +10,6 @@ import (
 
 type publicSettingsResponse struct {
 	LogoURL                            string                         `json:"LogoURL"`
-	DisplayExternalContributors        bool                           `json:"DisplayExternalContributors"`
 	AuthenticationMethod               portainer.AuthenticationMethod `json:"AuthenticationMethod"`
 	AllowBindMountsForRegularUsers     bool                           `json:"AllowBindMountsForRegularUsers"`
 	AllowPrivilegedModeForRegularUsers bool                           `json:"AllowPrivilegedModeForRegularUsers"`
@@ -25,7 +24,6 @@ func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *
 
 	publicSettings := &publicSettingsResponse{
 		LogoURL:                            settings.LogoURL,
-		DisplayExternalContributors:        settings.DisplayExternalContributors,
 		AuthenticationMethod:               settings.AuthenticationMethod,
 		AllowBindMountsForRegularUsers:     settings.AllowBindMountsForRegularUsers,
 		AllowPrivilegedModeForRegularUsers: settings.AllowPrivilegedModeForRegularUsers,

api/http/handler/settings/settings_update.go

@@ -12,10 +12,8 @@ import (
 )
 
 type settingsUpdatePayload struct {
-	TemplatesURL                       string
 	LogoURL                            string
 	BlackListedLabels                  []portainer.Pair
-	DisplayExternalContributors        bool
 	AuthenticationMethod               int
 	LDAPSettings                       portainer.LDAPSettings
 	AllowBindMountsForRegularUsers     bool
@@ -23,9 +21,6 @@ type settingsUpdatePayload struct {
 }
 
 func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
-	if govalidator.IsNull(payload.TemplatesURL) || !govalidator.IsURL(payload.TemplatesURL) {
-		return portainer.Error("Invalid templates URL. Must correspond to a valid URL format")
-	}
 	if payload.AuthenticationMethod == 0 {
 		return portainer.Error("Invalid authentication method")
 	}
@@ -47,10 +42,8 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 	}
 
 	settings := &portainer.Settings{
-		TemplatesURL:                       payload.TemplatesURL,
 		LogoURL:                            payload.LogoURL,
 		BlackListedLabels:                  payload.BlackListedLabels,
-		DisplayExternalContributors:        payload.DisplayExternalContributors,
 		LDAPSettings:                       payload.LDAPSettings,
 		AllowBindMountsForRegularUsers:     payload.AllowBindMountsForRegularUsers,
 		AllowPrivilegedModeForRegularUsers: payload.AllowPrivilegedModeForRegularUsers,

api/http/handler/stacks/stack_file.go

@@ -54,5 +54,5 @@ func (handler *Handler) stackFile(w http.ResponseWriter, r *http.Request) *httpe
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Compose file from disk", err}
 	}
 
-	return response.JSON(w, &stackFileResponse{StackFileContent: stackFileContent})
+	return response.JSON(w, &stackFileResponse{StackFileContent: string(stackFileContent)})
 }

api/http/handler/tags/tag_delete.go

@@ -9,7 +9,7 @@ import (
 	"github.com/portainer/portainer/http/response"
 )
 
-// DELETE request on /api/tags/:name
+// DELETE request on /api/tags/:id
 func (handler *Handler) tagDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	id, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {

api/http/handler/templates/handler.go

@@ -9,14 +9,10 @@ import (
 	"github.com/portainer/portainer/http/security"
 )
 
-const (
-	containerTemplatesURLLinuxServerIo = "https://tools.linuxserver.io/portainer.json"
-)
-
 // Handler represents an HTTP API handler for managing templates.
 type Handler struct {
 	*mux.Router
-	SettingsService portainer.SettingsService
+	TemplateService portainer.TemplateService
 }
 
 // NewHandler returns a new instance of Handler.
@@ -25,6 +21,14 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router: mux.NewRouter(),
 	}
 	h.Handle("/templates",
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.templateList))).Methods(http.MethodGet)
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.templateList))).Methods(http.MethodGet)
+	h.Handle("/templates",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateCreate))).Methods(http.MethodPost)
+	h.Handle("/templates/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateInspect))).Methods(http.MethodGet)
+	h.Handle("/templates/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateUpdate))).Methods(http.MethodPut)
+	h.Handle("/templates/{id}",
+		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateDelete))).Methods(http.MethodDelete)
 	return h
 }

api/http/handler/templates/template_create.go

@@ -0,0 +1,122 @@
+package templates
+
+import (
+	"net/http"
+
+	"github.com/asaskevich/govalidator"
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/filesystem"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type templateCreatePayload struct {
+	// Mandatory
+	Type              int
+	Title             string
+	Description       string
+	AdministratorOnly bool
+
+	// Opt stack/container
+	Name       string
+	Logo       string
+	Note       string
+	Platform   string
+	Categories []string
+	Env        []portainer.TemplateEnv
+
+	// Mandatory container
+	Image string
+
+	// Mandatory stack
+	Repository portainer.TemplateRepository
+
+	// Opt container
+	Registry      string
+	Command       string
+	Network       string
+	Volumes       []portainer.TemplateVolume
+	Ports         []string
+	Labels        []portainer.Pair
+	Privileged    bool
+	Interactive   bool
+	RestartPolicy string
+	Hostname      string
+}
+
+func (payload *templateCreatePayload) Validate(r *http.Request) error {
+	if payload.Type == 0 || (payload.Type != 1 && payload.Type != 2 && payload.Type != 3) {
+		return portainer.Error("Invalid template type. Valid values are: 1 (container), 2 (Swarm stack template) or 3 (Compose stack template).")
+	}
+	if govalidator.IsNull(payload.Title) {
+		return portainer.Error("Invalid template title")
+	}
+	if govalidator.IsNull(payload.Description) {
+		return portainer.Error("Invalid template description")
+	}
+
+	if payload.Type == 1 {
+		if govalidator.IsNull(payload.Image) {
+			return portainer.Error("Invalid template image")
+		}
+	}
+
+	if payload.Type == 2 || payload.Type == 3 {
+		if govalidator.IsNull(payload.Repository.URL) {
+			return portainer.Error("Invalid template repository URL")
+		}
+		if govalidator.IsNull(payload.Repository.StackFile) {
+			payload.Repository.StackFile = filesystem.ComposeFileDefaultName
+		}
+	}
+
+	return nil
+}
+
+// POST request on /api/templates
+func (handler *Handler) templateCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	var payload templateCreatePayload
+	err := request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	template := &portainer.Template{
+		Type:              portainer.TemplateType(payload.Type),
+		Title:             payload.Title,
+		Description:       payload.Description,
+		AdministratorOnly: payload.AdministratorOnly,
+		Name:              payload.Name,
+		Logo:              payload.Logo,
+		Note:              payload.Note,
+		Platform:          payload.Platform,
+		Categories:        payload.Categories,
+		Env:               payload.Env,
+	}
+
+	if template.Type == portainer.ContainerTemplate {
+		template.Image = payload.Image
+		template.Registry = payload.Registry
+		template.Command = payload.Command
+		template.Network = payload.Network
+		template.Volumes = payload.Volumes
+		template.Ports = payload.Ports
+		template.Labels = payload.Labels
+		template.Privileged = payload.Privileged
+		template.Interactive = payload.Interactive
+		template.RestartPolicy = payload.RestartPolicy
+		template.Hostname = payload.Hostname
+	}
+
+	if template.Type == portainer.SwarmStackTemplate || template.Type == portainer.ComposeStackTemplate {
+		template.Repository = payload.Repository
+	}
+
+	err = handler.TemplateService.CreateTemplate(template)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the template inside the database", err}
+	}
+
+	return response.JSON(w, template)
+}

api/http/handler/templates/template_delete.go

@@ -0,0 +1,25 @@
+package templates
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// DELETE request on /api/templates/:id
+func (handler *Handler) templateDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	id, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid template identifier route variable", err}
+	}
+
+	err = handler.TemplateService.DeleteTemplate(portainer.TemplateID(id))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the template from the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/templates/template_inspect.go

@@ -0,0 +1,27 @@
+package templates
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+// GET request on /api/templates/:id
+func (handler *Handler) templateInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	templateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid template identifier route variable", err}
+	}
+
+	template, err := handler.TemplateService.Template(portainer.TemplateID(templateID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a template with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a template with the specified identifier inside the database", err}
+	}
+
+	return response.JSON(w, template)
+}

api/http/handler/templates/template_list.go

@@ -1,50 +1,26 @@
 package templates
 
 import (
-	"io/ioutil"
 	"net/http"
 
 	httperror "github.com/portainer/portainer/http/error"
-	"github.com/portainer/portainer/http/request"
 	"github.com/portainer/portainer/http/response"
+	"github.com/portainer/portainer/http/security"
 )
 
-// GET request on /api/templates?key=<key>
+// GET request on /api/templates
 func (handler *Handler) templateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	key, err := request.RetrieveQueryParameter(r, "key", false)
+	templates, err := handler.TemplateService.Templates()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: key", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve templates from the database", err}
 	}
 
-	templatesURL, templateErr := handler.retrieveTemplateURLFromKey(key)
-	if templateErr != nil {
-		return templateErr
-	}
-
-	resp, err := http.Get(templatesURL)
+	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve templates via the network", err}
-	}
-	defer resp.Body.Close()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to read template response", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
 	}
 
-	return response.Bytes(w, body, "application/json")
-}
-
-func (handler *Handler) retrieveTemplateURLFromKey(key string) (string, *httperror.HandlerError) {
-	switch key {
-	case "containers":
-		settings, err := handler.SettingsService.Settings()
-		if err != nil {
-			return "", &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
-		}
-		return settings.TemplatesURL, nil
-	case "linuxserver.io":
-		return containerTemplatesURLLinuxServerIo, nil
-	}
-	return "", &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: key. Value must be one of: containers or linuxserver.io", request.ErrInvalidQueryParameter}
+	filteredTemplates := security.FilterTemplates(templates, securityContext)
+
+	return response.JSON(w, filteredTemplates)
 }

api/http/handler/templates/template_update.go

@@ -0,0 +1,164 @@
+package templates
+
+import (
+	"net/http"
+
+	"github.com/portainer/portainer"
+	httperror "github.com/portainer/portainer/http/error"
+	"github.com/portainer/portainer/http/request"
+	"github.com/portainer/portainer/http/response"
+)
+
+type templateUpdatePayload struct {
+	Title             *string
+	Description       *string
+	AdministratorOnly *bool
+	Name              *string
+	Logo              *string
+	Note              *string
+	Platform          *string
+	Categories        []string
+	Env               []portainer.TemplateEnv
+	Image             *string
+	Registry          *string
+	Repository        portainer.TemplateRepository
+	Command           *string
+	Network           *string
+	Volumes           []portainer.TemplateVolume
+	Ports             []string
+	Labels            []portainer.Pair
+	Privileged        *bool
+	Interactive       *bool
+	RestartPolicy     *string
+	Hostname          *string
+}
+
+func (payload *templateUpdatePayload) Validate(r *http.Request) error {
+	return nil
+}
+
+// PUT request on /api/templates/:id
+func (handler *Handler) templateUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	templateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid template identifier route variable", err}
+	}
+
+	template, err := handler.TemplateService.Template(portainer.TemplateID(templateID))
+	if err == portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a template with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a template with the specified identifier inside the database", err}
+	}
+
+	var payload templateUpdatePayload
+	err = request.DecodeAndValidateJSONPayload(r, &payload)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	updateTemplate(template, &payload)
+
+	err = handler.TemplateService.UpdateTemplate(template.ID, template)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to persist template changes inside the database", err}
+	}
+
+	return response.JSON(w, template)
+}
+
+func updateContainerProperties(template *portainer.Template, payload *templateUpdatePayload) {
+	if payload.Image != nil {
+		template.Image = *payload.Image
+	}
+
+	if payload.Registry != nil {
+		template.Registry = *payload.Registry
+	}
+
+	if payload.Command != nil {
+		template.Command = *payload.Command
+	}
+
+	if payload.Network != nil {
+		template.Network = *payload.Network
+	}
+
+	if payload.Volumes != nil {
+		template.Volumes = payload.Volumes
+	}
+
+	if payload.Ports != nil {
+		template.Ports = payload.Ports
+	}
+
+	if payload.Labels != nil {
+		template.Labels = payload.Labels
+	}
+
+	if payload.Privileged != nil {
+		template.Privileged = *payload.Privileged
+	}
+
+	if payload.Interactive != nil {
+		template.Interactive = *payload.Interactive
+	}
+
+	if payload.RestartPolicy != nil {
+		template.RestartPolicy = *payload.RestartPolicy
+	}
+
+	if payload.Hostname != nil {
+		template.Hostname = *payload.Hostname
+	}
+}
+
+func updateStackProperties(template *portainer.Template, payload *templateUpdatePayload) {
+	if payload.Repository.URL != "" && payload.Repository.StackFile != "" {
+		template.Repository = payload.Repository
+	}
+}
+
+func updateTemplate(template *portainer.Template, payload *templateUpdatePayload) {
+	if payload.Title != nil {
+		template.Title = *payload.Title
+	}
+
+	if payload.Description != nil {
+		template.Description = *payload.Description
+	}
+
+	if payload.Name != nil {
+		template.Name = *payload.Name
+	}
+
+	if payload.Logo != nil {
+		template.Logo = *payload.Logo
+	}
+
+	if payload.Note != nil {
+		template.Note = *payload.Note
+	}
+
+	if payload.Platform != nil {
+		template.Platform = *payload.Platform
+	}
+
+	if payload.Categories != nil {
+		template.Categories = payload.Categories
+	}
+
+	if payload.Env != nil {
+		template.Env = payload.Env
+	}
+
+	if payload.AdministratorOnly != nil {
+		template.AdministratorOnly = *payload.AdministratorOnly
+	}
+
+	if template.Type == portainer.ContainerTemplate {
+		updateContainerProperties(template, payload)
+	} else if template.Type == portainer.SwarmStackTemplate || template.Type == portainer.ComposeStackTemplate {
+		updateStackProperties(template, payload)
+	}
+}

api/http/response/response.go

@@ -23,10 +23,3 @@ func Empty(rw http.ResponseWriter) *httperror.HandlerError {
 	rw.WriteHeader(http.StatusNoContent)
 	return nil
 }
-
-// Bytes write data into rw. It also allows to set the Content-Type header.
-func Bytes(rw http.ResponseWriter, data []byte, contentType string) *httperror.HandlerError {
-	rw.Header().Set("Content-Type", contentType)
-	rw.Write(data)
-	return nil
-}

api/http/security/filter.go

@@ -77,6 +77,24 @@ func FilterRegistries(registries []portainer.Registry, context *RestrictedReques
 	return filteredRegistries
 }
 
+// FilterTemplates filters templates based on the user role.
+// Non-administrato template do not have access to templates where the AdministratorOnly flag is set to true.
+func FilterTemplates(templates []portainer.Template, context *RestrictedRequestContext) []portainer.Template {
+	filteredTemplates := templates
+
+	if !context.IsAdmin {
+		filteredTemplates = make([]portainer.Template, 0)
+
+		for _, template := range templates {
+			if !template.AdministratorOnly {
+				filteredTemplates = append(filteredTemplates, template)
+			}
+		}
+	}
+
+	return filteredTemplates
+}
+
 // FilterEndpoints filters endpoints based on user role and team memberships.
 // Non administrator users only have access to authorized endpoints (can be inherited via endoint groups).
 func FilterEndpoints(endpoints []portainer.Endpoint, groups []portainer.EndpointGroup, context *RestrictedRequestContext) []portainer.Endpoint {

api/http/server.go

@@ -55,6 +55,7 @@ type Server struct {
 	TagService             portainer.TagService
 	TeamService            portainer.TeamService
 	TeamMembershipService  portainer.TeamMembershipService
+	TemplateService        portainer.TemplateService
 	UserService            portainer.UserService
 	Handler                *handler.Handler
 	SSL                    bool
@@ -143,7 +144,7 @@ func (server *Server) Start() error {
 	var statusHandler = status.NewHandler(requestBouncer, server.Status)
 
 	var templatesHandler = templates.NewHandler(requestBouncer)
-	templatesHandler.SettingsService = server.SettingsService
+	templatesHandler.TemplateService = server.TemplateService
 
 	var uploadHandler = upload.NewHandler(requestBouncer)
 	uploadHandler.FileService = server.FileService