feat(extensions): add the ability to upload and enable an extension (#3345)

* feat(extensions): offline mode mockup

* feat(extensions): offline mode mockup

* feat(api): add support for extensionUpload API operation

* feat(extensions): offline extension upload

* feat(api): better support for extensions in offline mode

* feat(extension): update offline description

* feat(api): introduce local extension manifest

* fix(api): fix LocalExtensionManifestFile value

* feat(api): use a 5second timeout for online extension infos

* feat(extensions): add download archive link

* feat(extensions): add support for offline update

* fix(api): fix issues with offline install and online updates of extensions

* fix(extensions): fix extensions link URL

* fix(extension): hide screenshot in offline mode

api/http/client/client.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
@@ -87,6 +88,7 @@ func Get(url string, timeout int) ([]byte, error) {
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
+		log.Printf("[ERROR] [http,client] [message: unexpected status code] [status_code: %d]", response.StatusCode)
 		return nil, errInvalidResponseStatus
 	}
 

api/http/handler/extensions/extension_inspect.go

@@ -1,15 +1,14 @@
 package extensions
 
 import (
-	"encoding/json"
 	"net/http"
 
-	"github.com/coreos/go-semver/semver"
+	"github.com/portainer/portainer/api/http/client"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/client"
 )
 
 // GET request on /api/extensions/:id
@@ -18,46 +17,39 @@ func (handler *Handler) extensionInspect(w http.ResponseWriter, r *http.Request)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid extension identifier route variable", err}
 	}
+
 	extensionID := portainer.ExtensionID(extensionIdentifier)
 
-	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 30)
+	definitions, err := handler.ExtensionManager.FetchExtensionDefinitions()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extension definitions", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions informations", err}
 	}
 
-	var extensions []portainer.Extension
-	err = json.Unmarshal(extensionData, &extensions)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to parse external extension definitions", err}
+	localExtension, err := handler.ExtensionService.Extension(extensionID)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extension information from the database", err}
 	}
 
 	var extension portainer.Extension
-	for _, p := range extensions {
-		if p.ID == extensionID {
-			extension = p
-			if extension.DescriptionURL != "" {
-				description, _ := client.Get(extension.DescriptionURL, 10)
-				extension.Description = string(description)
-			}
+	var extensionDefinition portainer.Extension
+
+	for _, definition := range definitions {
+		if definition.ID == extensionID {
+			extensionDefinition = definition
 			break
 		}
 	}
 
-	storedExtension, err := handler.ExtensionService.Extension(extensionID)
-	if err == portainer.ErrObjectNotFound {
-		return response.JSON(w, extension)
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a extension with the specified identifier inside the database", err}
+	if localExtension == nil {
+		extension = extensionDefinition
+	} else {
+		extension = *localExtension
 	}
 
-	extension.Enabled = storedExtension.Enabled
+	mergeExtensionAndDefinition(&extension, &extensionDefinition)
 
-	extensionVer := semver.New(extension.Version)
-	pVer := semver.New(storedExtension.Version)
-
-	if pVer.LessThan(*extensionVer) {
-		extension.UpdateAvailable = true
-	}
+	description, _ := client.Get(extension.DescriptionURL, 5)
+	extension.Description = string(description)
 
 	return response.JSON(w, extension)
 }

api/http/handler/extensions/extension_list.go

@@ -3,54 +3,28 @@ package extensions
 import (
 	"net/http"
 
-	"github.com/coreos/go-semver/semver"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
 )
 
 // GET request on /api/extensions?store=<store>
 func (handler *Handler) extensionList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	storeDetails, _ := request.RetrieveBooleanQueryParameter(r, "store", true)
+	fetchManifestInformation, _ := request.RetrieveBooleanQueryParameter(r, "store", true)
 
 	extensions, err := handler.ExtensionService.Extensions()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions from the database", err}
 	}
 
-	if storeDetails {
+	if fetchManifestInformation {
 		definitions, err := handler.ExtensionManager.FetchExtensionDefinitions()
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions informations", err}
 		}
 
-		for idx := range definitions {
-			associateExtensionData(&definitions[idx], extensions)
-		}
-
-		extensions = definitions
+		extensions = mergeExtensionsAndDefinitions(extensions, definitions)
 	}
 
 	return response.JSON(w, extensions)
 }
-
-func associateExtensionData(definition *portainer.Extension, extensions []portainer.Extension) {
-	for _, extension := range extensions {
-		if extension.ID == definition.ID {
-
-			definition.Enabled = extension.Enabled
-			definition.License.Company = extension.License.Company
-			definition.License.Expiration = extension.License.Expiration
-			definition.License.Valid = extension.License.Valid
-
-			definitionVersion := semver.New(definition.Version)
-			extensionVersion := semver.New(extension.Version)
-			if extensionVersion.LessThan(*definitionVersion) {
-				definition.UpdateAvailable = true
-			}
-
-			break
-		}
-	}
-}

api/http/handler/extensions/extension_upload.go

@@ -0,0 +1,75 @@
+package extensions
+
+import (
+	"net/http"
+	"strconv"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+type extensionUploadPayload struct {
+	License          string
+	ExtensionArchive []byte
+	ArchiveFileName  string
+}
+
+func (payload *extensionUploadPayload) Validate(r *http.Request) error {
+	license, err := request.RetrieveMultiPartFormValue(r, "License", false)
+	if err != nil {
+		return portainer.Error("Invalid license")
+	}
+	payload.License = license
+
+	fileData, fileName, err := request.RetrieveMultiPartFormFile(r, "file")
+	if err != nil {
+		return portainer.Error("Invalid extension archive file. Ensure that the file is uploaded correctly")
+	}
+	payload.ExtensionArchive = fileData
+	payload.ArchiveFileName = fileName
+
+	return nil
+}
+
+func (handler *Handler) extensionUpload(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	payload := &extensionUploadPayload{}
+	err := payload.Validate(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	extensionIdentifier, err := strconv.Atoi(string(payload.License[0]))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid license format", err}
+	}
+	extensionID := portainer.ExtensionID(extensionIdentifier)
+
+	extension := &portainer.Extension{
+		ID: extensionID,
+	}
+
+	_ = handler.ExtensionManager.DisableExtension(extension)
+
+	err = handler.ExtensionManager.InstallExtension(extension, payload.License, payload.ArchiveFileName, payload.ExtensionArchive)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to install extension", err}
+	}
+
+	extension.Enabled = true
+
+	if extension.ID == portainer.RBACExtension {
+		err = handler.upgradeRBACData()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "An error occured during database update", err}
+		}
+	}
+
+	err = handler.ExtensionService.Persist(extension)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist extension status inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/extensions/handler.go

@@ -3,6 +3,8 @@ package extensions
 import (
 	"net/http"
 
+	"github.com/coreos/go-semver/semver"
+
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/portainer/api"
@@ -30,6 +32,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.extensionList))).Methods(http.MethodGet)
 	h.Handle("/extensions",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionCreate))).Methods(http.MethodPost)
+	h.Handle("/extensions/upload",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionUpload))).Methods(http.MethodPost)
 	h.Handle("/extensions/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionInspect))).Methods(http.MethodGet)
 	h.Handle("/extensions/{id}",
@@ -39,3 +43,44 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 
 	return h
 }
+
+func mergeExtensionsAndDefinitions(extensions, definitions []portainer.Extension) []portainer.Extension {
+	for _, definition := range definitions {
+		foundInDB := false
+
+		for idx, extension := range extensions {
+			if extension.ID == definition.ID {
+				foundInDB = true
+				mergeExtensionAndDefinition(&extensions[idx], &definition)
+				break
+			}
+		}
+
+		if !foundInDB {
+			extensions = append(extensions, definition)
+		}
+	}
+
+	return extensions
+}
+
+func mergeExtensionAndDefinition(extension, definition *portainer.Extension) {
+	extension.Name = definition.Name
+	extension.ShortDescription = definition.ShortDescription
+	extension.Deal = definition.Deal
+	extension.Available = definition.Available
+	extension.DescriptionURL = definition.DescriptionURL
+	extension.Images = definition.Images
+	extension.Logo = definition.Logo
+	extension.Price = definition.Price
+	extension.PriceDescription = definition.PriceDescription
+	extension.ShopURL = definition.ShopURL
+
+	definitionVersion := semver.New(definition.Version)
+	extensionVersion := semver.New(extension.Version)
+	if extensionVersion.LessThan(*definitionVersion) {
+		extension.UpdateAvailable = true
+	}
+
+	extension.Version = definition.Version
+}