feat(extensions): add the ability to upload and enable an extension (#3345)

* feat(extensions): offline mode mockup

* feat(extensions): offline mode mockup

* feat(api): add support for extensionUpload API operation

* feat(extensions): offline extension upload

* feat(api): better support for extensions in offline mode

* feat(extension): update offline description

* feat(api): introduce local extension manifest

* fix(api): fix LocalExtensionManifestFile value

* feat(api): use a 5second timeout for online extension infos

* feat(extensions): add download archive link

* feat(extensions): add support for offline update

* fix(api): fix issues with offline install and online updates of extensions

* fix(extensions): fix extensions link URL

* fix(extension): hide screenshot in offline mode

api/archive/zip.go

@@ -17,32 +17,38 @@ func UnzipArchive(archiveData []byte, dest string) error {
 	}
 
 	for _, zipFile := range zipReader.File {
-
-		f, err := zipFile.Open()
+		err := extractFileFromArchive(zipFile, dest)
 		if err != nil {
 			return err
 		}
-		defer f.Close()
-
-		data, err := ioutil.ReadAll(f)
-		if err != nil {
-			return err
-		}
-
-		fpath := filepath.Join(dest, zipFile.Name)
-
-		outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode())
-		if err != nil {
-			return err
-		}
-
-		_, err = io.Copy(outFile, bytes.NewReader(data))
-		if err != nil {
-			return err
-		}
-
-		outFile.Close()
 	}
 
 	return nil
 }
+
+func extractFileFromArchive(file *zip.File, dest string) error {
+	f, err := file.Open()
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+
+	fpath := filepath.Join(dest, file.Name)
+
+	outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(outFile, bytes.NewReader(data))
+	if err != nil {
+		return err
+	}
+
+	return outFile.Close()
+}

api/exec/extension.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"path"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -20,7 +22,8 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 )
 
-var extensionDownloadBaseURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com/extensions/"
+var extensionDownloadBaseURL = portainer.AssetsServerURL + "/extensions/"
+var extensionVersionRegexp = regexp.MustCompile(`\d+(\.\d+)+`)
 
 var extensionBinaryMap = map[portainer.ExtensionID]string{
 	portainer.RegistryManagementExtension:  "extension-registry-management",
@@ -50,20 +53,11 @@ func processKey(ID portainer.ExtensionID) string {
 }
 
 func buildExtensionURL(extension *portainer.Extension) string {
-	extensionURL := extensionDownloadBaseURL
-	extensionURL += extensionBinaryMap[extension.ID]
-	extensionURL += "-" + runtime.GOOS + "-" + runtime.GOARCH
-	extensionURL += "-" + extension.Version
-	extensionURL += ".zip"
-	return extensionURL
+	return fmt.Sprintf("%s%s-%s-%s-%s.zip", extensionDownloadBaseURL, extensionBinaryMap[extension.ID], runtime.GOOS, runtime.GOARCH, extension.Version)
 }
 
 func buildExtensionPath(binaryPath string, extension *portainer.Extension) string {
-
-	extensionFilename := extensionBinaryMap[extension.ID]
-	extensionFilename += "-" + runtime.GOOS + "-" + runtime.GOARCH
-	extensionFilename += "-" + extension.Version
-
+	extensionFilename := fmt.Sprintf("%s-%s-%s-%s", extensionBinaryMap[extension.ID], runtime.GOOS, runtime.GOARCH, extension.Version)
 	if runtime.GOOS == "windows" {
 		extensionFilename += ".exe"
 	}
@@ -76,11 +70,20 @@ func buildExtensionPath(binaryPath string, extension *portainer.Extension) strin
 }
 
 // FetchExtensionDefinitions will fetch the list of available
-// extension definitions from the official Portainer assets server
+// extension definitions from the official Portainer assets server.
+// If it cannot retrieve the data from the Internet it will fallback to the locally cached
+// manifest file.
 func (manager *ExtensionManager) FetchExtensionDefinitions() ([]portainer.Extension, error) {
-	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 30)
+	var extensionData []byte
+
+	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 5)
 	if err != nil {
-		return nil, err
+		log.Printf("[WARN] [exec,extensions] [message: unable to retrieve extensions manifest via Internet. Extensions will be retrieved from local cache and might not be up to date] [err: %s]", err)
+
+		extensionData, err = manager.fileService.GetFileContent(portainer.LocalExtensionManifestFile)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	var extensions []portainer.Extension
@@ -92,6 +95,37 @@ func (manager *ExtensionManager) FetchExtensionDefinitions() ([]portainer.Extens
 	return extensions, nil
 }
 
+// InstallExtension will install the extension from an archive. It will extract the extension version number from
+// the archive file name first and return an error if the file name is not valid (cannot find extension version).
+// It will then extract the archive and execute the EnableExtension function to enable the extension.
+// Since we're missing information about this extension (stored on Portainer.io server) we need to assume
+// default information based on the extension ID.
+func (manager *ExtensionManager) InstallExtension(extension *portainer.Extension, licenseKey string, archiveFileName string, extensionArchive []byte) error {
+	extensionVersion := extensionVersionRegexp.FindString(archiveFileName)
+	if extensionVersion == "" {
+		return errors.New("invalid extension archive filename: unable to retrieve extension version")
+	}
+
+	err := manager.fileService.ExtractExtensionArchive(extensionArchive)
+	if err != nil {
+		return err
+	}
+
+	switch extension.ID {
+	case portainer.RegistryManagementExtension:
+		extension.Name = "Registry Manager"
+	case portainer.OAuthAuthenticationExtension:
+		extension.Name = "External Authentication"
+	case portainer.RBACExtension:
+		extension.Name = "Role-Based Access Control"
+	}
+	extension.ShortDescription = "Extension enabled offline"
+	extension.Version = extensionVersion
+	extension.Available = true
+
+	return manager.EnableExtension(extension, licenseKey)
+}
+
 // EnableExtension will check for the existence of the extension binary on the filesystem
 // first. If it does not exist, it will download it from the official Portainer assets server.
 // After installing the binary on the filesystem, it will execute the binary in license check
@@ -268,6 +302,7 @@ func (manager *ExtensionManager) startExtensionProcess(extension *portainer.Exte
 
 	err := extensionProcess.Start()
 	if err != nil {
+		log.Printf("[DEBUG] [exec,extension] [message: unable to start extension process] [err: %s]", err)
 		return err
 	}
 

api/filesystem/filesystem.go

@@ -87,12 +87,7 @@ func (service *Service) GetBinaryFolder() string {
 // ExtractExtensionArchive extracts the content of an extension archive
 // specified as raw data into the binary store on the filesystem
 func (service *Service) ExtractExtensionArchive(data []byte) error {
-	err := archive.UnzipArchive(data, path.Join(service.fileStorePath, BinaryStorePath))
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return archive.UnzipArchive(data, path.Join(service.fileStorePath, BinaryStorePath))
 }
 
 // RemoveDirectory removes a directory on the filesystem.

api/http/client/client.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"strings"
@@ -87,6 +88,7 @@ func Get(url string, timeout int) ([]byte, error) {
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusOK {
+		log.Printf("[ERROR] [http,client] [message: unexpected status code] [status_code: %d]", response.StatusCode)
 		return nil, errInvalidResponseStatus
 	}
 

api/http/handler/extensions/extension_inspect.go

@@ -1,15 +1,14 @@
 package extensions
 
 import (
-	"encoding/json"
 	"net/http"
 
-	"github.com/coreos/go-semver/semver"
+	"github.com/portainer/portainer/api/http/client"
+
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/client"
 )
 
 // GET request on /api/extensions/:id
@@ -18,46 +17,39 @@ func (handler *Handler) extensionInspect(w http.ResponseWriter, r *http.Request)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid extension identifier route variable", err}
 	}
+
 	extensionID := portainer.ExtensionID(extensionIdentifier)
 
-	extensionData, err := client.Get(portainer.ExtensionDefinitionsURL, 30)
+	definitions, err := handler.ExtensionManager.FetchExtensionDefinitions()
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extension definitions", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions informations", err}
 	}
 
-	var extensions []portainer.Extension
-	err = json.Unmarshal(extensionData, &extensions)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to parse external extension definitions", err}
+	localExtension, err := handler.ExtensionService.Extension(extensionID)
+	if err != nil && err != portainer.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extension information from the database", err}
 	}
 
 	var extension portainer.Extension
-	for _, p := range extensions {
-		if p.ID == extensionID {
-			extension = p
-			if extension.DescriptionURL != "" {
-				description, _ := client.Get(extension.DescriptionURL, 10)
-				extension.Description = string(description)
-			}
+	var extensionDefinition portainer.Extension
+
+	for _, definition := range definitions {
+		if definition.ID == extensionID {
+			extensionDefinition = definition
 			break
 		}
 	}
 
-	storedExtension, err := handler.ExtensionService.Extension(extensionID)
-	if err == portainer.ErrObjectNotFound {
-		return response.JSON(w, extension)
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a extension with the specified identifier inside the database", err}
+	if localExtension == nil {
+		extension = extensionDefinition
+	} else {
+		extension = *localExtension
 	}
 
-	extension.Enabled = storedExtension.Enabled
+	mergeExtensionAndDefinition(&extension, &extensionDefinition)
 
-	extensionVer := semver.New(extension.Version)
-	pVer := semver.New(storedExtension.Version)
-
-	if pVer.LessThan(*extensionVer) {
-		extension.UpdateAvailable = true
-	}
+	description, _ := client.Get(extension.DescriptionURL, 5)
+	extension.Description = string(description)
 
 	return response.JSON(w, extension)
 }

api/http/handler/extensions/extension_list.go

@@ -3,54 +3,28 @@ package extensions
 import (
 	"net/http"
 
-	"github.com/coreos/go-semver/semver"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
 )
 
 // GET request on /api/extensions?store=<store>
 func (handler *Handler) extensionList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	storeDetails, _ := request.RetrieveBooleanQueryParameter(r, "store", true)
+	fetchManifestInformation, _ := request.RetrieveBooleanQueryParameter(r, "store", true)
 
 	extensions, err := handler.ExtensionService.Extensions()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions from the database", err}
 	}
 
-	if storeDetails {
+	if fetchManifestInformation {
 		definitions, err := handler.ExtensionManager.FetchExtensionDefinitions()
 		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions", err}
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions informations", err}
 		}
 
-		for idx := range definitions {
-			associateExtensionData(&definitions[idx], extensions)
-		}
-
-		extensions = definitions
+		extensions = mergeExtensionsAndDefinitions(extensions, definitions)
 	}
 
 	return response.JSON(w, extensions)
 }
-
-func associateExtensionData(definition *portainer.Extension, extensions []portainer.Extension) {
-	for _, extension := range extensions {
-		if extension.ID == definition.ID {
-
-			definition.Enabled = extension.Enabled
-			definition.License.Company = extension.License.Company
-			definition.License.Expiration = extension.License.Expiration
-			definition.License.Valid = extension.License.Valid
-
-			definitionVersion := semver.New(definition.Version)
-			extensionVersion := semver.New(extension.Version)
-			if extensionVersion.LessThan(*definitionVersion) {
-				definition.UpdateAvailable = true
-			}
-
-			break
-		}
-	}
-}

api/http/handler/extensions/extension_upload.go

@@ -0,0 +1,75 @@
+package extensions
+
+import (
+	"net/http"
+	"strconv"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	"github.com/portainer/portainer/api"
+)
+
+type extensionUploadPayload struct {
+	License          string
+	ExtensionArchive []byte
+	ArchiveFileName  string
+}
+
+func (payload *extensionUploadPayload) Validate(r *http.Request) error {
+	license, err := request.RetrieveMultiPartFormValue(r, "License", false)
+	if err != nil {
+		return portainer.Error("Invalid license")
+	}
+	payload.License = license
+
+	fileData, fileName, err := request.RetrieveMultiPartFormFile(r, "file")
+	if err != nil {
+		return portainer.Error("Invalid extension archive file. Ensure that the file is uploaded correctly")
+	}
+	payload.ExtensionArchive = fileData
+	payload.ArchiveFileName = fileName
+
+	return nil
+}
+
+func (handler *Handler) extensionUpload(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	payload := &extensionUploadPayload{}
+	err := payload.Validate(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
+	}
+
+	extensionIdentifier, err := strconv.Atoi(string(payload.License[0]))
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid license format", err}
+	}
+	extensionID := portainer.ExtensionID(extensionIdentifier)
+
+	extension := &portainer.Extension{
+		ID: extensionID,
+	}
+
+	_ = handler.ExtensionManager.DisableExtension(extension)
+
+	err = handler.ExtensionManager.InstallExtension(extension, payload.License, payload.ArchiveFileName, payload.ExtensionArchive)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to install extension", err}
+	}
+
+	extension.Enabled = true
+
+	if extension.ID == portainer.RBACExtension {
+		err = handler.upgradeRBACData()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "An error occured during database update", err}
+		}
+	}
+
+	err = handler.ExtensionService.Persist(extension)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist extension status inside the database", err}
+	}
+
+	return response.Empty(w)
+}

api/http/handler/extensions/handler.go

@@ -3,6 +3,8 @@ package extensions
 import (
 	"net/http"
 
+	"github.com/coreos/go-semver/semver"
+
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/portainer/api"
@@ -30,6 +32,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.extensionList))).Methods(http.MethodGet)
 	h.Handle("/extensions",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionCreate))).Methods(http.MethodPost)
+	h.Handle("/extensions/upload",
+		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionUpload))).Methods(http.MethodPost)
 	h.Handle("/extensions/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.extensionInspect))).Methods(http.MethodGet)
 	h.Handle("/extensions/{id}",
@@ -39,3 +43,44 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 
 	return h
 }
+
+func mergeExtensionsAndDefinitions(extensions, definitions []portainer.Extension) []portainer.Extension {
+	for _, definition := range definitions {
+		foundInDB := false
+
+		for idx, extension := range extensions {
+			if extension.ID == definition.ID {
+				foundInDB = true
+				mergeExtensionAndDefinition(&extensions[idx], &definition)
+				break
+			}
+		}
+
+		if !foundInDB {
+			extensions = append(extensions, definition)
+		}
+	}
+
+	return extensions
+}
+
+func mergeExtensionAndDefinition(extension, definition *portainer.Extension) {
+	extension.Name = definition.Name
+	extension.ShortDescription = definition.ShortDescription
+	extension.Deal = definition.Deal
+	extension.Available = definition.Available
+	extension.DescriptionURL = definition.DescriptionURL
+	extension.Images = definition.Images
+	extension.Logo = definition.Logo
+	extension.Price = definition.Price
+	extension.PriceDescription = definition.PriceDescription
+	extension.ShopURL = definition.ShopURL
+
+	definitionVersion := semver.New(definition.Version)
+	extensionVersion := semver.New(extension.Version)
+	if extensionVersion.LessThan(*definitionVersion) {
+		extension.UpdateAvailable = true
+	}
+
+	extension.Version = definition.Version
+}

api/portainer.go

@@ -891,6 +891,7 @@ type (
 	// ExtensionManager represents a service used to manage extensions
 	ExtensionManager interface {
 		FetchExtensionDefinitions() ([]Extension, error)
+		InstallExtension(extension *Extension, licenseKey string, archiveFileName string, extensionArchive []byte) error
 		EnableExtension(extension *Extension, licenseKey string) error
 		DisableExtension(extension *Extension) error
 		UpdateExtension(extension *Extension, version string) error
@@ -942,6 +943,8 @@ const (
 	ExtensionServer = "localhost"
 	// DefaultEdgeAgentCheckinIntervalInSeconds represents the default interval (in seconds) used by Edge agents to checkin with the Portainer instance
 	DefaultEdgeAgentCheckinIntervalInSeconds = 5
+	// LocalExtensionManifestFile represents the name of the local manifest file for extensions
+	LocalExtensionManifestFile = "/app/extensions.json"
 )
 
 const (