Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-25 00:09:40 +02:00
Code Issues Releases Activity

Merge pull request #4965 from portainer/feat(backup)-backup-restore-system

Browse source 
feat(backup): Add backup/restore to the server [EE-386] [EE-378] [CE-452]
This commit is contained in:
wheresolivia 2021-04-21 12:16:39 +12:00 • committed by GitHub
commit 335bfb81ba
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
73 changed files with 2807 additions and 567 deletions
Download patch file Download diff file Expand all files Collapse all files

53
api/http/handler/backup/backup.go Normal file
View file

@ -0,0 +1,53 @@
package backup
import (
"fmt"
"net/http"
"os"
"path/filepath"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
operations "github.com/portainer/portainer/api/backup"
)
type (
backupPayload struct {
Password string
}
)
func (p *backupPayload) Validate(r *http.Request) error {
return nil
}
// @id Backup
// @summary Creates an archive with a system data snapshot that could be used to restore the system.
// @description Creates an archive with a system data snapshot that could be used to restore the system.
// @description **Access policy**: admin
// @tags backup
// @security jwt
// @produce octet-stream
// @param Password body string false "Password to encrypt the backup with"
// @success 200 "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /backup [post]
func (h *Handler) backup(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload backupPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
}
archivePath, err := operations.CreateBackupArchive(payload.Password, h.gate, h.dataStore, h.filestorePath)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to create backup", Err: err}
}
defer os.RemoveAll(filepath.Dir(archivePath))
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fmt.Sprintf("portainer-backup_%s", filepath.Base(archivePath))))
http.ServeFile(w, r, archivePath)
return nil
}

122
api/http/handler/backup/backup_test.go Normal file
View file

@ -0,0 +1,122 @@
package backup
import (
"bytes"
"context"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"testing"
"time"
"github.com/docker/docker/pkg/ioutils"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/http/offlinegate"
i "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
)
func listFiles(dir string) []string {
items := make([]string, 0)
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if path == dir {
return nil
}
items = append(items, path)
return nil
})
return items
}
func contains(t *testing.T, list []string, path string) {
assert.Contains(t, list, path)
copyContent, _ := ioutil.ReadFile(path)
assert.Equal(t, "content\n", string(copyContent))
}
func Test_backupHandlerWithoutPassword_shouldCreateATarballArchive(t *testing.T) {
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":""}`))
w := httptest.NewRecorder()
gate := offlinegate.NewOfflineGate()
adminMonitor := adminmonitor.New(time.Hour, nil, context.Background())
handlerErr := NewHandler(nil, i.NewDatastore(), gate, "./test_assets/handler_test", func() {}, adminMonitor).backup(w, r)
assert.Nil(t, handlerErr, "Handler should not fail")
response := w.Result()
body, _ := io.ReadAll(response.Body)
tmpdir, _ := ioutils.TempDir("", "backup")
defer os.RemoveAll(tmpdir)
archivePath := filepath.Join(tmpdir, "archive.tar.gz")
err := ioutil.WriteFile(archivePath, body, 0600)
if err != nil {
t.Fatal("Failed to save downloaded .tar.gz archive: ", err)
}
cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
err = cmd.Run()
if err != nil {
t.Fatal("Failed to extract archive: ", err)
}
createdFiles := listFiles(tmpdir)
contains(t, createdFiles, path.Join(tmpdir, "portainer.key"))
contains(t, createdFiles, path.Join(tmpdir, "portainer.pub"))
contains(t, createdFiles, path.Join(tmpdir, "tls", "file1"))
contains(t, createdFiles, path.Join(tmpdir, "tls", "file2"))
assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_file"))
assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_folder", "file1"))
}
func Test_backupHandlerWithPassword_shouldCreateEncryptedATarballArchive(t *testing.T) {
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"password":"secret"}`))
w := httptest.NewRecorder()
gate := offlinegate.NewOfflineGate()
adminMonitor := adminmonitor.New(time.Hour, nil, nil)
handlerErr := NewHandler(nil, i.NewDatastore(), gate, "./test_assets/handler_test", func() {}, adminMonitor).backup(w, r)
assert.Nil(t, handlerErr, "Handler should not fail")
response := w.Result()
body, _ := io.ReadAll(response.Body)
tmpdir, _ := ioutils.TempDir("", "backup")
defer os.RemoveAll(tmpdir)
dr, err := crypto.AesDecrypt(bytes.NewReader(body), []byte("secret"))
if err != nil {
t.Fatal("Failed to decrypt archive")
}
archivePath := filepath.Join(tmpdir, "archive.tag.gz")
archive, _ := os.Create(archivePath)
defer archive.Close()
io.Copy(archive, dr)
cmd := exec.Command("tar", "-xzf", archivePath, "-C", tmpdir)
err = cmd.Run()
if err != nil {
t.Fatal("Failed to extract archive: ", err)
}
createdFiles := listFiles(tmpdir)
contains(t, createdFiles, path.Join(tmpdir, "portainer.key"))
contains(t, createdFiles, path.Join(tmpdir, "portainer.pub"))
contains(t, createdFiles, path.Join(tmpdir, "tls", "file1"))
contains(t, createdFiles, path.Join(tmpdir, "tls", "file2"))
assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_file"))
assert.NotContains(t, createdFiles, path.Join(tmpdir, "extra_folder", "file1"))
}

65
api/http/handler/backup/handler.go Normal file
View file

@ -0,0 +1,65 @@
package backup
import (
"context"
"net/http"
"github.com/gorilla/mux"
httperror "github.com/portainer/libhttp/error"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/http/offlinegate"
"github.com/portainer/portainer/api/http/security"
)
// Handler is an http handler responsible for backup and restore portainer state
type Handler struct {
*mux.Router
bouncer *security.RequestBouncer
dataStore portainer.DataStore
gate *offlinegate.OfflineGate
filestorePath string
shutdownTrigger context.CancelFunc
adminMonitor *adminmonitor.Monitor
}
// NewHandler creates an new instance of backup handler
func NewHandler(bouncer *security.RequestBouncer, dataStore portainer.DataStore, gate *offlinegate.OfflineGate, filestorePath string, shutdownTrigger context.CancelFunc, adminMonitor *adminmonitor.Monitor) *Handler {
h := &Handler{
Router: mux.NewRouter(),
bouncer: bouncer,
dataStore: dataStore,
gate: gate,
filestorePath: filestorePath,
shutdownTrigger: shutdownTrigger,
adminMonitor: adminMonitor,
}
h.Handle("/backup", bouncer.RestrictedAccess(adminAccess(httperror.LoggerHandler(h.backup)))).Methods(http.MethodPost)
h.Handle("/restore", bouncer.PublicAccess(httperror.LoggerHandler(h.restore))).Methods(http.MethodPost)
return h
}
func adminAccess(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user info from request context", err)
}
if !securityContext.IsAdmin {
httperror.WriteError(w, http.StatusUnauthorized, "User is not authorized to perfom the action", nil)
}
next.ServeHTTP(w, r)
})
}
func systemWasInitialized(dataStore portainer.DataStore) (bool, error) {
users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
if err != nil {
return false, err
}
return len(users) > 0, nil
}

69
api/http/handler/backup/restore.go Normal file
View file

@ -0,0 +1,69 @@
package backup
import (
"bytes"
"io"
"net/http"
"github.com/pkg/errors"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
operations "github.com/portainer/portainer/api/backup"
)
type restorePayload struct {
FileContent []byte
FileName string
Password string
}
// @id Restore
// @summary Triggers a system restore using provided backup file
// @description Triggers a system restore using provided backup file
// @description **Access policy**: public
// @tags backup
// @param FileContent body []byte true "Content of the backup"
// @param FileName body string true "File name"
// @param Password body string false "Password to decrypt the backup with"
// @success 200 "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /restore [post]
func (h *Handler) restore(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
initialized, err := h.adminMonitor.WasInitialized()
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to check system initialization", Err: err}
}
if initialized {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Cannot restore already initialized instance", Err: errors.New("system already initialized")}
}
h.adminMonitor.Stop()
defer h.adminMonitor.Start()
var payload restorePayload
err = decodeForm(r, &payload)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
}
var archiveReader io.Reader = bytes.NewReader(payload.FileContent)
err = operations.RestoreArchive(archiveReader, payload.Password, h.filestorePath, h.gate, h.dataStore, h.shutdownTrigger)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to restore the backup", Err: err}
}
return nil
}
func decodeForm(r *http.Request, p *restorePayload) error {
content, name, err := request.RetrieveMultiPartFormFile(r, "file")
if err != nil {
return err
}
p.FileContent = content
p.FileName = name
password, _ := request.RetrieveMultiPartFormValue(r, "password", true)
p.Password = password
return nil
}

123
api/http/handler/backup/restore_test.go Normal file
View file

@ -0,0 +1,123 @@
package backup
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/adminmonitor"
"github.com/portainer/portainer/api/http/offlinegate"
i "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
)
func Test_restoreArchive_usingCombinationOfPasswords(t *testing.T) {
tests := []struct {
name string
backupPassword string
restorePassword string
fails bool
}{
{
name: "empty password to both encrypt and decrypt",
backupPassword: "",
restorePassword: "",
fails: false,
},
{
name: "same password to encrypt and decrypt",
backupPassword: "secret",
restorePassword: "secret",
fails: false,
},
{
name: "different passwords to encrypt and decrypt",
backupPassword: "secret",
restorePassword: "terces",
fails: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
datastore := i.NewDatastore(i.WithUsers([]portainer.User{}), i.WithEdgeJobs([]portainer.EdgeJob{}))
adminMonitor := adminmonitor.New(time.Hour, datastore, context.Background())
h := NewHandler(nil, datastore, offlinegate.NewOfflineGate(), "./test_assets/handler_test", func() {}, adminMonitor)
//backup
archive := backup(t, h, test.backupPassword)
//restore
w := httptest.NewRecorder()
r, err := prepareMultipartRequest(test.restorePassword, archive)
assert.Nil(t, err, "Shouldn't fail to write multipart form")
restoreErr := h.restore(w, r)
assert.Equal(t, test.fails, restoreErr != nil, "Didn't meet expectation of failing restore handler")
})
}
}
func Test_restoreArchive_shouldFailIfSystemWasAlreadyInitialized(t *testing.T) {
admin := portainer.User{
Role: portainer.AdministratorRole,
}
datastore := i.NewDatastore(i.WithUsers([]portainer.User{admin}), i.WithEdgeJobs([]portainer.EdgeJob{}))
adminMonitor := adminmonitor.New(time.Hour, datastore, context.Background())
h := NewHandler(nil, datastore, offlinegate.NewOfflineGate(), "./test_assets/handler_test", func() {}, adminMonitor)
//backup
archive := backup(t, h, "password")
//restore
w := httptest.NewRecorder()
r, err := prepareMultipartRequest("password", archive)
assert.Nil(t, err, "Shouldn't fail to write multipart form")
restoreErr := h.restore(w, r)
assert.NotNil(t, restoreErr, "Should fail, because system it already initialized")
assert.Equal(t, "Cannot restore already initialized instance", restoreErr.Message, "Should fail with certain error")
}
func backup(t *testing.T, h *Handler, password string) []byte {
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(fmt.Sprintf(`{"password":"%s"}`, password)))
w := httptest.NewRecorder()
backupErr := h.backup(w, r)
assert.Nil(t, backupErr, "Backup should not fail")
response := w.Result()
archive, _ := io.ReadAll(response.Body)
return archive
}
func prepareMultipartRequest(password string, file []byte) (*http.Request, error) {
var body bytes.Buffer
w := multipart.NewWriter(&body)
err := w.WriteField("password", password)
if err != nil {
return nil, err
}
fw, err := w.CreateFormFile("file", "filename")
if err != nil {
return nil, err
}
io.Copy(fw, bytes.NewReader(file))
r := httptest.NewRequest(http.MethodPost, "http://localhost/", &body)
r.Header.Set("Content-Type", w.FormDataContentType())
w.Close()
return r, nil
}

1
api/http/handler/backup/test_assets/handler_test/extra_file Normal file
View file

@ -0,0 +1 @@
content

1
api/http/handler/backup/test_assets/handler_test/extra_folder/file1 Normal file
View file

@ -0,0 +1 @@
content

0
api/http/handler/backup/test_assets/handler_test/portainer.db Normal file
View file

1
api/http/handler/backup/test_assets/handler_test/portainer.key Normal file
View file

@ -0,0 +1 @@
content

1
api/http/handler/backup/test_assets/handler_test/portainer.pub Normal file
View file

@ -0,0 +1 @@
content

1
api/http/handler/backup/test_assets/handler_test/tls/file1 Normal file
View file

@ -0,0 +1 @@
content

1
api/http/handler/backup/test_assets/handler_test/tls/file2 Normal file
View file

@ -0,0 +1 @@
content

6
api/http/handler/handler.go
View file

@ -5,6 +5,7 @@ import (
"strings"
"github.com/portainer/portainer/api/http/handler/auth"
"github.com/portainer/portainer/api/http/handler/backup"
"github.com/portainer/portainer/api/http/handler/customtemplates"
"github.com/portainer/portainer/api/http/handler/dockerhub"
"github.com/portainer/portainer/api/http/handler/edgegroups"
@ -36,6 +37,7 @@ import (
// Handler is a collection of all the service handlers.
type Handler struct {
AuthHandler *auth.Handler
BackupHandler *backup.Handler
CustomTemplatesHandler *customtemplates.Handler
DockerHubHandler *dockerhub.Handler
EdgeGroupsHandler *edgegroups.Handler
@ -140,6 +142,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/api/auth"):
http.StripPrefix("/api", h.AuthHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/backup"):
http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/restore"):
http.StripPrefix("/api", h.BackupHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/dockerhub"):
http.StripPrefix("/api", h.DockerHubHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/custom_templates"):