fix(backups): improved archive encryption [EE-6764] (#11489)

api/crypto/aes.go

@@ -1,52 +1,216 @@
 package crypto
 
 import (
+	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
 	"io"
 
+	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/scrypt"
 )
 
-// NOTE: has to go with what is considered to be a simplistic in that it omits any
-// authentication of the encrypted data.
-// Person with better knowledge is welcomed to improve it.
-// sourced from https://golang.org/src/crypto/cipher/example_test.go
+const (
+	// AES GCM settings
+	aesGcmHeader    = "AES256-GCM" // The encrypted file header
+	aesGcmBlockSize = 1024 * 1024  // 1MB block for aes gcm
 
-var emptySalt []byte = make([]byte, 0)
+	// Argon2 settings
+	// Recommded settings lower memory hardware according to current OWASP recommendations
+	// Considering some people run portainer on a NAS I think it's prudent not to assume we're on server grade hardware
+	// https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id
+	argon2MemoryCost = 12 * 1024
+	argon2TimeCost   = 3
+	argon2Threads    = 1
+	argon2KeyLength  = 32
+)
 
-// AesEncrypt reads from input, encrypts with AES-256 and writes to the output.
-// passphrase is used to generate an encryption key.
+// AesEncrypt reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key
 func AesEncrypt(input io.Reader, output io.Writer, passphrase []byte) error {
-	// making a 32 bytes key that would correspond to AES-256
-	// don't necessarily need a salt, so just kept in empty
-	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
+	err := aesEncryptGCM(input, output, passphrase)
 	if err != nil {
-		return err
-	}
-
-	block, err := aes.NewCipher(key)
-	if err != nil {
-		return err
-	}
-
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
-	var iv [aes.BlockSize]byte
-	stream := cipher.NewOFB(block, iv[:])
-
-	writer := &cipher.StreamWriter{S: stream, W: output}
-	// Copy the input to the output, encrypting as we go.
-	if _, err := io.Copy(writer, input); err != nil {
-		return err
+		return fmt.Errorf("error encrypting file: %w", err)
 	}
 
 	return nil
 }
 
-// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
-// passphrase is used to generate an encryption key.
+// AesDecrypt reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from
 func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Read file header to determine how it was encrypted
+	inputReader := bufio.NewReader(input)
+	header, err := inputReader.Peek(len(aesGcmHeader))
+	if err != nil {
+		return nil, fmt.Errorf("error reading encrypted backup file header: %w", err)
+	}
+
+	if string(header) == aesGcmHeader {
+		reader, err := aesDecryptGCM(inputReader, passphrase)
+		if err != nil {
+			return nil, fmt.Errorf("error decrypting file: %w", err)
+		}
+
+		return reader, nil
+	}
+
+	// Use the previous decryption routine which has no header (to support older archives)
+	reader, err := aesDecryptOFB(inputReader, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("error decrypting legacy file backup: %w", err)
+	}
+
+	return reader, nil
+}
+
+// aesEncryptGCM reads from input, encrypts with AES-256 and writes to output. passphrase is used to generate an encryption key.
+func aesEncryptGCM(input io.Reader, output io.Writer, passphrase []byte) error {
+	// Derive key using argon2 with a random salt
+	salt := make([]byte, 16) // 16 bytes salt
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		return err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return err
+	}
+
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return err
+	}
+
+	// Generate nonce
+	nonce, err := NewRandomNonce(aesgcm.NonceSize())
+	if err != nil {
+		return err
+	}
+
+	// write the header
+	if _, err := output.Write([]byte(aesGcmHeader)); err != nil {
+		return err
+	}
+
+	// Write nonce and salt to the output file
+	if _, err := output.Write(salt); err != nil {
+		return err
+	}
+	if _, err := output.Write(nonce.Value()); err != nil {
+		return err
+	}
+
+	// Buffer for reading plaintext blocks
+	buf := make([]byte, aesGcmBlockSize) // Adjust buffer size as needed
+	ciphertext := make([]byte, len(buf)+aesgcm.Overhead())
+
+	// Encrypt plaintext in blocks
+	for {
+		n, err := io.ReadFull(input, buf)
+		if n == 0 {
+			break // end of plaintext input
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return err
+		}
+
+		// Seal encrypts the plaintext using the nonce returning the updated slice.
+		ciphertext = aesgcm.Seal(ciphertext[:0], nonce.Value(), buf[:n], nil)
+
+		_, err = output.Write(ciphertext)
+		if err != nil {
+			return err
+		}
+
+		nonce.Increment()
+	}
+
+	return nil
+}
+
+// aesDecryptGCM reads from input, decrypts with AES-256 and returns the reader to read the decrypted content from.
+func aesDecryptGCM(input io.Reader, passphrase []byte) (io.Reader, error) {
+	// Reader & verify header
+	header := make([]byte, len(aesGcmHeader))
+	if _, err := io.ReadFull(input, header); err != nil {
+		return nil, err
+	}
+
+	if string(header) != aesGcmHeader {
+		return nil, fmt.Errorf("invalid header")
+	}
+
+	// Read salt
+	salt := make([]byte, 16) // Salt size
+	if _, err := io.ReadFull(input, salt); err != nil {
+		return nil, err
+	}
+
+	key := argon2.IDKey(passphrase, salt, argon2TimeCost, argon2MemoryCost, argon2Threads, 32)
+
+	// Initialize AES cipher block
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create GCM mode with the cipher block
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read nonce from the input reader
+	nonce := NewNonce(aesgcm.NonceSize())
+	if err := nonce.Read(input); err != nil {
+		return nil, err
+	}
+
+	// Initialize a buffer to store decrypted data
+	buf := bytes.Buffer{}
+	plaintext := make([]byte, aesGcmBlockSize)
+
+	// Decrypt the ciphertext in blocks
+	for {
+		// Read a block of ciphertext from the input reader
+		ciphertextBlock := make([]byte, aesGcmBlockSize+aesgcm.Overhead()) // Adjust block size as needed
+		n, err := io.ReadFull(input, ciphertextBlock)
+		if n == 0 {
+			break // end of ciphertext
+		}
+
+		if err != nil && !(errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF)) {
+			return nil, err
+		}
+
+		// Decrypt the block of ciphertext
+		plaintext, err = aesgcm.Open(plaintext[:0], nonce.Value(), ciphertextBlock[:n], nil)
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = buf.Write(plaintext)
+		if err != nil {
+			return nil, err
+		}
+
+		nonce.Increment()
+	}
+
+	return &buf, nil
+}
+
+// aesDecryptOFB reads from input, decrypts with AES-256 and returns the reader to a read decrypted content from.
+// passphrase is used to generate an encryption key.
+// note: This function used to decrypt files that were encrypted without a header i.e. old archives
+func aesDecryptOFB(input io.Reader, passphrase []byte) (io.Reader, error) {
+	var emptySalt []byte = make([]byte, 0)
+
 	// making a 32 bytes key that would correspond to AES-256
 	// don't necessarily need a salt, so just kept in empty
 	key, err := scrypt.Key(passphrase, emptySalt, 32768, 8, 1, 32)
@@ -59,11 +223,9 @@ func AesDecrypt(input io.Reader, passphrase []byte) (io.Reader, error) {
 		return nil, err
 	}
 
-	// If the key is unique for each ciphertext, then it's ok to use a zero
-	// IV.
+	// If the key is unique for each ciphertext, then it's ok to use a zero IV.
 	var iv [aes.BlockSize]byte
 	stream := cipher.NewOFB(block, iv[:])
-
 	reader := &cipher.StreamReader{S: stream, R: input}
 
 	return reader, nil

api/crypto/aes_test.go

@@ -2,6 +2,7 @@ package crypto
 
 import (
 	"io"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"testing"
@@ -9,7 +10,19 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+
+func randBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
+	const passphrase = "passphrase"
+
 	tmpdir := t.TempDir()
 
 	var (
@@ -18,17 +31,99 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024*1024*100 + 523)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withStrongPassphrase(t *testing.T) {
+	const passphrase = "A strong passphrase with special characters: !@#$%^&*()_+"
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
+	os.WriteFile(originFilePath, content, 0600)
+
+	originFile, _ := os.Open(originFilePath)
+	defer originFile.Close()
+
+	encryptedFileWriter, _ := os.Create(encryptedFilePath)
+
+	err := AesEncrypt(originFile, encryptedFileWriter, []byte(passphrase))
+	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
+	encryptedContent, err := os.ReadFile(encryptedFilePath)
+	assert.Nil(t, err, "Couldn't read encrypted file")
+	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
+
+	encryptedFileReader, _ := os.Open(encryptedFilePath)
+	defer encryptedFileReader.Close()
+
+	decryptedFileWriter, _ := os.Create(decryptedFilePath)
+	defer decryptedFileWriter.Close()
+
+	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte(passphrase))
+	assert.Nil(t, err, "Failed to decrypt file")
+
+	io.Copy(decryptedFileWriter, decryptedReader)
+
+	decryptedContent, _ := os.ReadFile(decryptedFilePath)
+	assert.Equal(t, content, decryptedContent, "Original and decrypted content should match")
+}
+
+func Test_encryptAndDecrypt_withTheSamePasswordSmallFile(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var (
+		originFilePath    = filepath.Join(tmpdir, "origin2")
+		encryptedFilePath = filepath.Join(tmpdir, "encrypted2")
+		decryptedFilePath = filepath.Join(tmpdir, "decrypted2")
+	)
+
+	content := randBytes(500)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
 	defer originFile.Close()
 
 	encryptedFileWriter, _ := os.Create(encryptedFilePath)
-	defer encryptedFileWriter.Close()
 
 	err := AesEncrypt(originFile, encryptedFileWriter, []byte("passphrase"))
 	assert.Nil(t, err, "Failed to encrypt a file")
+	encryptedFileWriter.Close()
+
 	encryptedContent, err := os.ReadFile(encryptedFilePath)
 	assert.Nil(t, err, "Couldn't read encrypted file")
 	assert.NotEqual(t, encryptedContent, content, "Content wasn't encrypted")
@@ -57,7 +152,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1024 * 50)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -96,7 +191,7 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 		decryptedFilePath = filepath.Join(tmpdir, "decrypted")
 	)
 
-	content := []byte("content")
+	content := randBytes(1034)
 	os.WriteFile(originFilePath, content, 0600)
 
 	originFile, _ := os.Open(originFilePath)
@@ -117,11 +212,6 @@ func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T)
 	decryptedFileWriter, _ := os.Create(decryptedFilePath)
 	defer decryptedFileWriter.Close()
 
-	decryptedReader, err := AesDecrypt(encryptedFileReader, []byte("garbage"))
-	assert.Nil(t, err, "Should allow to decrypt with wrong passphrase")
-
-	io.Copy(decryptedFileWriter, decryptedReader)
-
-	decryptedContent, _ := os.ReadFile(decryptedFilePath)
-	assert.NotEqual(t, content, decryptedContent, "Original and decrypted content should NOT match")
+	_, err = AesDecrypt(encryptedFileReader, []byte("garbage"))
+	assert.NotNil(t, err, "Should not allow decrypt with wrong passphrase")
 }

api/crypto/nonce.go

@@ -0,0 +1,61 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"errors"
+	"io"
+)
+
+type Nonce struct {
+	val []byte
+}
+
+func NewNonce(size int) *Nonce {
+	return &Nonce{val: make([]byte, size)}
+}
+
+// NewRandomNonce generates a new initial nonce with the lower byte set to a random value
+// This ensures there are plenty of nonce values availble before rolling over
+// Based on ideas from the Secure Programming Cookbook for C and C++ by John Viega, Matt Messier
+// https://www.oreilly.com/library/view/secure-programming-cookbook/0596003943/ch04s09.html
+func NewRandomNonce(size int) (*Nonce, error) {
+	randomBytes := 1
+	if size <= randomBytes {
+		return nil, errors.New("nonce size must be greater than the number of random bytes")
+	}
+
+	randomPart := make([]byte, randomBytes)
+	if _, err := rand.Read(randomPart); err != nil {
+		return nil, err
+	}
+
+	zeroPart := make([]byte, size-randomBytes)
+	nonceVal := append(randomPart, zeroPart...)
+	return &Nonce{val: nonceVal}, nil
+}
+
+func (n *Nonce) Read(stream io.Reader) error {
+	_, err := io.ReadFull(stream, n.val)
+	return err
+}
+
+func (n *Nonce) Value() []byte {
+	return n.val
+}
+
+func (n *Nonce) Increment() error {
+	// Start incrementing from the least significant byte
+	for i := len(n.val) - 1; i >= 0; i-- {
+		// Increment the current byte
+		n.val[i]++
+
+		// Check for overflow
+		if n.val[i] != 0 {
+			// No overflow, nonce is successfully incremented
+			return nil
+		}
+	}
+
+	// If we reach here, it means the nonce has overflowed
+	return errors.New("nonce overflow")
+}