feat(api/stacks): use compose-unpacker to deploy stacks from git [EE-4758] (#8725)

* feat(api/stacks): use compose-unpacker to deploy stacks from git

* refactor(api/stacks): move stack operation as unpacker builder parameter + check builder func existence

* fix(api/stacks): defer removal of unpacker container after error check

* refactor(api/unpacker-builder): clearer code around client creation for standalone and swarm manager

* refactor(api/stacks): extract git stack check to utility function

* fix(api/stacks): apply skip tls when deploying with unpcker - ref EE-5023

* fix(api/stacks): defer close of docker client

api/cmd/portainer/main.go

@@ -538,7 +538,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
-	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
+	stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer, dockerClientFactory, dataStore)
 	deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()

api/http/handler/stacks/stack_delete.go

@@ -188,9 +188,15 @@ func (handler *Handler) deleteExternalStack(r *http.Request, w http.ResponseWrit
 
 func (handler *Handler) deleteStack(userID portainer.UserID, stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	if stack.Type == portainer.DockerSwarmStack {
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.UndeployRemoteSwarmStack(stack, endpoint)
+		}
 		return handler.SwarmStackManager.Remove(stack, endpoint)
 	}
 	if stack.Type == portainer.DockerComposeStack {
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.UndeployRemoteComposeStack(stack, endpoint)
+		}
 		return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint)
 	}
 	if stack.Type == portainer.KubernetesStack {

api/http/handler/stacks/stack_start.go

@@ -133,8 +133,14 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
 func (handler *Handler) startStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	switch stack.Type {
 	case portainer.DockerComposeStack:
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.StartRemoteComposeStack(stack, endpoint)
+		}
 		return handler.ComposeStackManager.Up(context.TODO(), stack, endpoint, false)
 	case portainer.DockerSwarmStack:
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.StartRemoteSwarmStack(stack, endpoint)
+		}
 		return handler.SwarmStackManager.Deploy(stack, true, true, endpoint)
 	}
 	return nil

api/http/handler/stacks/stack_stop.go

@@ -117,8 +117,14 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
 func (handler *Handler) stopStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
 	switch stack.Type {
 	case portainer.DockerComposeStack:
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.StopRemoteComposeStack(stack, endpoint)
+		}
 		return handler.ComposeStackManager.Down(context.TODO(), stack, endpoint)
 	case portainer.DockerSwarmStack:
+		if stackutils.IsGitStack(stack) {
+			return handler.StackDeployer.StopRemoteSwarmStack(stack, endpoint)
+		}
 		return handler.SwarmStackManager.Remove(stack, endpoint)
 	}
 	return nil

api/internal/testhelpers/compose_stack_manager.go

@@ -20,7 +20,7 @@ func (manager *composeStackManager) NormalizeStackName(name string) string {
 	return name
 }
 
-func (manager *composeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, forceRereate bool) error {
+func (manager *composeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, forceRecreate bool) error {
 	return nil
 }
 

api/portainer.go

@@ -1336,7 +1336,7 @@ type (
 	ComposeStackManager interface {
 		ComposeSyntaxMaxVersion() string
 		NormalizeStackName(name string) string
-		Up(ctx context.Context, stack *Stack, endpoint *Endpoint, forceRereate bool) error
+		Up(ctx context.Context, stack *Stack, endpoint *Endpoint, forceRecreate bool) error
 		Down(ctx context.Context, stack *Stack, endpoint *Endpoint) error
 		Pull(ctx context.Context, stack *Stack, endpoint *Endpoint) error
 	}

api/stacks/deployments/compose_unpacker_cmd_builder.go

@@ -0,0 +1,234 @@
+package deployments
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/registryutils"
+)
+
+type StackRemoteOperation string
+
+const (
+	OperationDeploy        StackRemoteOperation = "compose-deploy"
+	OperationUndeploy      StackRemoteOperation = "compose-undeploy"
+	OperationComposeStart  StackRemoteOperation = "compose-start"
+	OperationComposeStop   StackRemoteOperation = "compose-stop"
+	OperationSwarmDeploy   StackRemoteOperation = "swarm-deploy"
+	OperationSwarmUndeploy StackRemoteOperation = "swarm-undeploy"
+	OperationSwarmStart    StackRemoteOperation = "swarm-start"
+	OperationSwarmStop     StackRemoteOperation = "swarm-stop"
+)
+
+const (
+	UnpackerCmdDeploy        = "deploy"
+	UnpackerCmdUndeploy      = "undeploy"
+	UnpackerCmdSwarmDeploy   = "swarm-deploy"
+	UnpackerCmdSwarmUndeploy = "swarm-undeploy"
+)
+
+type unpackerCmdBuilderOptions struct {
+	pullImage          bool
+	prune              bool
+	composeDestination string
+	registries         []portainer.Registry
+}
+
+type buildCmdFunc func(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string
+
+var funcmap = map[StackRemoteOperation]buildCmdFunc{
+	OperationDeploy:        buildDeployCmd,
+	OperationUndeploy:      buildUndeployCmd,
+	OperationComposeStart:  buildComposeStartCmd,
+	OperationComposeStop:   buildComposeStopCmd,
+	OperationSwarmDeploy:   buildSwarmDeployCmd,
+	OperationSwarmUndeploy: buildSwarmUndeployCmd,
+	OperationSwarmStart:    buildSwarmStartCmd,
+	OperationSwarmStop:     buildSwarmStopCmd,
+}
+
+// build the unpacker cmd for stack based on stackOperation
+func (d *stackDeployer) buildUnpackerCmdForStack(stack *portainer.Stack, operation StackRemoteOperation, opts unpackerCmdBuilderOptions) ([]string, error) {
+
+	fn := funcmap[operation]
+	if fn == nil {
+		return nil, fmt.Errorf("unknown stack operation %s", operation)
+	}
+
+	registriesStrings := getRegistry(opts.registries, d.dataStore)
+	envStrings := getEnv(stack.Env)
+
+	return fn(stack, opts, registriesStrings, envStrings), nil
+}
+
+// deploy [-u username -p password] [--skip-tls-verify] [--env KEY1=VALUE1 --env KEY2=VALUE2] <git-repo-url> <ref> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdDeploy)
+	cmd = appendGitAuthIfNeeded(cmd, stack)
+	cmd = appendSkipTLSVerifyIfNeeded(cmd, stack)
+	cmd = append(cmd, env...)
+	cmd = append(cmd, registries...)
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.GitConfig.ReferenceName)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// undeploy [-u username -p password] [-k] <git-repo-url> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildUndeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdUndeploy)
+	cmd = appendGitAuthIfNeeded(cmd, stack)
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// deploy [-u username -p password] [--skip-tls-verify] [-k] [--env KEY1=VALUE1 --env KEY2=VALUE2] <git-repo-url> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildComposeStartCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdDeploy)
+	cmd = appendGitAuthIfNeeded(cmd, stack)
+	cmd = appendSkipTLSVerifyIfNeeded(cmd, stack)
+	cmd = append(cmd, "-k")
+	cmd = append(cmd, env...)
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.GitConfig.ReferenceName)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// undeploy [-u username -p password] [-k] <git-repo-url> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildComposeStopCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdUndeploy)
+	cmd = appendGitAuthIfNeeded(cmd, stack)
+	cmd = append(cmd, "-k")
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// swarm-deploy [-u username -p password] [--skip-tls-verify] [-f] [-r] [--env KEY1=VALUE1 --env KEY2=VALUE2] <git-repo-url> <git-ref> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildSwarmDeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdSwarmDeploy)
+	cmd = appendGitAuthIfNeeded(cmd, stack)
+	cmd = appendSkipTLSVerifyIfNeeded(cmd, stack)
+	if opts.pullImage {
+		cmd = append(cmd, "-f")
+	}
+
+	if opts.prune {
+		cmd = append(cmd, "-r")
+	}
+	cmd = append(cmd, env...)
+	cmd = append(cmd, registries...)
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.GitConfig.ReferenceName)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// swarm-undeploy [-k] <project-name> <destination>
+func buildSwarmUndeployCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdSwarmUndeploy)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	return cmd
+}
+
+// swarm-deploy [-u username -p password] [-f] [-r] [-k] [--skip-tls-verify] [--env KEY1=VALUE1 --env KEY2=VALUE2] <git-repo-url> <project-name> <destination> <compose-file-path> [<more-file-paths>...]
+func buildSwarmStartCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdSwarmDeploy, "-f", "-r", "-k")
+	cmd = appendSkipTLSVerifyIfNeeded(cmd, stack)
+	cmd = append(cmd, getEnv(stack.Env)...)
+	cmd = append(cmd, stack.GitConfig.URL)
+	cmd = append(cmd, stack.GitConfig.ReferenceName)
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	cmd = append(cmd, stack.EntryPoint)
+	cmd = appendAdditionalFiles(cmd, stack.AdditionalFiles)
+	return cmd
+}
+
+// swarm-undeploy [-k] <project-name> <destination>
+func buildSwarmStopCmd(stack *portainer.Stack, opts unpackerCmdBuilderOptions, registries []string, env []string) []string {
+	cmd := []string{}
+	cmd = append(cmd, UnpackerCmdSwarmUndeploy, "-k")
+	cmd = append(cmd, stack.Name)
+	cmd = append(cmd, opts.composeDestination)
+	return cmd
+}
+
+func appendGitAuthIfNeeded(cmd []string, stack *portainer.Stack) []string {
+	if stack.GitConfig.Authentication != nil && len(stack.GitConfig.Authentication.Password) != 0 {
+		cmd = append(cmd, "-u", stack.GitConfig.Authentication.Username, "-p", stack.GitConfig.Authentication.Password)
+	}
+	return cmd
+}
+
+func appendSkipTLSVerifyIfNeeded(cmd []string, stack *portainer.Stack) []string {
+	if stack.GitConfig.TLSSkipVerify {
+		cmd = append(cmd, "--skip-tls-verify")
+	}
+	return cmd
+}
+
+func appendAdditionalFiles(cmd []string, files []string) []string {
+	for i := 0; i < len(files); i++ {
+		cmd = append(cmd, files[i])
+	}
+	return cmd
+}
+
+func getRegistry(registries []portainer.Registry, dataStore dataservices.DataStore) []string {
+	cmds := []string{}
+
+	for _, registry := range registries {
+		if registry.Authentication {
+			err := registryutils.EnsureRegTokenValid(dataStore, &registry)
+			if err == nil {
+				username, password, err := registryutils.GetRegEffectiveCredential(&registry)
+				if err == nil {
+					cmd := fmt.Sprintf("--registry=%s:%s:%s", username, password, registry.URL)
+					cmds = append(cmds, cmd)
+				}
+			}
+		}
+	}
+
+	return cmds
+}
+
+func getEnv(env []portainer.Pair) []string {
+	if len(env) == 0 {
+		return nil
+	}
+
+	cmd := []string{}
+	for _, pair := range env {
+		cmd = append(cmd, fmt.Sprintf(`--env=%s=%s`, pair.Name, pair.Value))
+	}
+
+	return cmd
+}

api/stacks/deployments/deploy.go

@@ -8,6 +8,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/stacks/stackutils"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
@@ -83,12 +84,22 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 
 	switch stack.Type {
 	case portainer.DockerComposeStack:
-		err := deployer.DeployComposeStack(stack, endpoint, registries, true, false)
+
+		if stackutils.IsGitStack(stack) {
+			err = deployer.DeployRemoteComposeStack(stack, endpoint, registries, true, false)
+		} else {
+			err = deployer.DeployComposeStack(stack, endpoint, registries, true, false)
+		}
+
 		if err != nil {
 			return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID)
 		}
 	case portainer.DockerSwarmStack:
-		err := deployer.DeploySwarmStack(stack, endpoint, registries, true, true)
+		if stackutils.IsGitStack(stack) {
+			err = deployer.DeployRemoteSwarmStack(stack, endpoint, registries, true, true)
+		} else {
+			err = deployer.DeploySwarmStack(stack, endpoint, registries, true, true)
+		}
 		if err != nil {
 			return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID)
 		}

api/stacks/deployments/deploy_test.go

@@ -15,11 +15,12 @@ import (
 
 type noopDeployer struct{}
 
+// without unpacker
 func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {
 	return nil
 }
 
-func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRereate bool) error {
+func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error {
 	return nil
 }
 
@@ -27,6 +28,32 @@ func (s *noopDeployer) DeployKubernetesStack(stack *portainer.Stack, endpoint *p
 	return nil
 }
 
+// with unpacker
+func (s *noopDeployer) DeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error {
+	return nil
+}
+func (s *noopDeployer) UndeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+func (s *noopDeployer) StartRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+func (s *noopDeployer) StopRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+func (s *noopDeployer) DeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {
+	return nil
+}
+func (s *noopDeployer) UndeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+func (s *noopDeployer) StartRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+func (s *noopDeployer) StopRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return nil
+}
+
 func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) {
 	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()

api/stacks/deployments/deployer.go

@@ -7,32 +7,43 @@ import (
 	"github.com/pkg/errors"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/docker"
 	k "github.com/portainer/portainer/api/kubernetes"
 )
 
-type StackDeployer interface {
+type BaseStackDeployer interface {
 	DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error
-	DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRereate bool) error
+	DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error
 	DeployKubernetesStack(stack *portainer.Stack, endpoint *portainer.Endpoint, user *portainer.User) error
 }
 
+type StackDeployer interface {
+	BaseStackDeployer
+	RemoteStackDeployer
+}
+
 type stackDeployer struct {
 	lock                *sync.Mutex
 	swarmStackManager   portainer.SwarmStackManager
 	composeStackManager portainer.ComposeStackManager
 	kubernetesDeployer  portainer.KubernetesDeployer
+	ClientFactory       *docker.ClientFactory
+	dataStore           dataservices.DataStore
 }
 
 // NewStackDeployer inits a stackDeployer struct with a SwarmStackManager, a ComposeStackManager and a KubernetesDeployer
-func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager, kubernetesDeployer portainer.KubernetesDeployer) *stackDeployer {
+func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager,
+	kubernetesDeployer portainer.KubernetesDeployer, clientFactory *docker.ClientFactory, dataStore dataservices.DataStore) *stackDeployer {
 	return &stackDeployer{
 		lock:                &sync.Mutex{},
 		swarmStackManager:   swarmStackManager,
 		composeStackManager: composeStackManager,
 		kubernetesDeployer:  kubernetesDeployer,
+		ClientFactory:       clientFactory,
+		dataStore:           dataStore,
 	}
 }
-
 func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
@@ -43,7 +54,7 @@ func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *porta
 	return d.swarmStackManager.Deploy(stack, prune, pullImage, endpoint)
 }
 
-func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRereate bool) error {
+func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 
@@ -58,7 +69,7 @@ func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *por
 		}
 	}
 
-	err := d.composeStackManager.Up(context.TODO(), stack, endpoint, forceRereate)
+	err := d.composeStackManager.Up(context.TODO(), stack, endpoint, forceRecreate)
 	if err != nil {
 		d.composeStackManager.Down(context.TODO(), stack, endpoint)
 	}

api/stacks/deployments/deployer_remote.go

@@ -0,0 +1,276 @@
+package deployments
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/filesystem"
+	"github.com/rs/zerolog/log"
+
+	dockerclient "github.com/docker/docker/client"
+)
+
+const (
+	defaultUnpackerImage       = "portainer/compose-unpacker:latest"
+	composeUnpackerImageEnvVar = "COMPOSE_UNPACKER_IMAGE"
+	composePathPrefix          = "portainer-compose-unpacker"
+)
+
+type RemoteStackDeployer interface {
+	// compose
+	DeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error
+	UndeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+	StartRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+	StopRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+	// swarm
+	DeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error
+	UndeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+	StartRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+	StopRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error
+}
+
+// Deploy a compose stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) DeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, forcePullImage bool, forceRecreate bool) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.swarmStackManager.Login(registries, endpoint)
+	defer d.swarmStackManager.Logout(endpoint)
+	// --force-recreate doesn't pull updated images
+	if forcePullImage {
+		err := d.composeStackManager.Pull(context.TODO(), stack, endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return d.remoteStack(stack, endpoint, OperationDeploy, unpackerCmdBuilderOptions{
+		registries: registries,
+	})
+}
+
+// Undeploy a compose stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) UndeployRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	return d.remoteStack(stack, endpoint, OperationUndeploy, unpackerCmdBuilderOptions{})
+}
+
+// Start a compose stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) StartRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return d.remoteStack(stack, endpoint, OperationComposeStart, unpackerCmdBuilderOptions{})
+}
+
+// Stop a compose stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) StopRemoteComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return d.remoteStack(stack, endpoint, OperationComposeStop, unpackerCmdBuilderOptions{})
+}
+
+// Deploy a swarm stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) DeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.swarmStackManager.Login(registries, endpoint)
+	defer d.swarmStackManager.Logout(endpoint)
+
+	return d.remoteStack(stack, endpoint, OperationSwarmDeploy, unpackerCmdBuilderOptions{
+
+		pullImage:  pullImage,
+		prune:      prune,
+		registries: registries,
+	})
+}
+
+// Undeploy a swarm stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) UndeployRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	return d.remoteStack(stack, endpoint, OperationSwarmUndeploy, unpackerCmdBuilderOptions{})
+}
+
+// Start a swarm stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) StartRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return d.remoteStack(stack, endpoint, OperationSwarmStart, unpackerCmdBuilderOptions{})
+}
+
+// Stop a swarm stack on remote environment using a https://github.com/portainer/compose-unpacker container
+func (d *stackDeployer) StopRemoteSwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint) error {
+	return d.remoteStack(stack, endpoint, OperationSwarmStop, unpackerCmdBuilderOptions{})
+}
+
+// Does all the heavy lifting:
+// * connect to env
+// * build the args for compose-unpacker
+// * deploy compose-unpacker container
+// * wait for deployment to end
+// * gather deployment logs and bubble them up
+func (d *stackDeployer) remoteStack(stack *portainer.Stack, endpoint *portainer.Endpoint, operation StackRemoteOperation, opts unpackerCmdBuilderOptions) error {
+	ctx := context.TODO()
+
+	cli, err := d.createDockerClient(ctx, endpoint)
+	if err != nil {
+		return errors.WithMessage(err, "unable to create docker client")
+	}
+	defer cli.Close()
+
+	image := getUnpackerImage()
+
+	reader, err := cli.ImagePull(ctx, image, types.ImagePullOptions{})
+	if err != nil {
+		return errors.Wrap(err, "unable to pull unpacker image")
+	}
+	defer reader.Close()
+	io.Copy(io.Discard, reader)
+
+	info, err := cli.Info(ctx)
+	if err != nil {
+		return errors.Wrap(err, "unable to get agent info")
+	}
+	targetSocketBind := getTargetSocketBind(info.OSType)
+
+	composeDestination := filesystem.JoinPaths(stack.ProjectPath, composePathPrefix)
+
+	opts.composeDestination = composeDestination
+
+	cmd, err := d.buildUnpackerCmdForStack(stack, operation, opts)
+	if err != nil {
+		return errors.Wrap(err, "unable to build command for unpacker")
+	}
+
+	log.Debug().
+		Str("image", image).
+		Str("cmd", strings.Join(cmd, " ")).
+		Msg("running unpacker")
+
+	rand.Seed(time.Now().UnixNano())
+	unpackerContainer, err := cli.ContainerCreate(ctx, &container.Config{
+		Image: image,
+		Cmd:   cmd,
+	}, &container.HostConfig{
+		Binds: []string{
+			fmt.Sprintf("%s:%s", composeDestination, composeDestination),
+			fmt.Sprintf("%s:%s", targetSocketBind, targetSocketBind),
+		},
+	}, nil, nil, fmt.Sprintf("portainer-unpacker-%d-%s-%d", stack.ID, stack.Name, rand.Intn(100)))
+
+	if err != nil {
+		return errors.Wrap(err, "unable to create unpacker container")
+	}
+	defer cli.ContainerRemove(ctx, unpackerContainer.ID, types.ContainerRemoveOptions{})
+
+	if err := cli.ContainerStart(ctx, unpackerContainer.ID, types.ContainerStartOptions{}); err != nil {
+		return errors.Wrap(err, "start unpacker container error")
+	}
+
+	statusCh, errCh := cli.ContainerWait(ctx, unpackerContainer.ID, container.WaitConditionNotRunning)
+	select {
+	case err := <-errCh:
+		if err != nil {
+			return errors.Wrap(err, "An error occurred while waiting for the deployment of the stack.")
+		}
+	case <-statusCh:
+	}
+
+	out, err := cli.ContainerLogs(ctx, unpackerContainer.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
+	if err != nil {
+		log.Error().Err(err).Msg("unable to get logs from unpacker container")
+	} else {
+		outputBytes, err := io.ReadAll(out)
+		if err != nil {
+			log.Error().Err(err).Msg("unable to parse logs from unpacker container")
+		} else {
+			log.Info().
+				Str("output", string(outputBytes)).
+				Msg("Stack deployment output")
+		}
+	}
+
+	status, err := cli.ContainerInspect(ctx, unpackerContainer.ID)
+	if err != nil {
+		return errors.Wrap(err, "fetch container information error")
+	}
+
+	if status.State.ExitCode != 0 {
+		return fmt.Errorf("an error occurred while running unpacker container with exit code %d", status.State.ExitCode)
+	}
+
+	return nil
+}
+
+// Creates a docker client with 1 hour timeout
+func (d *stackDeployer) createDockerClient(ctx context.Context, endpoint *portainer.Endpoint) (*dockerclient.Client, error) {
+	timeout := 3600 * time.Second
+	cli, err := d.ClientFactory.CreateClient(endpoint, "", &timeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to create Docker client")
+	}
+
+	info, err := cli.Info(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to get agent info")
+	}
+
+	if isNotInASwarm(&info) {
+		return cli, nil
+	}
+	defer cli.Close()
+
+	nodes, err := cli.NodeList(ctx, types.NodeListOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to list nodes")
+	}
+
+	if len(nodes) == 0 {
+		return nil, errors.New("no nodes available")
+	}
+
+	var managerNode swarm.Node
+	for _, node := range nodes {
+		if node.ManagerStatus != nil && node.ManagerStatus.Leader {
+			managerNode = node
+			break
+		}
+	}
+
+	if managerNode.ID == "" {
+		return nil, errors.New("no leader node available")
+	}
+
+	return d.ClientFactory.CreateClient(endpoint, managerNode.Description.Hostname, &timeout)
+}
+
+func getUnpackerImage() string {
+	image := os.Getenv(composeUnpackerImageEnvVar)
+	if image == "" {
+		image = defaultUnpackerImage
+	}
+
+	return image
+}
+
+func getTargetSocketBind(osType string) string {
+	targetSocketBind := "//./pipe/docker_engine"
+	if strings.EqualFold(osType, "linux") {
+		targetSocketBind = "/var/run/docker.sock"
+	}
+	return targetSocketBind
+}
+
+// Per https://stackoverflow.com/a/50590287 and Docker's LocalNodeState possible values
+// `LocalNodeStateInactive` means the node is not in a swarm cluster
+func isNotInASwarm(info *types.Info) bool {
+	return info.Swarm.LocalNodeState == swarm.LocalNodeStateInactive
+}

api/stacks/deployments/deployment_compose_config.go

@@ -84,6 +84,9 @@ func (config *ComposeStackDeploymentConfig) Deploy() error {
 			return err
 		}
 	}
+	if stackutils.IsGitStack(config.stack) {
+		return config.StackDeployer.DeployRemoteComposeStack(config.stack, config.endpoint, config.registries, config.forcePullImage, config.ForceCreate)
+	}
 
 	return config.StackDeployer.DeployComposeStack(config.stack, config.endpoint, config.registries, config.forcePullImage, config.ForceCreate)
 }

api/stacks/deployments/deployment_swarm_config.go

@@ -78,6 +78,10 @@ func (config *SwarmStackDeploymentConfig) Deploy() error {
 		}
 	}
 
+	if stackutils.IsGitStack(config.stack) {
+		return config.StackDeployer.DeployRemoteSwarmStack(config.stack, config.endpoint, config.registries, config.prune, config.pullImage)
+	}
+
 	return config.StackDeployer.DeploySwarmStack(config.stack, config.endpoint, config.registries, config.prune, config.pullImage)
 }
 

api/stacks/stackutils/util.go

@@ -42,3 +42,8 @@ func SanitizeLabel(value string) string {
 	re := regexp.MustCompile(`[^A-Za-z0-9\.\-\_]+`)
 	return re.ReplaceAllString(value, ".")
 }
+
+// IsGitStack checks if the stack is a git stack or not
+func IsGitStack(stack *portainer.Stack) bool {
+	return stack.GitConfig != nil && len(stack.GitConfig.URL) != 0
+}