feat(compose): add docker-compose wrapper (#4713)

* feat(compose): add docker-compose wrapper

ce-187

* fix(compose): pick compose implementation upon startup

* Add static compose build for linux

* Fix wget

* Fix platofrm specific docker-compose download

* Keep amd64 architecture as download parameter

* Add tmp folder for docker-compose

* fix: line endings

* add proxy server

* logs

* Proxy

* Add lite transport for compose

* Fix local deployment

* refactor: pass proxyManager by ref

* fix: string conversion

* refactor: compose wrapper remove unused code

* fix: tests

* Add edge

* Fix merge issue

* refactor: remove unused code

* Move server to proxy implementation

* Cleanup wrapper and manager

* feat: pass max supported compose syntax version with each endpoint

* fix: pick compose syntax version

* fix: store wrapper version in portainer

* Get and show composeSyntaxMaxVersion at stack creation screen

* Get and show composeSyntaxMaxVersion at stack editor screen

* refactor: proxy server

* Fix used tmp

* Bump docker-compose to 1.28.0

* remove message for docker compose limitation

* fix: markup typo

* Rollback docker compose to 1.27.4

* * attempt to fix the windows build issue

* * attempt to debug grunt issue

* * use console log in grunt file

* fix: try to fix windows build by removing indirect deps from go.mod

* Remove tmp folder

* Remove builder stage

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose - fixed verbose output

* refactor: renames

* fix(stack): get endpoint by EndpointProvider

* fix(stack): use margin to add space between line instead of using br tag

Co-authored-by: Stéphane Busso <stephane.busso@gmail.com>
Co-authored-by: Simon Meng <simon.meng@portainer.io>
Co-authored-by: yi-portainer <yi.chen@portainer.io>
Co-authored-by: Steven Kang <skan070@gmail.com>

api/http/handler/endpoints/endpoint_inspect.go

@@ -6,7 +6,7 @@ import (
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/errors"
 )
 
@@ -30,6 +30,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 	}
 
 	hideFields(endpoint)
+	endpoint.ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 
 	return response.JSON(w, endpoint)
 }

api/http/handler/endpoints/endpoint_list.go

@@ -5,12 +5,11 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/portainer/portainer/api"
-
 	"github.com/portainer/libhttp/request"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 )
 
@@ -89,6 +88,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 
 	for idx := range paginatedEndpoints {
 		hideFields(&paginatedEndpoints[idx])
+		paginatedEndpoints[idx].ComposeSyntaxMaxVersion = handler.ComposeStackManager.ComposeSyntaxMaxVersion()
 	}
 
 	w.Header().Set("X-Total-Count", strconv.Itoa(filteredEndpointCount))

api/http/handler/endpoints/handler.go

@@ -27,6 +27,7 @@ type Handler struct {
 	ProxyManager         *proxy.Manager
 	ReverseTunnelService portainer.ReverseTunnelService
 	SnapshotService      portainer.SnapshotService
+	ComposeStackManager  portainer.ComposeStackManager
 }
 
 // NewHandler creates a handler to manage endpoint operations.

api/http/handler/stacks/create_compose_stack.go

@@ -357,7 +357,6 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig)
 		!isAdminOrEndpointAdmin {
 
 		composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint)
-
 		stackContent, err := handler.FileService.GetFileContent(composeFilePath)
 		if err != nil {
 			return err

api/http/handler/stacks/handler.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/gorilla/mux"
 	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/http/security"
 	"github.com/portainer/portainer/api/internal/authorization"
 )

api/http/handler/stacks/stack_delete.go

@@ -155,5 +155,6 @@ func (handler *Handler) deleteStack(stack *portainer.Stack, endpoint *portainer.
 	if stack.Type == portainer.DockerSwarmStack {
 		return handler.SwarmStackManager.Remove(stack, endpoint)
 	}
+
 	return handler.ComposeStackManager.Down(stack, endpoint)
 }

api/http/handler/stacks/stack_start.go

@@ -4,13 +4,13 @@ import (
 	"errors"
 	"net/http"
 
+	portainer "github.com/portainer/portainer/api"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 	"github.com/portainer/portainer/api/http/security"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 )
 

api/http/handler/stacks/stack_stop.go

@@ -4,15 +4,14 @@ import (
 	"errors"
 	"net/http"
 
-	httperrors "github.com/portainer/portainer/api/http/errors"
-
-	"github.com/portainer/portainer/api/http/security"
-
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
-	"github.com/portainer/portainer/api"
+
+	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
 )
 
 // POST request on /api/stacks/:id/stop

api/http/proxy/factory/docker_compose.go

@@ -0,0 +1,88 @@
+package factory
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/proxy/factory/dockercompose"
+)
+
+// ProxyServer provide an extedned proxy with a local server to forward requests
+type ProxyServer struct {
+	server *http.Server
+	Port   int
+}
+
+func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) {
+
+	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+		return &ProxyServer{
+			Port: factory.reverseTunnelService.GetTunnelDetails(endpoint.ID).Port,
+		}, nil
+	}
+
+	endpointURL, err := url.Parse(endpoint.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	endpointURL.Scheme = "http"
+	httpTransport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
+		config, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+
+		httpTransport.TLSClientConfig = config
+		endpointURL.Scheme = "https"
+	}
+
+	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
+
+	proxy.Transport = dockercompose.NewAgentTransport(factory.signatureService, httpTransport)
+
+	proxyServer := &ProxyServer{
+		&http.Server{
+			Handler: proxy,
+		},
+		0,
+	}
+
+	return proxyServer, proxyServer.start()
+}
+
+func (proxy *ProxyServer) start() error {
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return err
+	}
+
+	proxy.Port = listener.Addr().(*net.TCPAddr).Port
+	go func() {
+		proxyHost := fmt.Sprintf("127.0.0.1:%d", proxy.Port)
+		log.Printf("Starting Proxy server on %s...\n", proxyHost)
+
+		err := proxy.server.Serve(listener)
+		log.Printf("Exiting Proxy server %s\n", proxyHost)
+
+		if err != http.ErrServerClosed {
+			log.Printf("Proxy server %s exited with an error: %s\n", proxyHost, err)
+		}
+	}()
+
+	return nil
+}
+
+// Close shuts down the server
+func (proxy *ProxyServer) Close() {
+	if proxy.server != nil {
+		proxy.server.Close()
+	}
+}

api/http/proxy/factory/dockercompose/transport.go

@@ -0,0 +1,40 @@
+package dockercompose
+
+import (
+	"net/http"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+type (
+	// AgentTransport is an http.Transport wrapper that adds custom http headers to communicate to an Agent
+	AgentTransport struct {
+		httpTransport      *http.Transport
+		signatureService   portainer.DigitalSignatureService
+		endpointIdentifier portainer.EndpointID
+	}
+)
+
+// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
+func NewAgentTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *AgentTransport {
+	transport := &AgentTransport{
+		httpTransport:    httpTransport,
+		signatureService: signatureService,
+	}
+
+	return transport
+}
+
+// RoundTrip is the implementation of the the http.RoundTripper interface
+func (transport *AgentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+
+	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
+	request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	return transport.httpTransport.RoundTrip(request)
+}

api/http/proxy/manager.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
@@ -43,13 +44,19 @@ func (manager *Manager) CreateAndRegisterEndpointProxy(endpoint *portainer.Endpo
 		return nil, err
 	}
 
-	manager.endpointProxies.Set(string(endpoint.ID), proxy)
+	manager.endpointProxies.Set(fmt.Sprint(endpoint.ID), proxy)
 	return proxy, nil
 }
 
+// CreateComposeProxyServer creates a new HTTP reverse proxy based on endpoint properties and and adds it to the registered proxies.
+// It can also be used to create a new HTTP reverse proxy and replace an already registered proxy.
+func (manager *Manager) CreateComposeProxyServer(endpoint *portainer.Endpoint) (*factory.ProxyServer, error) {
+	return manager.proxyFactory.NewDockerComposeAgentProxy(endpoint)
+}
+
 // GetEndpointProxy returns the proxy associated to a key
 func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Handler {
-	proxy, ok := manager.endpointProxies.Get(string(endpoint.ID))
+	proxy, ok := manager.endpointProxies.Get(fmt.Sprint(endpoint.ID))
 	if !ok {
 		return nil
 	}
@@ -61,7 +68,7 @@ func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Hand
 // and cleans the k8s endpoint client cache. DeleteEndpointProxy
 // is currently only called for edge connection clean up.
 func (manager *Manager) DeleteEndpointProxy(endpoint *portainer.Endpoint) {
-	manager.endpointProxies.Remove(string(endpoint.ID))
+	manager.endpointProxies.Remove(fmt.Sprint(endpoint.ID))
 	manager.k8sClientFactory.RemoveKubeClient(endpoint)
 }
 

api/http/server.go

@@ -39,39 +39,41 @@ import (
 	"github.com/portainer/portainer/api/http/proxy"
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
 	"github.com/portainer/portainer/api/http/security"
+
 	"github.com/portainer/portainer/api/kubernetes/cli"
 )
 
 // Server implements the portainer.Server interface
 type Server struct {
-	BindAddress             string
-	AssetsPath              string
-	Status                  *portainer.Status
-	ReverseTunnelService    portainer.ReverseTunnelService
-	ComposeStackManager     portainer.ComposeStackManager
-	CryptoService           portainer.CryptoService
-	SignatureService        portainer.DigitalSignatureService
-	SnapshotService         portainer.SnapshotService
-	FileService             portainer.FileService
-	DataStore               portainer.DataStore
-	GitService              portainer.GitService
-	JWTService              portainer.JWTService
-	LDAPService             portainer.LDAPService
-	OAuthService            portainer.OAuthService
-	SwarmStackManager       portainer.SwarmStackManager
-	Handler                 *handler.Handler
-	SSL                     bool
-	SSLCert                 string
-	SSLKey                  string
-	DockerClientFactory     *docker.ClientFactory
-	KubernetesClientFactory *cli.ClientFactory
-	KubernetesDeployer      portainer.KubernetesDeployer
+	BindAddress                 string
+	AssetsPath                  string
+	Status                      *portainer.Status
+	ReverseTunnelService        portainer.ReverseTunnelService
+	ComposeStackManager         portainer.ComposeStackManager
+	CryptoService               portainer.CryptoService
+	SignatureService            portainer.DigitalSignatureService
+	SnapshotService             portainer.SnapshotService
+	FileService                 portainer.FileService
+	DataStore                   portainer.DataStore
+	GitService                  portainer.GitService
+	JWTService                  portainer.JWTService
+	LDAPService                 portainer.LDAPService
+	OAuthService                portainer.OAuthService
+	SwarmStackManager           portainer.SwarmStackManager
+	ProxyManager                *proxy.Manager
+	KubernetesTokenCacheManager *kubernetes.TokenCacheManager
+	Handler                     *handler.Handler
+	SSL                         bool
+	SSLCert                     string
+	SSLKey                      string
+	DockerClientFactory         *docker.ClientFactory
+	KubernetesClientFactory     *cli.ClientFactory
+	KubernetesDeployer          portainer.KubernetesDeployer
 }
 
 // Start starts the HTTP server
 func (server *Server) Start() error {
-	kubernetesTokenCacheManager := kubernetes.NewTokenCacheManager()
-	proxyManager := proxy.NewManager(server.DataStore, server.SignatureService, server.ReverseTunnelService, server.DockerClientFactory, server.KubernetesClientFactory, kubernetesTokenCacheManager)
+	kubernetesTokenCacheManager := server.KubernetesTokenCacheManager
 
 	requestBouncer := security.NewRequestBouncer(server.DataStore, server.JWTService)
 
@@ -82,7 +84,7 @@ func (server *Server) Start() error {
 	authHandler.CryptoService = server.CryptoService
 	authHandler.JWTService = server.JWTService
 	authHandler.LDAPService = server.LDAPService
-	authHandler.ProxyManager = proxyManager
+	authHandler.ProxyManager = server.ProxyManager
 	authHandler.KubernetesTokenCacheManager = kubernetesTokenCacheManager
 	authHandler.OAuthService = server.OAuthService
 
@@ -116,10 +118,10 @@ func (server *Server) Start() error {
 	var endpointHandler = endpoints.NewHandler(requestBouncer)
 	endpointHandler.DataStore = server.DataStore
 	endpointHandler.FileService = server.FileService
-	endpointHandler.ProxyManager = proxyManager
+	endpointHandler.ProxyManager = server.ProxyManager
 	endpointHandler.SnapshotService = server.SnapshotService
-	endpointHandler.ProxyManager = proxyManager
 	endpointHandler.ReverseTunnelService = server.ReverseTunnelService
+	endpointHandler.ComposeStackManager = server.ComposeStackManager
 
 	var endpointEdgeHandler = endpointedge.NewHandler(requestBouncer)
 	endpointEdgeHandler.DataStore = server.DataStore
@@ -131,7 +133,7 @@ func (server *Server) Start() error {
 
 	var endpointProxyHandler = endpointproxy.NewHandler(requestBouncer)
 	endpointProxyHandler.DataStore = server.DataStore
-	endpointProxyHandler.ProxyManager = proxyManager
+	endpointProxyHandler.ProxyManager = server.ProxyManager
 	endpointProxyHandler.ReverseTunnelService = server.ReverseTunnelService
 
 	var fileHandler = file.NewHandler(filepath.Join(server.AssetsPath, "public"))
@@ -141,7 +143,7 @@ func (server *Server) Start() error {
 	var registryHandler = registries.NewHandler(requestBouncer)
 	registryHandler.DataStore = server.DataStore
 	registryHandler.FileService = server.FileService
-	registryHandler.ProxyManager = proxyManager
+	registryHandler.ProxyManager = server.ProxyManager
 
 	var resourceControlHandler = resourcecontrols.NewHandler(requestBouncer)
 	resourceControlHandler.DataStore = server.DataStore