feat(compose): add docker-compose wrapper (#4713)

* feat(compose): add docker-compose wrapper

ce-187

* fix(compose): pick compose implementation upon startup

* Add static compose build for linux

* Fix wget

* Fix platofrm specific docker-compose download

* Keep amd64 architecture as download parameter

* Add tmp folder for docker-compose

* fix: line endings

* add proxy server

* logs

* Proxy

* Add lite transport for compose

* Fix local deployment

* refactor: pass proxyManager by ref

* fix: string conversion

* refactor: compose wrapper remove unused code

* fix: tests

* Add edge

* Fix merge issue

* refactor: remove unused code

* Move server to proxy implementation

* Cleanup wrapper and manager

* feat: pass max supported compose syntax version with each endpoint

* fix: pick compose syntax version

* fix: store wrapper version in portainer

* Get and show composeSyntaxMaxVersion at stack creation screen

* Get and show composeSyntaxMaxVersion at stack editor screen

* refactor: proxy server

* Fix used tmp

* Bump docker-compose to 1.28.0

* remove message for docker compose limitation

* fix: markup typo

* Rollback docker compose to 1.27.4

* * attempt to fix the windows build issue

* * attempt to debug grunt issue

* * use console log in grunt file

* fix: try to fix windows build by removing indirect deps from go.mod

* Remove tmp folder

* Remove builder stage

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose

* feat(build/windows): add git for Docker Compose - fixed verbose output

* refactor: renames

* fix(stack): get endpoint by EndpointProvider

* fix(stack): use margin to add space between line instead of using br tag

Co-authored-by: Stéphane Busso <stephane.busso@gmail.com>
Co-authored-by: Simon Meng <simon.meng@portainer.io>
Co-authored-by: yi-portainer <yi.chen@portainer.io>
Co-authored-by: Steven Kang <skan070@gmail.com>

api/http/proxy/factory/docker_compose.go

@@ -0,0 +1,88 @@
+package factory
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/http/proxy/factory/dockercompose"
+)
+
+// ProxyServer provide an extedned proxy with a local server to forward requests
+type ProxyServer struct {
+	server *http.Server
+	Port   int
+}
+
+func (factory *ProxyFactory) NewDockerComposeAgentProxy(endpoint *portainer.Endpoint) (*ProxyServer, error) {
+
+	if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+		return &ProxyServer{
+			Port: factory.reverseTunnelService.GetTunnelDetails(endpoint.ID).Port,
+		}, nil
+	}
+
+	endpointURL, err := url.Parse(endpoint.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	endpointURL.Scheme = "http"
+	httpTransport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
+		config, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return nil, err
+		}
+
+		httpTransport.TLSClientConfig = config
+		endpointURL.Scheme = "https"
+	}
+
+	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
+
+	proxy.Transport = dockercompose.NewAgentTransport(factory.signatureService, httpTransport)
+
+	proxyServer := &ProxyServer{
+		&http.Server{
+			Handler: proxy,
+		},
+		0,
+	}
+
+	return proxyServer, proxyServer.start()
+}
+
+func (proxy *ProxyServer) start() error {
+	listener, err := net.Listen("tcp", ":0")
+	if err != nil {
+		return err
+	}
+
+	proxy.Port = listener.Addr().(*net.TCPAddr).Port
+	go func() {
+		proxyHost := fmt.Sprintf("127.0.0.1:%d", proxy.Port)
+		log.Printf("Starting Proxy server on %s...\n", proxyHost)
+
+		err := proxy.server.Serve(listener)
+		log.Printf("Exiting Proxy server %s\n", proxyHost)
+
+		if err != http.ErrServerClosed {
+			log.Printf("Proxy server %s exited with an error: %s\n", proxyHost, err)
+		}
+	}()
+
+	return nil
+}
+
+// Close shuts down the server
+func (proxy *ProxyServer) Close() {
+	if proxy.server != nil {
+		proxy.server.Close()
+	}
+}

api/http/proxy/factory/dockercompose/transport.go

@@ -0,0 +1,40 @@
+package dockercompose
+
+import (
+	"net/http"
+
+	portainer "github.com/portainer/portainer/api"
+)
+
+type (
+	// AgentTransport is an http.Transport wrapper that adds custom http headers to communicate to an Agent
+	AgentTransport struct {
+		httpTransport      *http.Transport
+		signatureService   portainer.DigitalSignatureService
+		endpointIdentifier portainer.EndpointID
+	}
+)
+
+// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
+func NewAgentTransport(signatureService portainer.DigitalSignatureService, httpTransport *http.Transport) *AgentTransport {
+	transport := &AgentTransport{
+		httpTransport:    httpTransport,
+		signatureService: signatureService,
+	}
+
+	return transport
+}
+
+// RoundTrip is the implementation of the the http.RoundTripper interface
+func (transport *AgentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
+
+	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
+	request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	return transport.httpTransport.RoundTrip(request)
+}

api/http/proxy/manager.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
@@ -43,13 +44,19 @@ func (manager *Manager) CreateAndRegisterEndpointProxy(endpoint *portainer.Endpo
 		return nil, err
 	}
 
-	manager.endpointProxies.Set(string(endpoint.ID), proxy)
+	manager.endpointProxies.Set(fmt.Sprint(endpoint.ID), proxy)
 	return proxy, nil
 }
 
+// CreateComposeProxyServer creates a new HTTP reverse proxy based on endpoint properties and and adds it to the registered proxies.
+// It can also be used to create a new HTTP reverse proxy and replace an already registered proxy.
+func (manager *Manager) CreateComposeProxyServer(endpoint *portainer.Endpoint) (*factory.ProxyServer, error) {
+	return manager.proxyFactory.NewDockerComposeAgentProxy(endpoint)
+}
+
 // GetEndpointProxy returns the proxy associated to a key
 func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Handler {
-	proxy, ok := manager.endpointProxies.Get(string(endpoint.ID))
+	proxy, ok := manager.endpointProxies.Get(fmt.Sprint(endpoint.ID))
 	if !ok {
 		return nil
 	}
@@ -61,7 +68,7 @@ func (manager *Manager) GetEndpointProxy(endpoint *portainer.Endpoint) http.Hand
 // and cleans the k8s endpoint client cache. DeleteEndpointProxy
 // is currently only called for edge connection clean up.
 func (manager *Manager) DeleteEndpointProxy(endpoint *portainer.Endpoint) {
-	manager.endpointProxies.Remove(string(endpoint.ID))
+	manager.endpointProxies.Remove(fmt.Sprint(endpoint.ID))
 	manager.k8sClientFactory.RemoveKubeClient(endpoint)
 }