feat(edge): introduce support for Edge agent (#3031)

* feat(edge): fix webconsole and agent deployment command

* feat(edge): display agent features when connected to IoT endpoint

* feat(edge): add -e CAP_HOST_MANAGEMENT=1 to agent command

* feat(edge): add -v /:/host and --name portainer_agent_iot to agent command

* style(endpoint-creation): refactor IoT agent to Edge agent

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* feat(endpoint-creation): update Edge agent deployment instructions

* feat(edge): wip edge

* feat(edge): refactor key creation

* feat(edge): update deployment instructions

* feat(home): update Edge agent endpoint item

* feat(edge): support dynamic ports

* feat(edge): support sleep/wake and snapshots

* feat(edge): support offline mode

* feat(edge): host job support for Edge endpoints

* feat(edge): introduce STANDBY state

* feat(edge): update Edge agent deployment command

* feat(edge): introduce EDGE_ID support

* feat(edge): update default inactivity interval to 5min

* feat(edge): reload Edge schedules after restart

* fix(edge): fix execution of endpoint job against an Edge endpoint

* fix(edge): fix minor issues with scheduling UI/UX

* feat(edge): introduce EdgeSchedule version management

* feat(edge): switch back to REQUIRED state from ACTIVE on error

* refactor(edge): remove comment

* feat(edge): updated tunnel status management

* feat(edge): fix flickering UI when accessing Edge endpoint from home view

* feat(edge): remove STANDBY status

* fix(edge): fix an issue with console and Swarm endpoint

* fix(edge): fix an issue with stack deployment

* fix(edge): reset timer when applying active status

* feat(edge): add background ping for Edge endpoints

* fix(edge): fix infinite loading loop after Edge endpoint connection failure

* fix(home): fix an issue with merge

* feat(api): remove SnapshotRaw from EndpointList response

* feat(api): add pagination for EndpointList operation

* feat(api): rename last_id query parameter to start

* feat(api): implement filter for EndpointList operation

* fix(edge): prevent a pointer issue after removing an active Edge endpoint

* feat(home): front - endpoint backend pagination (#2990)

* feat(home): endpoint pagination with backend

* feat(api): remove default limit value

* fix(endpoints): fix a minor issue with column span

* fix(endpointgroup-create): fix an issue with endpoint group creation

* feat(app): minor loading optimizations

* refactor(api): small refactor of EndpointList operation

* fix(home): fix minor loading text display issue

* refactor(api): document bolt services functions

* feat(home): minor optimization

* fix(api): replace seek with index scanning for EndpointPaginated

* fix(api): fix invalid starting index issue

* fix(api): first implementation of working filter

* fix(home): endpoints list keeps backend pagination when it needs to

* fix(api): endpoint pagination doesn't drop the first item on pages >=2 anymore

* fix(home): UI flickering on page/filter load/change

* feat(auth): login spinner

* feat(api): support searching in associated endpoint group data

* refactor(api): remove unused API endpoint

* refactor(api): remove comment

* refactor(api): refactor proxy manager

* feat(api): declare EndpointList params as optional

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(edge): new icon for Edge agent endpoint

* fix(edge): fix missing exec quick action

* fix(edge): add loading indicator when connecting to Edge endpoint

* feat(edge): disable service webhooks for Edge endpoints

* feat(endpoints): backend pagination for endpoints view (#3004)

* feat(edge): dynamic loading for stack migration feature

* feat(edge): wordwrap edge key

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(websocket): minor refactor associated to Edge agent

* feat(endpoint-group): enable backend pagination (#3017)

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(api): endpoint group endpoint association refactor

* refactor(api): rename files and remove comments

* refactor(api): remove usage of utils

* refactor(api): optional parameters

* Merge branch 'feat-endpoint-backend-pagination' into edge

# Conflicts:
#	api/bolt/endpoint/endpoint.go
#	api/http/handler/endpointgroups/endpointgroup_update.go
#	api/http/handler/endpointgroups/handler.go
#	api/http/handler/endpoints/endpoint_list.go
#	app/portainer/services/api/endpointService.js

* fix(api): fix default tunnel server credentials

* feat(api): update endpointListOperation behavior and parameters

* fix(api): fix interface declaration

* feat(edge): support configurable Edge agent checkin interval

* feat(edge): support dynamic tunnel credentials

* feat(edge): update Edge agent deployment commands

* style(edge): update Edge agent settings text

* refactor(edge): remove unused credentials management methods

* feat(edge): associate a remote addr to tunnel credentials

* style(edge): update Edge endpoint icon

* feat(edge): support encrypted tunnel credentials

* fix(edge): fix invalid pointer cast

* feat(bolt): decode endpoints with jsoniter

* feat(edge): persist reverse tunnel keyseed

* refactor(edge): minor refactor

* feat(edge): update chisel library usage

* refactor(endpoint): use controller function

* feat(api): database migration to DBVersion 19

* refactor(api): refactor AddSchedule function

* refactor(schedules): remove comment

* refactor(api): remove comment

* refactor(api): remove comment

* feat(api): tunnel manager now only manage Edge endpoints

* refactor(api): clean-up and clarification of the Edge service

* refactor(api): clean-up and clarification of the Edge service

* fix(api): fix an issue with Edge agent snapshots

* refactor(api): add missing comments

* refactor(api): update constant description

* style(home): remove loading text on error

* feat(endpoint): remove 15s timeout for ping request

* style(home): display information about associated Edge endpoints

* feat(home): redirect to endpoint details on click on unassociated Edge endpoint

* feat(settings): remove 60s Edge poll frequency option

api/http/proxy/docker_transport.go

@@ -24,7 +24,9 @@ type (
 		DockerHubService       portainer.DockerHubService
 		SettingsService        portainer.SettingsService
 		SignatureService       portainer.DigitalSignatureService
+		ReverseTunnelService   portainer.ReverseTunnelService
 		endpointIdentifier     portainer.EndpointID
+		endpointType           portainer.EndpointType
 	}
 	restrictedDockerOperationContext struct {
 		isAdmin                bool
@@ -58,7 +60,19 @@ func (p *proxyTransport) RoundTrip(request *http.Request) (*http.Response, error
 }
 
 func (p *proxyTransport) executeDockerRequest(request *http.Request) (*http.Response, error) {
-	return p.dockerTransport.RoundTrip(request)
+	response, err := p.dockerTransport.RoundTrip(request)
+
+	if p.endpointType != portainer.EdgeAgentEnvironment {
+		return response, err
+	}
+
+	if err == nil {
+		p.ReverseTunnelService.SetTunnelStatusToActive(p.endpointIdentifier)
+	} else {
+		p.ReverseTunnelService.SetTunnelStatusToIdle(p.endpointIdentifier)
+	}
+
+	return response, err
 }
 
 func (p *proxyTransport) proxyDockerRequest(request *http.Request) (*http.Response, error) {

api/http/proxy/factory.go

@@ -21,6 +21,7 @@ type proxyFactory struct {
 	RegistryService        portainer.RegistryService
 	DockerHubService       portainer.DockerHubService
 	SignatureService       portainer.DigitalSignatureService
+	ReverseTunnelService   portainer.ReverseTunnelService
 }
 
 func (factory *proxyFactory) newHTTPProxy(u *url.URL) http.Handler {
@@ -29,21 +30,21 @@ func (factory *proxyFactory) newHTTPProxy(u *url.URL) http.Handler {
 }
 
 func newAzureProxy(credentials *portainer.AzureCredentials) (http.Handler, error) {
-	url, err := url.Parse(AzureAPIBaseURL)
+	remoteURL, err := url.Parse(AzureAPIBaseURL)
 	if err != nil {
 		return nil, err
 	}
 
-	proxy := newSingleHostReverseProxyWithHostHeader(url)
+	proxy := newSingleHostReverseProxyWithHostHeader(remoteURL)
 	proxy.Transport = NewAzureTransport(credentials)
 
 	return proxy, nil
 }
 
-func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portainer.TLSConfiguration, enableSignature bool, endpointID portainer.EndpointID) (http.Handler, error) {
+func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portainer.TLSConfiguration, endpoint *portainer.Endpoint) (http.Handler, error) {
 	u.Scheme = "https"
 
-	proxy := factory.createDockerReverseProxy(u, enableSignature, endpointID)
+	proxy := factory.createDockerReverseProxy(u, endpoint)
 	config, err := crypto.CreateTLSConfigurationFromDisk(tlsConfig.TLSCACertPath, tlsConfig.TLSCertPath, tlsConfig.TLSKeyPath, tlsConfig.TLSSkipVerify)
 	if err != nil {
 		return nil, err
@@ -53,13 +54,19 @@ func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portaine
 	return proxy, nil
 }
 
-func (factory *proxyFactory) newDockerHTTPProxy(u *url.URL, enableSignature bool, endpointID portainer.EndpointID) http.Handler {
+func (factory *proxyFactory) newDockerHTTPProxy(u *url.URL, endpoint *portainer.Endpoint) http.Handler {
 	u.Scheme = "http"
-	return factory.createDockerReverseProxy(u, enableSignature, endpointID)
+	return factory.createDockerReverseProxy(u, endpoint)
 }
 
-func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, enableSignature bool, endpointID portainer.EndpointID) *httputil.ReverseProxy {
+func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, endpoint *portainer.Endpoint) *httputil.ReverseProxy {
 	proxy := newSingleHostReverseProxyWithHostHeader(u)
+
+	enableSignature := false
+	if endpoint.Type == portainer.AgentOnDockerEnvironment {
+		enableSignature = true
+	}
+
 	transport := &proxyTransport{
 		enableSignature:        enableSignature,
 		ResourceControlService: factory.ResourceControlService,
@@ -67,8 +74,10 @@ func (factory *proxyFactory) createDockerReverseProxy(u *url.URL, enableSignatur
 		SettingsService:        factory.SettingsService,
 		RegistryService:        factory.RegistryService,
 		DockerHubService:       factory.DockerHubService,
+		ReverseTunnelService:   factory.ReverseTunnelService,
 		dockerTransport:        &http.Transport{},
-		endpointIdentifier:     endpointID,
+		endpointIdentifier:     endpoint.ID,
+		endpointType:           endpoint.Type,
 	}
 
 	if enableSignature {

api/http/proxy/factory_local.go

@@ -8,7 +8,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 )
 
-func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.EndpointID) http.Handler {
+func (factory *proxyFactory) newLocalProxy(path string, endpoint *portainer.Endpoint) http.Handler {
 	proxy := &localProxy{}
 	transport := &proxyTransport{
 		enableSignature:        false,
@@ -18,7 +18,9 @@ func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.End
 		RegistryService:        factory.RegistryService,
 		DockerHubService:       factory.DockerHubService,
 		dockerTransport:        newSocketTransport(path),
-		endpointIdentifier:     endpointID,
+		ReverseTunnelService:   factory.ReverseTunnelService,
+		endpointIdentifier:     endpoint.ID,
+		endpointType:           endpoint.Type,
 	}
 	proxy.Transport = transport
 	return proxy

api/http/proxy/factory_local_windows.go

@@ -5,12 +5,11 @@ package proxy
 import (
 	"net"
 	"net/http"
-	"github.com/Microsoft/go-winio"
-	
+
 	portainer "github.com/portainer/portainer/api"
 )
 
-func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.EndpointID) http.Handler {
+func (factory *proxyFactory) newLocalProxy(path string, endpoint *portainer.Endpoint) http.Handler {
 	proxy := &localProxy{}
 	transport := &proxyTransport{
 		enableSignature:        false,
@@ -19,8 +18,10 @@ func (factory *proxyFactory) newLocalProxy(path string, endpointID portainer.End
 		SettingsService:        factory.SettingsService,
 		RegistryService:        factory.RegistryService,
 		DockerHubService:       factory.DockerHubService,
+		ReverseTunnelService:   factory.ReverseTunnelService,
 		dockerTransport:        newNamedPipeTransport(path),
-		endpointIdentifier:     endpointID,
+		endpointIdentifier:     endpoint.ID,
+		endpointType:           endpoint.Type,
 	}
 	proxy.Transport = transport
 	return proxy

api/http/proxy/manager.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -21,6 +22,7 @@ type (
 	// Manager represents a service used to manage Docker proxies.
 	Manager struct {
 		proxyFactory           *proxyFactory
+		reverseTunnelService   portainer.ReverseTunnelService
 		proxies                cmap.ConcurrentMap
 		extensionProxies       cmap.ConcurrentMap
 		legacyExtensionProxies cmap.ConcurrentMap
@@ -34,6 +36,7 @@ type (
 		RegistryService        portainer.RegistryService
 		DockerHubService       portainer.DockerHubService
 		SignatureService       portainer.DigitalSignatureService
+		ReverseTunnelService   portainer.ReverseTunnelService
 	}
 )
 
@@ -50,13 +53,15 @@ func NewManager(parameters *ManagerParams) *Manager {
 			RegistryService:        parameters.RegistryService,
 			DockerHubService:       parameters.DockerHubService,
 			SignatureService:       parameters.SignatureService,
+			ReverseTunnelService:   parameters.ReverseTunnelService,
 		},
+		reverseTunnelService: parameters.ReverseTunnelService,
 	}
 }
 
 // GetProxy returns the proxy associated to a key
-func (manager *Manager) GetProxy(key string) http.Handler {
-	proxy, ok := manager.proxies.Get(key)
+func (manager *Manager) GetProxy(endpoint *portainer.Endpoint) http.Handler {
+	proxy, ok := manager.proxies.Get(string(endpoint.ID))
 	if !ok {
 		return nil
 	}
@@ -76,8 +81,8 @@ func (manager *Manager) CreateAndRegisterProxy(endpoint *portainer.Endpoint) (ht
 }
 
 // DeleteProxy deletes the proxy associated to a key
-func (manager *Manager) DeleteProxy(key string) {
-	manager.proxies.Remove(key)
+func (manager *Manager) DeleteProxy(endpoint *portainer.Endpoint) {
+	manager.proxies.Remove(string(endpoint.ID))
 }
 
 // GetExtensionProxy returns an extension proxy associated to an extension identifier
@@ -136,28 +141,40 @@ func (manager *Manager) CreateLegacyExtensionProxy(key, extensionAPIURL string)
 	return proxy, nil
 }
 
-func (manager *Manager) createDockerProxy(endpointURL *url.URL, tlsConfig *portainer.TLSConfiguration, endpointID portainer.EndpointID) (http.Handler, error) {
-	if endpointURL.Scheme == "tcp" {
-		if tlsConfig.TLS || tlsConfig.TLSSkipVerify {
-			return manager.proxyFactory.newDockerHTTPSProxy(endpointURL, tlsConfig, false, endpointID)
-		}
-		return manager.proxyFactory.newDockerHTTPProxy(endpointURL, false, endpointID), nil
+func (manager *Manager) createDockerProxy(endpoint *portainer.Endpoint) (http.Handler, error) {
+	baseURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentEnvironment {
+		tunnel := manager.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		baseURL = fmt.Sprintf("http://localhost:%d", tunnel.Port)
 	}
-	return manager.proxyFactory.newLocalProxy(endpointURL.Path, endpointID), nil
-}
 
-func (manager *Manager) createProxy(endpoint *portainer.Endpoint) (http.Handler, error) {
-	endpointURL, err := url.Parse(endpoint.URL)
+	endpointURL, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
 	}
 
 	switch endpoint.Type {
 	case portainer.AgentOnDockerEnvironment:
-		return manager.proxyFactory.newDockerHTTPSProxy(endpointURL, &endpoint.TLSConfig, true, endpoint.ID)
-	case portainer.AzureEnvironment:
-		return newAzureProxy(&endpoint.AzureCredentials)
-	default:
-		return manager.createDockerProxy(endpointURL, &endpoint.TLSConfig, endpoint.ID)
+		return manager.proxyFactory.newDockerHTTPSProxy(endpointURL, &endpoint.TLSConfig, endpoint)
+	case portainer.EdgeAgentEnvironment:
+		return manager.proxyFactory.newDockerHTTPProxy(endpointURL, endpoint), nil
 	}
+
+	if endpointURL.Scheme == "tcp" {
+		if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
+			return manager.proxyFactory.newDockerHTTPSProxy(endpointURL, &endpoint.TLSConfig, endpoint)
+		}
+
+		return manager.proxyFactory.newDockerHTTPProxy(endpointURL, endpoint), nil
+	}
+
+	return manager.proxyFactory.newLocalProxy(endpointURL.Path, endpoint), nil
+}
+
+func (manager *Manager) createProxy(endpoint *portainer.Endpoint) (http.Handler, error) {
+	if endpoint.Type == portainer.AzureEnvironment {
+		return newAzureProxy(&endpoint.AzureCredentials)
+	}
+
+	return manager.createDockerProxy(endpoint)
 }