feat(edge): introduce support for Edge agent (#3031)

* feat(edge): fix webconsole and agent deployment command

* feat(edge): display agent features when connected to IoT endpoint

* feat(edge): add -e CAP_HOST_MANAGEMENT=1 to agent command

* feat(edge): add -v /:/host and --name portainer_agent_iot to agent command

* style(endpoint-creation): refactor IoT agent to Edge agent

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* refactor(api): rename AgentIoTEnvironment to AgentEdgeEnvironment

* feat(endpoint-creation): update Edge agent deployment instructions

* feat(edge): wip edge

* feat(edge): refactor key creation

* feat(edge): update deployment instructions

* feat(home): update Edge agent endpoint item

* feat(edge): support dynamic ports

* feat(edge): support sleep/wake and snapshots

* feat(edge): support offline mode

* feat(edge): host job support for Edge endpoints

* feat(edge): introduce STANDBY state

* feat(edge): update Edge agent deployment command

* feat(edge): introduce EDGE_ID support

* feat(edge): update default inactivity interval to 5min

* feat(edge): reload Edge schedules after restart

* fix(edge): fix execution of endpoint job against an Edge endpoint

* fix(edge): fix minor issues with scheduling UI/UX

* feat(edge): introduce EdgeSchedule version management

* feat(edge): switch back to REQUIRED state from ACTIVE on error

* refactor(edge): remove comment

* feat(edge): updated tunnel status management

* feat(edge): fix flickering UI when accessing Edge endpoint from home view

* feat(edge): remove STANDBY status

* fix(edge): fix an issue with console and Swarm endpoint

* fix(edge): fix an issue with stack deployment

* fix(edge): reset timer when applying active status

* feat(edge): add background ping for Edge endpoints

* fix(edge): fix infinite loading loop after Edge endpoint connection failure

* fix(home): fix an issue with merge

* feat(api): remove SnapshotRaw from EndpointList response

* feat(api): add pagination for EndpointList operation

* feat(api): rename last_id query parameter to start

* feat(api): implement filter for EndpointList operation

* fix(edge): prevent a pointer issue after removing an active Edge endpoint

* feat(home): front - endpoint backend pagination (#2990)

* feat(home): endpoint pagination with backend

* feat(api): remove default limit value

* fix(endpoints): fix a minor issue with column span

* fix(endpointgroup-create): fix an issue with endpoint group creation

* feat(app): minor loading optimizations

* refactor(api): small refactor of EndpointList operation

* fix(home): fix minor loading text display issue

* refactor(api): document bolt services functions

* feat(home): minor optimization

* fix(api): replace seek with index scanning for EndpointPaginated

* fix(api): fix invalid starting index issue

* fix(api): first implementation of working filter

* fix(home): endpoints list keeps backend pagination when it needs to

* fix(api): endpoint pagination doesn't drop the first item on pages >=2 anymore

* fix(home): UI flickering on page/filter load/change

* feat(auth): login spinner

* feat(api): support searching in associated endpoint group data

* refactor(api): remove unused API endpoint

* refactor(api): remove comment

* refactor(api): refactor proxy manager

* feat(api): declare EndpointList params as optional

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(edge): new icon for Edge agent endpoint

* fix(edge): fix missing exec quick action

* fix(edge): add loading indicator when connecting to Edge endpoint

* feat(edge): disable service webhooks for Edge endpoints

* feat(endpoints): backend pagination for endpoints view (#3004)

* feat(edge): dynamic loading for stack migration feature

* feat(edge): wordwrap edge key

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(websocket): minor refactor associated to Edge agent

* feat(endpoint-group): enable backend pagination (#3017)

* feat(api): support groupID filter on endpoints route

* feat(api): add new API operations endpointGroupAddEndpoint and endpointGroupDeleteEndpoint

* feat(endpoint-groups): backend pagination support for create and edit

* feat(endpoint-groups): debounce on filter for create/edit views

* feat(endpoint-groups): filter assigned on create view

* (endpoint-groups): unassigned endpoints edit view

* refactor(endpoint-groups): code clean

* feat(endpoint-groups): remove message for Unassigned group

* refactor(api): endpoint group endpoint association refactor

* refactor(api): rename files and remove comments

* refactor(api): remove usage of utils

* refactor(api): optional parameters

* Merge branch 'feat-endpoint-backend-pagination' into edge

# Conflicts:
#	api/bolt/endpoint/endpoint.go
#	api/http/handler/endpointgroups/endpointgroup_update.go
#	api/http/handler/endpointgroups/handler.go
#	api/http/handler/endpoints/endpoint_list.go
#	app/portainer/services/api/endpointService.js

* fix(api): fix default tunnel server credentials

* feat(api): update endpointListOperation behavior and parameters

* fix(api): fix interface declaration

* feat(edge): support configurable Edge agent checkin interval

* feat(edge): support dynamic tunnel credentials

* feat(edge): update Edge agent deployment commands

* style(edge): update Edge agent settings text

* refactor(edge): remove unused credentials management methods

* feat(edge): associate a remote addr to tunnel credentials

* style(edge): update Edge endpoint icon

* feat(edge): support encrypted tunnel credentials

* fix(edge): fix invalid pointer cast

* feat(bolt): decode endpoints with jsoniter

* feat(edge): persist reverse tunnel keyseed

* refactor(edge): minor refactor

* feat(edge): update chisel library usage

* refactor(endpoint): use controller function

* feat(api): database migration to DBVersion 19

* refactor(api): refactor AddSchedule function

* refactor(schedules): remove comment

* refactor(api): remove comment

* refactor(api): remove comment

* feat(api): tunnel manager now only manage Edge endpoints

* refactor(api): clean-up and clarification of the Edge service

* refactor(api): clean-up and clarification of the Edge service

* fix(api): fix an issue with Edge agent snapshots

* refactor(api): add missing comments

* refactor(api): update constant description

* style(home): remove loading text on error

* feat(endpoint): remove 15s timeout for ping request

* style(home): display information about associated Edge endpoints

* feat(home): redirect to endpoint details on click on unassociated Edge endpoint

* feat(settings): remove 60s Edge poll frequency option

api/portainer.go

@@ -1,5 +1,7 @@
 package portainer
 
+import "time"
+
 type (
 	// Pair defines a key/value string pair
 	Pair struct {
@@ -10,6 +12,8 @@ type (
 	// CLIFlags represents the available flags on the CLI
 	CLIFlags struct {
 		Addr              *string
+		TunnelAddr        *string
+		TunnelPort        *string
 		AdminPassword     *string
 		AdminPasswordFile *string
 		Assets            *string
@@ -105,6 +109,7 @@ type (
 		SnapshotInterval                   string               `json:"SnapshotInterval"`
 		TemplatesURL                       string               `json:"TemplatesURL"`
 		EnableHostManagementFeatures       bool                 `json:"EnableHostManagementFeatures"`
+		EdgeAgentCheckinInterval           int                  `json:"EdgeAgentCheckinInterval"`
 
 		// Deprecated fields
 		DisplayDonationHeader       bool
@@ -250,7 +255,8 @@ type (
 		Snapshots          []Snapshot          `json:"Snapshots"`
 		UserAccessPolicies UserAccessPolicies  `json:"UserAccessPolicies"`
 		TeamAccessPolicies TeamAccessPolicies  `json:"TeamAccessPolicies"`
-
+		EdgeID             string              `json:"EdgeID,omitempty"`
+		EdgeKey            string              `json:"EdgeKey"`
 		// Deprecated fields
 		// Deprecated in DBVersion == 4
 		TLS           bool   `json:"TLS,omitempty"`
@@ -333,11 +339,21 @@ type (
 		Recurring          bool
 		Created            int64
 		JobType            JobType
+		EdgeSchedule       *EdgeSchedule
 		ScriptExecutionJob *ScriptExecutionJob
 		SnapshotJob        *SnapshotJob
 		EndpointSyncJob    *EndpointSyncJob
 	}
 
+	// EdgeSchedule represents a scheduled job that can run on Edge environments.
+	EdgeSchedule struct {
+		ID             ScheduleID   `json:"Id"`
+		CronExpression string       `json:"CronExpression"`
+		Script         string       `json:"Script"`
+		Version        int          `json:"Version"`
+		Endpoints      []EndpointID `json:"Endpoints"`
+	}
+
 	// WebhookID represents a webhook identifier.
 	WebhookID int
 
@@ -575,6 +591,20 @@ type (
 		Valid      bool   `json:"Valid,omitempty"`
 	}
 
+	// TunnelDetails represents information associated to a tunnel
+	TunnelDetails struct {
+		Status       string
+		LastActivity time.Time
+		Port         int
+		Schedules    []EdgeSchedule
+		Credentials  string
+	}
+
+	// TunnelServerInfo represents information associated to the tunnel server
+	TunnelServerInfo struct {
+		PrivateKeySeed string `json:"PrivateKeySeed"`
+	}
+
 	// CLIService represents a service for managing CLI
 	CLIService interface {
 		ParseFlags(version string) (*CLIFlags, error)
@@ -692,6 +722,12 @@ type (
 		StoreDBVersion(version int) error
 	}
 
+	// TunnelServerService represents a service for managing data associated to the tunnel server
+	TunnelServerService interface {
+		Info() (*TunnelServerInfo, error)
+		UpdateInfo(info *TunnelServerInfo) error
+	}
+
 	// WebhookService represents a service for managing webhook data.
 	WebhookService interface {
 		Webhooks() ([]Webhook, error)
@@ -850,13 +886,25 @@ type (
 		DisableExtension(extension *Extension) error
 		UpdateExtension(extension *Extension, version string) error
 	}
+
+	// ReverseTunnelService represensts a service used to manage reverse tunnel connections.
+	ReverseTunnelService interface {
+		StartTunnelServer(addr, port string, snapshotter Snapshotter) error
+		GenerateEdgeKey(url, host string, endpointIdentifier int) string
+		SetTunnelStatusToActive(endpointID EndpointID)
+		SetTunnelStatusToRequired(endpointID EndpointID) error
+		SetTunnelStatusToIdle(endpointID EndpointID)
+		GetTunnelDetails(endpointID EndpointID) *TunnelDetails
+		AddSchedule(endpointID EndpointID, schedule *EdgeSchedule)
+		RemoveSchedule(scheduleID ScheduleID)
+	}
 )
 
 const (
 	// APIVersion is the version number of the Portainer API
 	APIVersion = "1.21.0"
 	// DBVersion is the version number of the Portainer database
-	DBVersion = 18
+	DBVersion = 19
 	// AssetsServerURL represents the URL of the Portainer asset server
 	AssetsServerURL = "https://portainer-io-assets.sfo2.digitaloceanspaces.com"
 	// MessageOfTheDayURL represents the URL where Portainer MOTD message can be retrieved
@@ -865,6 +913,8 @@ const (
 	ExtensionDefinitionsURL = AssetsServerURL + "/extensions-1.21.0.json"
 	// PortainerAgentHeader represents the name of the header available in any agent response
 	PortainerAgentHeader = "Portainer-Agent"
+	// PortainerAgentEdgeIDHeader represent the name of the header containing the Edge ID associated to an agent/agent cluster
+	PortainerAgentEdgeIDHeader = "X-PortainerAgent-EdgeID"
 	// PortainerAgentTargetHeader represent the name of the header containing the target node name
 	PortainerAgentTargetHeader = "X-PortainerAgent-Target"
 	// PortainerAgentSignatureHeader represent the name of the header containing the digital signature
@@ -878,6 +928,8 @@ const (
 	SupportedDockerAPIVersion = "1.24"
 	// ExtensionServer represents the server used by Portainer to communicate with extensions
 	ExtensionServer = "localhost"
+	// DefaultEdgeAgentCheckinIntervalInSeconds represents the default interval (in seconds) used by Edge agents to checkin with the Portainer instance
+	DefaultEdgeAgentCheckinIntervalInSeconds = 5
 )
 
 const (
@@ -953,6 +1005,8 @@ const (
 	AgentOnDockerEnvironment
 	// AzureEnvironment represents an endpoint connected to an Azure environment
 	AzureEnvironment
+	// EdgeAgentEnvironment represents an endpoint connected to an Edge agent
+	EdgeAgentEnvironment
 )
 
 const (
@@ -1019,6 +1073,15 @@ const (
 	CustomRegistry
 )
 
+const (
+	// EdgeAgentIdle represents an idle state for a tunnel connected to an Edge endpoint.
+	EdgeAgentIdle string = "IDLE"
+	// EdgeAgentManagementRequired represents a required state for a tunnel connected to an Edge endpoint
+	EdgeAgentManagementRequired string = "REQUIRED"
+	// EdgeAgentActive represents an active state for a tunnel connected to an Edge endpoint
+	EdgeAgentActive string = "ACTIVE"
+)
+
 const (
 	OperationDockerContainerArchiveInfo         Authorization = "DockerContainerArchiveInfo"
 	OperationDockerContainerList                Authorization = "DockerContainerList"