fix(chisel): use a variable tunnel timeout depending on the environment EE-6843

api/chisel/service.go

@@ -11,6 +11,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/http/proxy"
+	"github.com/portainer/portainer/api/internal/edge"
 
 	chserver "github.com/jpillora/chisel/server"
 	"github.com/jpillora/chisel/share/ccrypto"
@@ -19,7 +20,7 @@ import (
 
 const (
 	tunnelCleanupInterval = 10 * time.Second
-	requiredTimeout       = 15 * time.Second
+	requiredTimeoutFactor = 3
 	activeTimeout         = 4*time.Minute + 30*time.Second
 	pingTimeout           = 3 * time.Second
 )
@@ -232,6 +233,7 @@ func (service *Service) startTunnelVerificationLoop() {
 
 func (service *Service) checkTunnels() {
 	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
+	envTimeout := make(map[portainer.EndpointID]time.Duration)
 
 	service.mu.Lock()
 	for key, tunnel := range service.tunnelDetailsMap {
@@ -239,15 +241,30 @@ func (service *Service) checkTunnels() {
 			continue
 		}
 
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
-			continue
-		}
-
 		if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
 			continue
 		}
 
+		endpoint, err := service.dataStore.Endpoint().Endpoint(key)
+		if err != nil {
+			log.Warn().Err(err).Int("endpoint_id", int(key)).Msg("unable to retrieve endpoint from database")
+			continue
+		}
+
+		checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint)
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to retrieve checking interval")
+			continue
+		}
+
+		requiredTimeout := requiredTimeoutFactor * time.Duration(checkinInterval) * time.Second
+
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
+			continue
+		}
+
 		tunnels[key] = *tunnel
+		envTimeout[key] = requiredTimeout
 	}
 	service.mu.Unlock()
 
@@ -259,12 +276,12 @@ func (service *Service) checkTunnels() {
 			Float64("status_time_seconds", elapsed.Seconds()).
 			Msg("environment tunnel monitoring")
 
-		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
+		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > envTimeout[endpointID] {
 			log.Debug().
 				Int("endpoint_id", int(endpointID)).
 				Str("status", tunnel.Status).
 				Float64("status_time_seconds", elapsed.Seconds()).
-				Float64("timeout_seconds", requiredTimeout.Seconds()).
+				Float64("timeout_seconds", envTimeout[endpointID].Seconds()).
 				Msg("REQUIRED state timeout exceeded")
 		}
 

api/chisel/tunnel.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 	"github.com/portainer/portainer/pkg/libcrypto"
 
@@ -73,27 +74,22 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer
 
 	tunnel := service.GetTunnelDetails(endpoint.ID)
 
-	if tunnel.Status == portainer.EdgeAgentActive {
+	switch tunnel.Status {
+	case portainer.EdgeAgentActive:
 		// update the LastActivity
 		service.SetTunnelStatusToActive(endpoint.ID)
-	}
 
-	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
+	case portainer.EdgeAgentIdle, portainer.EdgeAgentManagementRequired:
-		err := service.SetTunnelStatusToRequired(endpoint.ID)
+		if err := service.SetTunnelStatusToRequired(endpoint.ID); err != nil {
-		if err != nil {
 			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
 		}
 
-		if endpoint.EdgeCheckinInterval == 0 {
+		checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint)
-			settings, err := service.dataStore.Settings().Settings()
+		if err != nil {
-			if err != nil {
+			return portainer.TunnelDetails{}, fmt.Errorf("failed fetching checkin interval: %w", err)
-				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
-			}
-
-			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
 		}
 
-		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
+		time.Sleep(2 * time.Duration(checkinInterval) * time.Second)
 	}
 
 	return service.GetTunnelDetails(endpoint.ID), nil
@@ -147,38 +143,38 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
 	defer cache.Del(endpointID)
 
-	tunnel := service.getTunnelDetails(endpointID)
-
 	service.mu.Lock()
 	defer service.mu.Unlock()
 
-	if tunnel.Port == 0 {
+	tunnel := service.getTunnelDetails(endpointID)
-		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
+	if tunnel.Port != 0 {
-		if err != nil {
+		return nil
-			return err
-		}
-
-		tunnel.Status = portainer.EdgeAgentManagementRequired
-		tunnel.Port = service.getUnusedPort()
-		tunnel.LastActivity = time.Now()
-
-		username, password := generateRandomCredentials()
-		authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
-
-		if service.chiselServer != nil {
-			err = service.chiselServer.AddUser(username, password, authorizedRemote)
-			if err != nil {
-				return err
-			}
-		}
-
-		credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
-		if err != nil {
-			return err
-		}
-		tunnel.Credentials = credentials
 	}
 
+	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
+	if err != nil {
+		return err
+	}
+
+	tunnel.Status = portainer.EdgeAgentManagementRequired
+	tunnel.Port = service.getUnusedPort()
+	tunnel.LastActivity = time.Now()
+
+	username, password := generateRandomCredentials()
+	authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
+
+	if service.chiselServer != nil {
+		if err = service.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
+			return err
+		}
+	}
+
+	credentials, err := encryptCredentials(username, password, endpoint.EdgeID)
+	if err != nil {
+		return err
+	}
+	tunnel.Credentials = credentials
+
 	return nil
 }
 

api/http/handler/endpointedge/endpointedge_status_inspect.go

@@ -15,6 +15,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/edge"
 	"github.com/portainer/portainer/api/internal/edge/cache"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
@@ -153,13 +154,9 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
 		return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err)
 	}
 
-	checkinInterval := endpoint.EdgeCheckinInterval
+	checkinInterval, err := edge.GetEffectiveCheckinInterval(tx, endpoint)
-	if endpoint.EdgeCheckinInterval == 0 {
+	if err != nil {
-		settings, err := tx.Settings().Settings()
+		return nil, httperror.InternalServerError("Unable to retrieve the checkin interval", err)
-		if err != nil {
-			return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
-		}
-		checkinInterval = settings.EdgeAgentCheckinInterval
 	}
 
 	tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)

api/internal/edge/endpoint.go

@@ -1,6 +1,9 @@
 package edge
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+)
 
 // EndpointRelatedEdgeStacks returns a list of Edge stacks related to this Environment(Endpoint)
 func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *portainer.EndpointGroup, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) []portainer.EdgeStackID {
@@ -24,3 +27,16 @@ func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *port
 
 	return relatedEdgeStacks
 }
+
+func GetEffectiveCheckinInterval(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) (int, error) {
+	if endpoint.EdgeCheckinInterval != 0 {
+		return endpoint.EdgeCheckinInterval, nil
+	}
+
+	settings, err := tx.Settings().Settings()
+	if err != nil {
+		return 0, err
+	}
+
+	return settings.EdgeAgentCheckinInterval, nil
+}