diff --git a/api/chisel/service.go b/api/chisel/service.go index 374b7d075..b0d0c9b03 100644 --- a/api/chisel/service.go +++ b/api/chisel/service.go @@ -11,6 +11,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/http/proxy" + "github.com/portainer/portainer/api/internal/edge" chserver "github.com/jpillora/chisel/server" "github.com/jpillora/chisel/share/ccrypto" @@ -19,7 +20,7 @@ import ( const ( tunnelCleanupInterval = 10 * time.Second - requiredTimeout = 15 * time.Second + requiredTimeoutFactor = 3 activeTimeout = 4*time.Minute + 30*time.Second pingTimeout = 3 * time.Second ) @@ -232,6 +233,7 @@ func (service *Service) startTunnelVerificationLoop() { func (service *Service) checkTunnels() { tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails) + envTimeout := make(map[portainer.EndpointID]time.Duration) service.mu.Lock() for key, tunnel := range service.tunnelDetailsMap { @@ -239,15 +241,30 @@ func (service *Service) checkTunnels() { continue } - if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout { - continue - } - if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout { continue } + endpoint, err := service.dataStore.Endpoint().Endpoint(key) + if err != nil { + log.Warn().Err(err).Int("endpoint_id", int(key)).Msg("unable to retrieve endpoint from database") + continue + } + + checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint) + if err != nil { + log.Warn().Err(err).Msg("unable to retrieve checking interval") + continue + } + + requiredTimeout := requiredTimeoutFactor * time.Duration(checkinInterval) * time.Second + + if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout { + continue + } + tunnels[key] = *tunnel + envTimeout[key] = requiredTimeout } service.mu.Unlock() @@ -259,12 +276,12 @@ func (service *Service) checkTunnels() { Float64("status_time_seconds", elapsed.Seconds()). Msg("environment tunnel monitoring") - if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout { + if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > envTimeout[endpointID] { log.Debug(). Int("endpoint_id", int(endpointID)). Str("status", tunnel.Status). Float64("status_time_seconds", elapsed.Seconds()). - Float64("timeout_seconds", requiredTimeout.Seconds()). + Float64("timeout_seconds", envTimeout[endpointID].Seconds()). Msg("REQUIRED state timeout exceeded") } diff --git a/api/chisel/tunnel.go b/api/chisel/tunnel.go index 7a38ba359..df0c9881f 100644 --- a/api/chisel/tunnel.go +++ b/api/chisel/tunnel.go @@ -9,6 +9,7 @@ import ( "time" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/edge" "github.com/portainer/portainer/api/internal/edge/cache" "github.com/portainer/portainer/pkg/libcrypto" @@ -73,27 +74,22 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer tunnel := service.GetTunnelDetails(endpoint.ID) - if tunnel.Status == portainer.EdgeAgentActive { + switch tunnel.Status { + case portainer.EdgeAgentActive: // update the LastActivity service.SetTunnelStatusToActive(endpoint.ID) - } - if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired { - err := service.SetTunnelStatusToRequired(endpoint.ID) - if err != nil { + case portainer.EdgeAgentIdle, portainer.EdgeAgentManagementRequired: + if err := service.SetTunnelStatusToRequired(endpoint.ID); err != nil { return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err) } - if endpoint.EdgeCheckinInterval == 0 { - settings, err := service.dataStore.Settings().Settings() - if err != nil { - return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err) - } - - endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval + checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint) + if err != nil { + return portainer.TunnelDetails{}, fmt.Errorf("failed fetching checkin interval: %w", err) } - time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second) + time.Sleep(2 * time.Duration(checkinInterval) * time.Second) } return service.GetTunnelDetails(endpoint.ID), nil @@ -147,38 +143,38 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) { func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error { defer cache.Del(endpointID) - tunnel := service.getTunnelDetails(endpointID) - service.mu.Lock() defer service.mu.Unlock() - if tunnel.Port == 0 { - endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID) - if err != nil { - return err - } - - tunnel.Status = portainer.EdgeAgentManagementRequired - tunnel.Port = service.getUnusedPort() - tunnel.LastActivity = time.Now() - - username, password := generateRandomCredentials() - authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port) - - if service.chiselServer != nil { - err = service.chiselServer.AddUser(username, password, authorizedRemote) - if err != nil { - return err - } - } - - credentials, err := encryptCredentials(username, password, endpoint.EdgeID) - if err != nil { - return err - } - tunnel.Credentials = credentials + tunnel := service.getTunnelDetails(endpointID) + if tunnel.Port != 0 { + return nil } + endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID) + if err != nil { + return err + } + + tunnel.Status = portainer.EdgeAgentManagementRequired + tunnel.Port = service.getUnusedPort() + tunnel.LastActivity = time.Now() + + username, password := generateRandomCredentials() + authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port) + + if service.chiselServer != nil { + if err = service.chiselServer.AddUser(username, password, authorizedRemote); err != nil { + return err + } + } + + credentials, err := encryptCredentials(username, password, endpoint.EdgeID) + if err != nil { + return err + } + tunnel.Credentials = credentials + return nil } diff --git a/api/http/handler/endpointedge/endpointedge_status_inspect.go b/api/http/handler/endpointedge/endpointedge_status_inspect.go index a6f8e21fc..01101ebba 100644 --- a/api/http/handler/endpointedge/endpointedge_status_inspect.go +++ b/api/http/handler/endpointedge/endpointedge_status_inspect.go @@ -15,6 +15,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/internal/edge" "github.com/portainer/portainer/api/internal/edge/cache" httperror "github.com/portainer/portainer/pkg/libhttp/error" "github.com/portainer/portainer/pkg/libhttp/request" @@ -153,13 +154,9 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err) } - checkinInterval := endpoint.EdgeCheckinInterval - if endpoint.EdgeCheckinInterval == 0 { - settings, err := tx.Settings().Settings() - if err != nil { - return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err) - } - checkinInterval = settings.EdgeAgentCheckinInterval + checkinInterval, err := edge.GetEffectiveCheckinInterval(tx, endpoint) + if err != nil { + return nil, httperror.InternalServerError("Unable to retrieve the checkin interval", err) } tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID) diff --git a/api/internal/edge/endpoint.go b/api/internal/edge/endpoint.go index 843561107..6c0721cba 100644 --- a/api/internal/edge/endpoint.go +++ b/api/internal/edge/endpoint.go @@ -1,6 +1,9 @@ package edge -import portainer "github.com/portainer/portainer/api" +import ( + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" +) // EndpointRelatedEdgeStacks returns a list of Edge stacks related to this Environment(Endpoint) func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *portainer.EndpointGroup, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) []portainer.EdgeStackID { @@ -24,3 +27,16 @@ func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *port return relatedEdgeStacks } + +func GetEffectiveCheckinInterval(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) (int, error) { + if endpoint.EdgeCheckinInterval != 0 { + return endpoint.EdgeCheckinInterval, nil + } + + settings, err := tx.Settings().Settings() + if err != nil { + return 0, err + } + + return settings.EdgeAgentCheckinInterval, nil +}