Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-08 23:35:31 +02:00
Code Issues Releases Activity

fix(chisel): use a variable tunnel timeout depending on the environment EE-6843

Browse source
This commit is contained in:
andres-portainer 2024-03-14 17:53:24 -03:00
parent 3cd58cac54
commit 48967ec231
4 changed files with 81 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

31
api/chisel/service.go
View file

@ -11,6 +11,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/internal/edge"
chserver "github.com/jpillora/chisel/server"
"github.com/jpillora/chisel/share/ccrypto"
@ -19,7 +20,7 @@ import (
const (
tunnelCleanupInterval = 10 * time.Second
requiredTimeout = 15 * time.Second
requiredTimeoutFactor = 3
activeTimeout = 4*time.Minute + 30*time.Second
pingTimeout = 3 * time.Second
)
@ -232,6 +233,7 @@ func (service *Service) startTunnelVerificationLoop() {
func (service *Service) checkTunnels() {
tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
envTimeout := make(map[portainer.EndpointID]time.Duration)
service.mu.Lock()
for key, tunnel := range service.tunnelDetailsMap {
@ -239,15 +241,30 @@ func (service *Service) checkTunnels() {
continue
}
if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
continue
}
if tunnel.Status == portainer.EdgeAgentActive && time.Since(tunnel.LastActivity) < activeTimeout {
continue
}
endpoint, err := service.dataStore.Endpoint().Endpoint(key)
if err != nil {
log.Warn().Err(err).Int("endpoint_id", int(key)).Msg("unable to retrieve endpoint from database")
continue
}
checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint)
if err != nil {
log.Warn().Err(err).Msg("unable to retrieve checking interval")
continue
}
requiredTimeout := requiredTimeoutFactor * time.Duration(checkinInterval) * time.Second
if tunnel.Status == portainer.EdgeAgentManagementRequired && time.Since(tunnel.LastActivity) < requiredTimeout {
continue
}
tunnels[key] = *tunnel
envTimeout[key] = requiredTimeout
}
service.mu.Unlock()
@ -259,12 +276,12 @@ func (service *Service) checkTunnels() {
Float64("status_time_seconds", elapsed.Seconds()).
Msg("environment tunnel monitoring")
if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > requiredTimeout {
if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed > envTimeout[endpointID] {
log.Debug().
Int("endpoint_id", int(endpointID)).
Str("status", tunnel.Status).
Float64("status_time_seconds", elapsed.Seconds()).
Float64("timeout_seconds", requiredTimeout.Seconds()).
Float64("timeout_seconds", envTimeout[endpointID].Seconds()).
Msg("REQUIRED state timeout exceeded")
}

32
api/chisel/tunnel.go
View file

@ -9,6 +9,7 @@ import (
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/pkg/libcrypto"
@ -73,27 +74,22 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer
tunnel := service.GetTunnelDetails(endpoint.ID)
if tunnel.Status == portainer.EdgeAgentActive {
switch tunnel.Status {
case portainer.EdgeAgentActive:
// update the LastActivity
service.SetTunnelStatusToActive(endpoint.ID)
}
if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
err := service.SetTunnelStatusToRequired(endpoint.ID)
if err != nil {
case portainer.EdgeAgentIdle, portainer.EdgeAgentManagementRequired:
if err := service.SetTunnelStatusToRequired(endpoint.ID); err != nil {
return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
}
if endpoint.EdgeCheckinInterval == 0 {
settings, err := service.dataStore.Settings().Settings()
checkinInterval, err := edge.GetEffectiveCheckinInterval(service.dataStore, endpoint)
if err != nil {
return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
return portainer.TunnelDetails{}, fmt.Errorf("failed fetching checkin interval: %w", err)
}
endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
time.Sleep(2 * time.Duration(checkinInterval) * time.Second)
}
return service.GetTunnelDetails(endpoint.ID), nil
@ -147,12 +143,14 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
defer cache.Del(endpointID)
tunnel := service.getTunnelDetails(endpointID)
service.mu.Lock()
defer service.mu.Unlock()
if tunnel.Port == 0 {
tunnel := service.getTunnelDetails(endpointID)
if tunnel.Port != 0 {
return nil
}
endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
if err != nil {
return err
@ -166,8 +164,7 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
authorizedRemote := fmt.Sprintf("^R:0.0.0.0:%d$", tunnel.Port)
if service.chiselServer != nil {
err = service.chiselServer.AddUser(username, password, authorizedRemote)
if err != nil {
if err = service.chiselServer.AddUser(username, password, authorizedRemote); err != nil {
return err
}
}
@ -177,7 +174,6 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
return err
}
tunnel.Credentials = credentials
}
return nil
}

9
api/http/handler/endpointedge/endpointedge_status_inspect.go
View file

@ -15,6 +15,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@ -153,13 +154,9 @@ func (handler *Handler) inspectStatus(tx dataservices.DataStoreTx, r *http.Reque
return nil, httperror.InternalServerError("Unable to persist environment changes inside the database", err)
}
checkinInterval := endpoint.EdgeCheckinInterval
if endpoint.EdgeCheckinInterval == 0 {
settings, err := tx.Settings().Settings()
checkinInterval, err := edge.GetEffectiveCheckinInterval(tx, endpoint)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
checkinInterval = settings.EdgeAgentCheckinInterval
return nil, httperror.InternalServerError("Unable to retrieve the checkin interval", err)
}
tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)

18
api/internal/edge/endpoint.go
View file

@ -1,6 +1,9 @@
package edge
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
// EndpointRelatedEdgeStacks returns a list of Edge stacks related to this Environment(Endpoint)
func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *portainer.EndpointGroup, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) []portainer.EdgeStackID {
@ -24,3 +27,16 @@ func EndpointRelatedEdgeStacks(endpoint *portainer.Endpoint, endpointGroup *port
return relatedEdgeStacks
}
func GetEffectiveCheckinInterval(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint) (int, error) {
if endpoint.EdgeCheckinInterval != 0 {
return endpoint.EdgeCheckinInterval, nil
}
settings, err := tx.Settings().Settings()
if err != nil {
return 0, err
}
return settings.EdgeAgentCheckinInterval, nil
}