Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-02 20:35:25 +02:00
Code Issues Releases Activity

fix(edgejobs): decouple the Edge Jobs from the reverse tunnel service BE-10866 (#11)

Browse source
This commit is contained in:
andres-portainer 2024-10-14 10:37:13 -03:00 • committed by GitHub
parent 7a35b5b0e4
commit 61c5269353
14 changed files with 179 additions and 243 deletions
Download patch file Download diff file Expand all files Collapse all files

23
api/http/handler/edgejobs/edgejob_create.go
View file

@ -11,6 +11,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@ -113,11 +114,14 @@ func (handler *Handler) createEdgeJob(tx dataservices.DataStoreTx, payload *edge
}
}
err = handler.addAndPersistEdgeJob(tx, edgeJob, fileContent, endpoints)
if err != nil {
if err := handler.addAndPersistEdgeJob(tx, edgeJob, fileContent, endpoints); err != nil {
return nil, httperror.InternalServerError("Unable to schedule Edge job", err)
}
for _, endpointID := range endpoints {
cache.Del(endpointID)
}
return edgeJob, nil
}
@ -144,15 +148,13 @@ func (payload *edgeJobCreateFromFilePayload) Validate(r *http.Request) error {
payload.CronExpression = cronExpression
var endpoints []portainer.EndpointID
err = request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, true)
if err != nil {
if err := request.RetrieveMultiPartFormJSONValue(r, "Endpoints", &endpoints, true); err != nil {
return errors.New("invalid environments")
}
payload.Endpoints = endpoints
var edgeGroups []portainer.EdgeGroupID
err = request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, true)
if err != nil {
if err := request.RetrieveMultiPartFormJSONValue(r, "EdgeGroups", &edgeGroups, true); err != nil {
return errors.New("invalid edge groups")
}
payload.EdgeGroups = edgeGroups
@ -267,15 +269,6 @@ func (handler *Handler) addAndPersistEdgeJob(tx dataservices.DataStoreTx, edgeJo
return errors.New("environments or edge groups are mandatory for an Edge job")
}
for endpointID := range endpointsMap {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
}
return tx.EdgeJob().CreateWithID(edgeJob.ID, edgeJob)
}

17
api/http/handler/edgejobs/edgejob_delete.go
View file

@ -9,9 +9,11 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
@ -33,10 +35,9 @@ func (handler *Handler) edgeJobDelete(w http.ResponseWriter, r *http.Request) *h
return httperror.BadRequest("Invalid Edge job identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.deleteEdgeJob(tx, portainer.EdgeJobID(edgeJobID))
})
if err != nil {
}); err != nil {
var handlerError *httperror.HandlerError
if errors.As(err, &handlerError) {
return handlerError
@ -57,13 +58,10 @@ func (handler *Handler) deleteEdgeJob(tx dataservices.DataStoreTx, edgeJobID por
}
edgeJobFolder := handler.FileService.GetEdgeJobFolder(strconv.Itoa(int(edgeJobID)))
err = handler.FileService.RemoveDirectory(edgeJobFolder)
if err != nil {
if err := handler.FileService.RemoveDirectory(edgeJobFolder); err != nil {
log.Warn().Err(err).Msg("Unable to remove the files associated to the Edge job on the filesystem")
}
handler.ReverseTunnelService.RemoveEdgeJob(edgeJob.ID)
var endpointsMap map[portainer.EndpointID]portainer.EdgeJobEndpointMeta
if len(edgeJob.EdgeGroups) > 0 {
endpoints, err := edge.GetEndpointsFromEdgeGroups(edgeJob.EdgeGroups, tx)
@ -78,11 +76,10 @@ func (handler *Handler) deleteEdgeJob(tx dataservices.DataStoreTx, edgeJobID por
}
for endpointID := range endpointsMap {
handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
cache.Del(endpointID)
}
err = tx.EdgeJob().Delete(edgeJob.ID)
if err != nil {
if err := tx.EdgeJob().Delete(edgeJob.ID); err != nil {
return httperror.InternalServerError("Unable to remove the Edge job from the database", err)
}

19
api/http/handler/edgejobs/edgejob_tasklogs_clear.go
View file

@ -9,6 +9,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
@ -53,7 +54,7 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
}
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
updateEdgeJobFn := func(edgeJob *portainer.EdgeJob, endpointID portainer.EndpointID, endpointsFromGroups []portainer.EndpointID) error {
mutationFn(edgeJob, endpointID, endpointsFromGroups)
@ -61,8 +62,7 @@ func (handler *Handler) edgeJobTasksClear(w http.ResponseWriter, r *http.Request
}
return handler.clearEdgeJobTaskLogs(tx, portainer.EdgeJobID(edgeJobID), portainer.EndpointID(taskID), updateEdgeJobFn)
})
if err != nil {
}); err != nil {
var handlerError *httperror.HandlerError
if errors.As(err, &handlerError) {
return handlerError
@ -82,8 +82,7 @@ func (handler *Handler) clearEdgeJobTaskLogs(tx dataservices.DataStoreTx, edgeJo
return httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
err = handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(int(edgeJobID)), strconv.Itoa(int(endpointID)))
if err != nil {
if err := handler.FileService.ClearEdgeJobTaskLogs(strconv.Itoa(int(edgeJobID)), strconv.Itoa(int(endpointID))); err != nil {
return httperror.InternalServerError("Unable to clear log file from disk", err)
}
@ -92,17 +91,11 @@ func (handler *Handler) clearEdgeJobTaskLogs(tx dataservices.DataStoreTx, edgeJo
return httperror.InternalServerError("Unable to get Endpoints from EdgeGroups", err)
}
err = updateEdgeJob(edgeJob, endpointID, endpointsFromGroups)
if err != nil {
if err := updateEdgeJob(edgeJob, endpointID, endpointsFromGroups); err != nil {
return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
}
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return httperror.NotFound("Unable to retrieve environment from the database", err)
}
handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
cache.Del(endpointID)
return nil
}

14
api/http/handler/edgejobs/edgejob_tasklogs_collect.go
View file

@ -8,6 +8,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
@ -38,7 +39,7 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
return httperror.BadRequest("Invalid Task identifier route variable", err)
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
edgeJob, err := tx.EdgeJob().Read(portainer.EdgeJobID(edgeJobID))
if tx.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an Edge job with the specified identifier inside the database", err)
@ -64,8 +65,7 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
edgeJob.Endpoints[endpointID] = meta
}
err = tx.EdgeJob().Update(edgeJob.ID, edgeJob)
if err != nil {
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
return httperror.InternalServerError("Unable to persist Edge job changes in the database", err)
}
@ -74,16 +74,14 @@ func (handler *Handler) edgeJobTasksCollect(w http.ResponseWriter, r *http.Reque
return httperror.InternalServerError("Unable to retrieve environment from the database", err)
}
cache.Del(endpointID)
if endpoint.Edge.AsyncMode {
return httperror.BadRequest("Async Edge Endpoints are not supported in Portainer CE", nil)
}
handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
return nil
})
if err != nil {
}); err != nil {
var handlerError *httperror.HandlerError
if errors.As(err, &handlerError) {
return handlerError

25
api/http/handler/edgejobs/edgejob_update.go
View file

@ -10,6 +10,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/edge/cache"
"github.com/portainer/portainer/api/internal/endpointutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@ -56,8 +57,7 @@ func (handler *Handler) edgeJobUpdate(w http.ResponseWriter, r *http.Request) *h
}
var payload edgeJobUpdatePayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
@ -78,13 +78,11 @@ func (handler *Handler) updateEdgeJob(tx dataservices.DataStoreTx, edgeJobID por
return nil, httperror.InternalServerError("Unable to find an Edge job with the specified identifier inside the database", err)
}
err = handler.updateEdgeSchedule(tx, edgeJob, &payload)
if err != nil {
if err := handler.updateEdgeSchedule(tx, edgeJob, &payload); err != nil {
return nil, httperror.InternalServerError("Unable to update Edge job", err)
}
err = tx.EdgeJob().Update(edgeJob.ID, edgeJob)
if err != nil {
if err := tx.EdgeJob().Update(edgeJob.ID, edgeJob); err != nil {
return nil, httperror.InternalServerError("Unable to persist Edge job changes inside the database", err)
}
@ -149,8 +147,7 @@ func (handler *Handler) updateEdgeSchedule(tx dataservices.DataStoreTx, edgeJob
if len(payload.EdgeGroups) > 0 {
for _, edgeGroupID := range payload.EdgeGroups {
_, err := tx.EdgeGroup().Read(edgeGroupID)
if err != nil {
if _, err := tx.EdgeGroup().Read(edgeGroupID); err != nil {
return err
}
@ -203,8 +200,7 @@ func (handler *Handler) updateEdgeSchedule(tx dataservices.DataStoreTx, edgeJob
if payload.FileContent != nil && *payload.FileContent != string(fileContent) {
fileContent = []byte(*payload.FileContent)
_, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), fileContent)
if err != nil {
if _, err := handler.FileService.StoreEdgeJobFileFromBytes(strconv.Itoa(int(edgeJob.ID)), fileContent); err != nil {
return err
}
@ -223,16 +219,11 @@ func (handler *Handler) updateEdgeSchedule(tx dataservices.DataStoreTx, edgeJob
maps.Copy(endpointsFromGroupsToAddMap, edgeJob.Endpoints)
for endpointID := range endpointsFromGroupsToAddMap {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return err
}
handler.ReverseTunnelService.AddEdgeJob(endpoint, edgeJob)
cache.Del(endpointID)
}
for endpointID := range endpointsToRemove {
handler.ReverseTunnelService.RemoveEdgeJobFromEndpoint(endpointID, edgeJob.ID)
cache.Del(endpointID)
}
return nil