diff --git a/api/cron/job_script_execution.go b/api/cron/job_script_execution.go index aafbc4892..b1b53d878 100644 --- a/api/cron/job_script_execution.go +++ b/api/cron/job_script_execution.go @@ -2,6 +2,7 @@ package cron import ( "log" + "time" "github.com/portainer/portainer" ) @@ -46,6 +47,7 @@ func (runner *ScriptExecutionJobRunner) Run() { return } + targets := make([]*portainer.Endpoint, 0) for _, endpointID := range runner.job.Endpoints { endpoint, err := runner.context.endpointService.Endpoint(endpointID) if err != nil { @@ -53,11 +55,32 @@ func (runner *ScriptExecutionJobRunner) Run() { return } - err = runner.context.jobService.Execute(endpoint, "", runner.job.Image, scriptFile) - if err != nil { - log.Printf("scheduled job error (script execution). Unable to execute scrtip (endpoint=%s) (err=%s)\n", endpoint.Name, err) + targets = append(targets, endpoint) + } + + runner.executeAndRetry(targets, scriptFile, 0) +} + +func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) { + retryTargets := make([]*portainer.Endpoint, 0) + + for _, endpoint := range endpoints { + err := runner.context.jobService.Execute(endpoint, "", runner.job.Image, script) + if err == portainer.ErrUnableToPingEndpoint { + retryTargets = append(retryTargets, endpoint) + } else if err != nil { + log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err) } } + + retryCount++ + if retryCount >= runner.job.RetryCount { + return + } + + time.Sleep(time.Duration(runner.job.RetryInterval) * time.Second) + + runner.executeAndRetry(retryTargets, script, retryCount) } // GetScheduleID returns the schedule identifier associated to the runner diff --git a/api/docker/job.go b/api/docker/job.go index eba82e3e5..ed721cd4b 100644 --- a/api/docker/job.go +++ b/api/docker/job.go @@ -41,6 +41,11 @@ func (service *JobService) Execute(endpoint *portainer.Endpoint, nodeName, image } defer cli.Close() + _, err = cli.Ping(context.Background()) + if err != nil { + return portainer.ErrUnableToPingEndpoint + } + err = pullImage(cli, image) if err != nil { return err diff --git a/api/errors.go b/api/errors.go index e348aaf48..da6c4edfe 100644 --- a/api/errors.go +++ b/api/errors.go @@ -88,6 +88,11 @@ const ( ErrUndefinedTLSFileType = Error("Undefined TLS file type") ) +// Docker errors. +const ( + ErrUnableToPingEndpoint = Error("Unable to communicate with the endpoint") +) + // Error represents an application error. type Error string diff --git a/api/http/handler/schedules/schedule_create.go b/api/http/handler/schedules/schedule_create.go index 625d707ca..03db5203e 100644 --- a/api/http/handler/schedules/schedule_create.go +++ b/api/http/handler/schedules/schedule_create.go @@ -14,23 +14,27 @@ import ( "github.com/portainer/portainer/cron" ) -type scheduleFromFilePayload struct { +type scheduleCreateFromFilePayload struct { Name string Image string CronExpression string Endpoints []portainer.EndpointID File []byte + RetryCount int + RetryInterval int } -type scheduleFromFileContentPayload struct { +type scheduleCreateFromFileContentPayload struct { Name string CronExpression string Image string Endpoints []portainer.EndpointID FileContent string + RetryCount int + RetryInterval int } -func (payload *scheduleFromFilePayload) Validate(r *http.Request) error { +func (payload *scheduleCreateFromFilePayload) Validate(r *http.Request) error { name, err := request.RetrieveMultiPartFormValue(r, "Name", false) if err != nil { return errors.New("Invalid name") @@ -62,10 +66,16 @@ func (payload *scheduleFromFilePayload) Validate(r *http.Request) error { } payload.File = file + retryCount, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryCount", true) + payload.RetryCount = retryCount + + retryInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryInterval", true) + payload.RetryInterval = retryInterval + return nil } -func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error { +func (payload *scheduleCreateFromFileContentPayload) Validate(r *http.Request) error { if govalidator.IsNull(payload.Name) { return portainer.Error("Invalid schedule name") } @@ -86,6 +96,10 @@ func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error { return portainer.Error("Invalid script file content") } + if payload.RetryCount != 0 && payload.RetryInterval == 0 { + return portainer.Error("RetryInterval must be set") + } + return nil } @@ -107,71 +121,100 @@ func (handler *Handler) scheduleCreate(w http.ResponseWriter, r *http.Request) * } func (handler *Handler) createScheduleFromFileContent(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - var payload scheduleFromFileContentPayload + var payload scheduleCreateFromFileContentPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} } - schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, []byte(payload.FileContent)) + schedule := handler.createScheduleObjectFromFileContentPayload(&payload) + + err = handler.addAndPersistSchedule(schedule, []byte(payload.FileContent)) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err} + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err} } return response.JSON(w, schedule) } func (handler *Handler) createScheduleFromFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { - payload := &scheduleFromFilePayload{} + payload := &scheduleCreateFromFilePayload{} err := payload.Validate(r) if err != nil { return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} } - schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, payload.File) + schedule := handler.createScheduleObjectFromFilePayload(payload) + + err = handler.addAndPersistSchedule(schedule, payload.File) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err} + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err} } return response.JSON(w, schedule) } -func (handler *Handler) createSchedule(name, image, cronExpression string, endpoints []portainer.EndpointID, file []byte) (*portainer.Schedule, error) { +func (handler *Handler) createScheduleObjectFromFilePayload(payload *scheduleCreateFromFilePayload) *portainer.Schedule { scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier()) - scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(scheduleIdentifier)), file) - if err != nil { - return nil, err - } - job := &portainer.ScriptExecutionJob{ - Endpoints: endpoints, - Image: image, - ScriptPath: scriptPath, - ScheduleID: scheduleIdentifier, + Endpoints: payload.Endpoints, + Image: payload.Image, + ScheduleID: scheduleIdentifier, + RetryCount: payload.RetryCount, + RetryInterval: payload.RetryInterval, } schedule := &portainer.Schedule{ ID: scheduleIdentifier, - Name: name, - CronExpression: cronExpression, + Name: payload.Name, + CronExpression: payload.CronExpression, JobType: portainer.ScriptExecutionJobType, ScriptExecutionJob: job, Created: time.Now().Unix(), } + return schedule +} + +func (handler *Handler) createScheduleObjectFromFileContentPayload(payload *scheduleCreateFromFileContentPayload) *portainer.Schedule { + scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier()) + + job := &portainer.ScriptExecutionJob{ + Endpoints: payload.Endpoints, + Image: payload.Image, + ScheduleID: scheduleIdentifier, + RetryCount: payload.RetryCount, + RetryInterval: payload.RetryInterval, + } + + schedule := &portainer.Schedule{ + ID: scheduleIdentifier, + Name: payload.Name, + CronExpression: payload.CronExpression, + JobType: portainer.ScriptExecutionJobType, + ScriptExecutionJob: job, + Created: time.Now().Unix(), + } + + return schedule +} + +func (handler *Handler) addAndPersistSchedule(schedule *portainer.Schedule, file []byte) error { + scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(schedule.ID)), file) + if err != nil { + return err + } + + schedule.ScriptExecutionJob.ScriptPath = scriptPath + jobContext := cron.NewScriptExecutionJobContext(handler.JobService, handler.EndpointService, handler.FileService) - jobRunner := cron.NewScriptExecutionJobRunner(job, jobContext) + jobRunner := cron.NewScriptExecutionJobRunner(schedule.ScriptExecutionJob, jobContext) err = handler.JobScheduler.CreateSchedule(schedule, jobRunner) if err != nil { - return nil, err + return err } - err = handler.ScheduleService.CreateSchedule(schedule) - if err != nil { - return nil, err - } - - return schedule, nil + return handler.ScheduleService.CreateSchedule(schedule) } diff --git a/api/http/handler/schedules/schedule_update.go b/api/http/handler/schedules/schedule_update.go index e4de5b5f1..2c3b376f4 100644 --- a/api/http/handler/schedules/schedule_update.go +++ b/api/http/handler/schedules/schedule_update.go @@ -17,6 +17,8 @@ type scheduleUpdatePayload struct { CronExpression *string Endpoints []portainer.EndpointID FileContent *string + RetryCount *int + RetryInterval *int } func (payload *scheduleUpdatePayload) Validate(r *http.Request) error { @@ -91,5 +93,15 @@ func updateSchedule(schedule *portainer.Schedule, payload *scheduleUpdatePayload updateJobSchedule = true } + if payload.RetryCount != nil { + schedule.ScriptExecutionJob.RetryCount = *payload.RetryCount + updateJobSchedule = true + } + + if payload.RetryInterval != nil { + schedule.ScriptExecutionJob.RetryInterval = *payload.RetryInterval + updateJobSchedule = true + } + return updateJobSchedule } diff --git a/api/portainer.go b/api/portainer.go index 9e3efb8e8..30ac5983d 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -228,10 +228,12 @@ type ( // ScriptExecutionJob represents a scheduled job that can execute a script via a privileged container ScriptExecutionJob struct { - ScheduleID ScheduleID `json:"ScheduleId"` - Endpoints []EndpointID - Image string - ScriptPath string + ScheduleID ScheduleID `json:"ScheduleId"` + Endpoints []EndpointID + Image string + ScriptPath string + RetryCount int + RetryInterval int } // SnapshotJob represents a scheduled job that can create endpoint snapshots diff --git a/app/portainer/components/forms/schedule-form/scheduleForm.html b/app/portainer/components/forms/schedule-form/scheduleForm.html index 02ed2ebbe..4cdae3d96 100644 --- a/app/portainer/components/forms/schedule-form/scheduleForm.html +++ b/app/portainer/components/forms/schedule-form/scheduleForm.html @@ -42,8 +42,8 @@
- -
+ +
@@ -55,12 +55,24 @@
+
- - This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the - /host folder. - + +
+ +
+ +
+ +
+
@@ -98,6 +110,12 @@
Web editor
+
+ + This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the + /host folder. + +