feat(schedules): add retry policy to script schedules (#2445)

api/cron/job_script_execution.go

@@ -2,6 +2,7 @@ package cron
 
 import (
 	"log"
+	"time"
 
 	"github.com/portainer/portainer"
 )
@@ -46,6 +47,7 @@ func (runner *ScriptExecutionJobRunner) Run() {
 		return
 	}
 
+	targets := make([]*portainer.Endpoint, 0)
 	for _, endpointID := range runner.job.Endpoints {
 		endpoint, err := runner.context.endpointService.Endpoint(endpointID)
 		if err != nil {
@@ -53,11 +55,32 @@ func (runner *ScriptExecutionJobRunner) Run() {
 			return
 		}
 
-		err = runner.context.jobService.Execute(endpoint, "", runner.job.Image, scriptFile)
+		targets = append(targets, endpoint)
-		if err != nil {
+	}
-			log.Printf("scheduled job error (script execution). Unable to execute scrtip (endpoint=%s) (err=%s)\n", endpoint.Name, err)
+
+	runner.executeAndRetry(targets, scriptFile, 0)
+}
+
+func (runner *ScriptExecutionJobRunner) executeAndRetry(endpoints []*portainer.Endpoint, script []byte, retryCount int) {
+	retryTargets := make([]*portainer.Endpoint, 0)
+
+	for _, endpoint := range endpoints {
+		err := runner.context.jobService.Execute(endpoint, "", runner.job.Image, script)
+		if err == portainer.ErrUnableToPingEndpoint {
+			retryTargets = append(retryTargets, endpoint)
+		} else if err != nil {
+			log.Printf("scheduled job error (script execution). Unable to execute script (endpoint=%s) (err=%s)\n", endpoint.Name, err)
 		}
 	}
+
+	retryCount++
+	if retryCount >= runner.job.RetryCount {
+		return
+	}
+
+	time.Sleep(time.Duration(runner.job.RetryInterval) * time.Second)
+
+	runner.executeAndRetry(retryTargets, script, retryCount)
 }
 
 // GetScheduleID returns the schedule identifier associated to the runner

api/docker/job.go

@@ -41,6 +41,11 @@ func (service *JobService) Execute(endpoint *portainer.Endpoint, nodeName, image
 	}
 	defer cli.Close()
 
+	_, err = cli.Ping(context.Background())
+	if err != nil {
+		return portainer.ErrUnableToPingEndpoint
+	}
+
 	err = pullImage(cli, image)
 	if err != nil {
 		return err

api/errors.go

@@ -88,6 +88,11 @@ const (
 	ErrUndefinedTLSFileType = Error("Undefined TLS file type")
 )
 
+// Docker errors.
+const (
+	ErrUnableToPingEndpoint = Error("Unable to communicate with the endpoint")
+)
+
 // Error represents an application error.
 type Error string
 

api/http/handler/schedules/schedule_create.go

@@ -14,23 +14,27 @@ import (
 	"github.com/portainer/portainer/cron"
 )
 
-type scheduleFromFilePayload struct {
+type scheduleCreateFromFilePayload struct {
 	Name           string
 	Image          string
 	CronExpression string
 	Endpoints      []portainer.EndpointID
 	File           []byte
+	RetryCount     int
+	RetryInterval  int
 }
 
-type scheduleFromFileContentPayload struct {
+type scheduleCreateFromFileContentPayload struct {
 	Name           string
 	CronExpression string
 	Image          string
 	Endpoints      []portainer.EndpointID
 	FileContent    string
+	RetryCount     int
+	RetryInterval  int
 }
 
-func (payload *scheduleFromFilePayload) Validate(r *http.Request) error {
+func (payload *scheduleCreateFromFilePayload) Validate(r *http.Request) error {
 	name, err := request.RetrieveMultiPartFormValue(r, "Name", false)
 	if err != nil {
 		return errors.New("Invalid name")
@@ -62,10 +66,16 @@ func (payload *scheduleFromFilePayload) Validate(r *http.Request) error {
 	}
 	payload.File = file
 
+	retryCount, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryCount", true)
+	payload.RetryCount = retryCount
+
+	retryInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "RetryInterval", true)
+	payload.RetryInterval = retryInterval
+
 	return nil
 }
 
-func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error {
+func (payload *scheduleCreateFromFileContentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.Name) {
 		return portainer.Error("Invalid schedule name")
 	}
@@ -86,6 +96,10 @@ func (payload *scheduleFromFileContentPayload) Validate(r *http.Request) error {
 		return portainer.Error("Invalid script file content")
 	}
 
+	if payload.RetryCount != 0 && payload.RetryInterval == 0 {
+		return portainer.Error("RetryInterval must be set")
+	}
+
 	return nil
 }
 
@@ -107,71 +121,100 @@ func (handler *Handler) scheduleCreate(w http.ResponseWriter, r *http.Request) *
 }
 
 func (handler *Handler) createScheduleFromFileContent(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	var payload scheduleFromFileContentPayload
+	var payload scheduleCreateFromFileContentPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, []byte(payload.FileContent))
+	schedule := handler.createScheduleObjectFromFileContentPayload(&payload)
+
+	err = handler.addAndPersistSchedule(schedule, []byte(payload.FileContent))
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err}
 	}
 
 	return response.JSON(w, schedule)
 }
 
 func (handler *Handler) createScheduleFromFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	payload := &scheduleFromFilePayload{}
+	payload := &scheduleCreateFromFilePayload{}
 	err := payload.Validate(r)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	schedule, err := handler.createSchedule(payload.Name, payload.Image, payload.CronExpression, payload.Endpoints, payload.File)
+	schedule := handler.createScheduleObjectFromFilePayload(payload)
+
+	err = handler.addAndPersistSchedule(schedule, payload.File)
 	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Failed executing job", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to schedule script job", err}
 	}
 
 	return response.JSON(w, schedule)
 }
 
-func (handler *Handler) createSchedule(name, image, cronExpression string, endpoints []portainer.EndpointID, file []byte) (*portainer.Schedule, error) {
+func (handler *Handler) createScheduleObjectFromFilePayload(payload *scheduleCreateFromFilePayload) *portainer.Schedule {
 	scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier())
 
-	scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(scheduleIdentifier)), file)
-	if err != nil {
-		return nil, err
-	}
-
 	job := &portainer.ScriptExecutionJob{
-		Endpoints:  endpoints,
+		Endpoints:     payload.Endpoints,
-		Image:      image,
+		Image:         payload.Image,
-		ScriptPath: scriptPath,
+		ScheduleID:    scheduleIdentifier,
-		ScheduleID: scheduleIdentifier,
+		RetryCount:    payload.RetryCount,
+		RetryInterval: payload.RetryInterval,
 	}
 
 	schedule := &portainer.Schedule{
 		ID:                 scheduleIdentifier,
-		Name:               name,
+		Name:               payload.Name,
-		CronExpression:     cronExpression,
+		CronExpression:     payload.CronExpression,
 		JobType:            portainer.ScriptExecutionJobType,
 		ScriptExecutionJob: job,
 		Created:            time.Now().Unix(),
 	}
 
+	return schedule
+}
+
+func (handler *Handler) createScheduleObjectFromFileContentPayload(payload *scheduleCreateFromFileContentPayload) *portainer.Schedule {
+	scheduleIdentifier := portainer.ScheduleID(handler.ScheduleService.GetNextIdentifier())
+
+	job := &portainer.ScriptExecutionJob{
+		Endpoints:     payload.Endpoints,
+		Image:         payload.Image,
+		ScheduleID:    scheduleIdentifier,
+		RetryCount:    payload.RetryCount,
+		RetryInterval: payload.RetryInterval,
+	}
+
+	schedule := &portainer.Schedule{
+		ID:                 scheduleIdentifier,
+		Name:               payload.Name,
+		CronExpression:     payload.CronExpression,
+		JobType:            portainer.ScriptExecutionJobType,
+		ScriptExecutionJob: job,
+		Created:            time.Now().Unix(),
+	}
+
+	return schedule
+}
+
+func (handler *Handler) addAndPersistSchedule(schedule *portainer.Schedule, file []byte) error {
+	scriptPath, err := handler.FileService.StoreScheduledJobFileFromBytes(strconv.Itoa(int(schedule.ID)), file)
+	if err != nil {
+		return err
+	}
+
+	schedule.ScriptExecutionJob.ScriptPath = scriptPath
+
 	jobContext := cron.NewScriptExecutionJobContext(handler.JobService, handler.EndpointService, handler.FileService)
-	jobRunner := cron.NewScriptExecutionJobRunner(job, jobContext)
+	jobRunner := cron.NewScriptExecutionJobRunner(schedule.ScriptExecutionJob, jobContext)
 
 	err = handler.JobScheduler.CreateSchedule(schedule, jobRunner)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	err = handler.ScheduleService.CreateSchedule(schedule)
+	return handler.ScheduleService.CreateSchedule(schedule)
-	if err != nil {
-		return nil, err
-	}
-
-	return schedule, nil
 }

api/http/handler/schedules/schedule_update.go

@@ -17,6 +17,8 @@ type scheduleUpdatePayload struct {
 	CronExpression *string
 	Endpoints      []portainer.EndpointID
 	FileContent    *string
+	RetryCount     *int
+	RetryInterval  *int
 }
 
 func (payload *scheduleUpdatePayload) Validate(r *http.Request) error {
@@ -91,5 +93,15 @@ func updateSchedule(schedule *portainer.Schedule, payload *scheduleUpdatePayload
 		updateJobSchedule = true
 	}
 
+	if payload.RetryCount != nil {
+		schedule.ScriptExecutionJob.RetryCount = *payload.RetryCount
+		updateJobSchedule = true
+	}
+
+	if payload.RetryInterval != nil {
+		schedule.ScriptExecutionJob.RetryInterval = *payload.RetryInterval
+		updateJobSchedule = true
+	}
+
 	return updateJobSchedule
 }

api/portainer.go

@@ -228,10 +228,12 @@ type (
 
 	// ScriptExecutionJob represents a scheduled job that can execute a script via a privileged container
 	ScriptExecutionJob struct {
-		ScheduleID ScheduleID `json:"ScheduleId"`
+		ScheduleID    ScheduleID `json:"ScheduleId"`
-		Endpoints  []EndpointID
+		Endpoints     []EndpointID
-		Image      string
+		Image         string
-		ScriptPath string
+		ScriptPath    string
+		RetryCount    int
+		RetryInterval int
 	}
 
 	// SnapshotJob represents a scheduled job that can create endpoint snapshots

app/portainer/components/forms/schedule-form/scheduleForm.html

@@ -42,8 +42,8 @@
   </div>
   <!-- image-input -->
   <div class="form-group">
-    <label for="schedule_image" class="col-sm-1 control-label text-left">Image</label>
+    <label for="schedule_image" class="col-sm-2 control-label text-left">Image</label>
-    <div class="col-sm-11">
+    <div class="col-sm-10">
       <input type="text" class="form-control" ng-model="$ctrl.model.Job.Image" id="schedule_image" name="schedule_image" placeholder="e.g. ubuntu:latest" required>
     </div>
   </div>
@@ -55,12 +55,24 @@
     </div>
   </div>
   <!-- !image-input -->
+  <!-- retry-policy -->
   <div class="form-group">
-    <span class="col-sm-12 text-muted small">
+    <label for="retrycount" class="col-sm-2 control-label text-left">
-      This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the
+      Retry count
-      <code>/host</code> folder.
+      <portainer-tooltip position="bottom" message="Number of retries when it's not possible to reach the endpoint."></portainer-tooltip>
-    </span>
+    </label>
+    <div class="col-sm-10 col-md-4">
+      <input type="number" class="form-control" ng-model="$ctrl.model.Job.RetryCount" id="retrycount" name="retrycount" placeholder="3">
+    </div>
+    <label for="retryinterval" class="col-sm-2 control-label text-left">
+      Retry interval
+      <portainer-tooltip position="bottom" message="Retry interval in seconds."></portainer-tooltip>
+    </label>
+    <div class="col-sm-10 col-md-4">
+      <input type="number" class="form-control" ng-model="$ctrl.model.Job.RetryInterval" id="retryinterval" name="retryinterval" placeholder="30">
+    </div>
   </div>
+  <!-- !retry-policy -->
   <!-- execution-method -->
   <div ng-if="!$ctrl.model.Id">
     <div class="col-sm-12 form-section-title">
@@ -98,6 +110,12 @@
     <div class="col-sm-12 form-section-title">
       Web editor
     </div>
+    <div class="form-group">
+      <span class="col-sm-12 text-muted small">
+        This schedule will be executed via a privileged container on the target hosts. You can access the host filesystem under the
+        <code>/host</code> folder.
+      </span>
+    </div>
     <div class="form-group">
       <div class="col-sm-12">
         <code-editor

app/portainer/models/schedule.js

@@ -29,6 +29,8 @@ function ScriptExecutionJobModel(data) {
   this.Endpoints = data.Endpoints;
   this.FileContent = '';
   this.Method = 'editor';
+  this.RetryCount = data.RetryCount;
+  this.RetryInterval = data.RetryInterval;
 }
 
 function ScheduleCreateRequest(model) {
@@ -37,6 +39,8 @@ function ScheduleCreateRequest(model) {
   this.Image = model.Job.Image;
   this.Endpoints = model.Job.Endpoints;
   this.FileContent = model.Job.FileContent;
+  this.RetryCount = model.Job.RetryCount;
+  this.RetryInterval = model.Job.RetryInterval;
   this.File = model.Job.File;
 }
 
@@ -47,4 +51,6 @@ function ScheduleUpdateRequest(model) {
   this.Image = model.Job.Image;
   this.Endpoints = model.Job.Endpoints;
   this.FileContent = model.Job.FileContent;
+  this.RetryCount = model.Job.RetryCount;
+  this.RetryInterval = model.Job.RetryInterval;
 }

app/portainer/services/fileUpload.js

@@ -47,7 +47,9 @@ angular.module('portainer.app')
         Name: payload.Name,
         CronExpression: payload.CronExpression,
         Image: payload.Image,
-        Endpoints: Upload.json(payload.Endpoints)
+        Endpoints: Upload.json(payload.Endpoints),
+        RetryCount: payload.RetryCount,
+        RetryInterval: payload.RetryInterval
       }
     });
   };