fix(gitops): avoid cancelling the auto updates for any error EE-5604 (#10294)

api/scheduler/scheduler.go

@@ -17,6 +17,18 @@ type Scheduler struct {
 	mu         sync.Mutex
 }
 
+type PermanentError struct {
+	err error
+}
+
+func NewPermanentError(err error) *PermanentError {
+	return &PermanentError{err: err}
+}
+
+func (e *PermanentError) Error() string {
+	return e.err.Error()
+}
+
 func NewScheduler(ctx context.Context) *Scheduler {
 	crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger)))
 	crontab.Start()
@@ -84,14 +96,24 @@ func (s *Scheduler) StopJob(jobID string) error {
 func (s *Scheduler) StartJobEvery(duration time.Duration, job func() error) string {
 	ctx, cancel := context.WithCancel(context.Background())
 
-	j := cron.FuncJob(func() {
-		if err := job(); err != nil {
-			log.Debug().Msg("job returned an error")
-			cancel()
+	jobFn := cron.FuncJob(func() {
+		err := job()
+		if err == nil {
+			return
 		}
+
+		var permErr *PermanentError
+		if errors.As(err, &permErr) {
+			log.Error().Err(permErr).Msg("job returned a permanent error, it will be stopped")
+			cancel()
+
+			return
+		}
+
+		log.Error().Err(err).Msg("job returned an error, it will be rescheduled")
 	})
 
-	entryID := s.crontab.Schedule(cron.Every(duration), j)
+	entryID := s.crontab.Schedule(cron.Every(duration), jobFn)
 
 	s.mu.Lock()
 	s.activeJobs[entryID] = cancel