feat(edge-compute): move host jobs to edge (#3840)

* feat(endpoints): create an associated endpoints selector

* feat(schedules): remove edge specific explanations

* refactor(schedule): replace multi-endpoint-selector

* refactor(schedule): move controller to single file

* refactor(endpoints): remove multi-endpoint-selector

* feat(edge): rename host jobs to edge jobs

* feat(edge-jobs): remove edge warning

* refactor(edge-jobs): move schedule pages to edge

* refactor(edge-jobs): mv views to edgeJobs

* refactor(edge-jobs): rename edge jobs

* refactor(edge-jobs): move services to edge

* refactor(edge-jobs): move tasks datatable

* fix(edge-jobs): fix import

* fix(edge-jobs): use right services

* feat(settings): adjust host management description

* feat(edge-jobs): introduce interfaces and types

* feat(edge-jobs): implement bolt service

* refactor(edge-jobs): replace schedule routes

* refactor(edge-job): replace Schedule service

* refactor(edge-jobs): remove job_script_exec

* refactor(host): remove jobs table

* feat(edge-jobs): replace schedule

* feat(edge-jobs): load file on inspect

* fix(edge-job): parse cron correctly

* feat(edge-jobs): show tasks

* feat(host): rename tooltip

* refactor(host): remove old components

* refactor(main): remove schedule types

* refactor(snapshot): replace job service with snapshot service

* refactor(jobs): remove jobs form and datatable

* feat(edge-jobs): create db migration

* fix(main): start snapshot service with correct interval

* feat(settings): change host tooltip

* feat(edge-jobs): load endpoints

* fix(edge-job): disable form submit when form is invalid

* refactor(edge-compute): use const

* refactor(edge-jobs): use generic controller

* refactor(edge-jobs): replace $scope with controllerAs

* refactor(edge-jobs): replace routes with components

* refactor(edge-jobs): replace functions with classes

* refactor(edge-jobs): use async/await

* refactor(edge-jobs): rename functions

* feat(edge-jobs): introduce beta panel

* feat(edge-jobs): allow single character names

* fix(snapshot): run snapshot in coroutine

* feat(edge-jobs): add logs status

* feat(filesystem): add edge job logs methods

* feat(edge-jobs): intoduce edge jobs tasks api

* feat(edge-jobs): remove schedule task model

* fix(fs): build edge job task file path

* fix(edge-jobs): update task meta

* fix(edge-jobs): return a list of endpoints

* feat(edge-jobs): update logs from agent

* feat(edge-jobs): collect logs

* feat(edge-jobs): rename url

* feat(edge-jobs): refresh to same tab

* feat(edge-jobs): remove old info

* refactor(edge-jobs): rename script path json

* fix(edge-job): save file before adding job

* feat(edge-job): show retrieving logs label

* feat(edge-job): replace cron with 5 places

* refactor(edge-jobs): replace tasks with results

* feat(edge-jobs): add auto refresh until logs are collected

* feat(edge-jobs): fix column size

* feat(edge-job): display editor

* feat(edge-job): add name validation

* feat(edge-job): set default time for 1 hour from now

* feat(edge-job): add validation for cron format

* feat(edge-job): add a note about timezone

* fix(edge-job): replace regex

* fix(edge-job): check for every minute cron

* style(edge-jobs): add reference for cron regex

* refactor(edge-jobs): rename migration name

* refactor(edge-job): rename edge job response

* refactor(snapshot): rename snapshot endpoint method

* refactor(edge-jobs): move tasks handler to edgejobs

* feat(security): introduce a middleware for edge compute operations

* feat(edge-job): use edge compute middleware

* feat(edge-groups): filter http actions based on edge setting

* fix(security): return from edge bouncer if failed

* feat(edge-stacks): filter http actions based on edge setting

* feat(edge-groups): show error when failed to load groups

* refactor(db): remove edge-jobs migration

* refactor(migrator): remove unused dependency

Co-authored-by: Anthony Lapenna <lapenna.anthony@gmail.com>

api/bolt/datastore.go

@@ -9,6 +9,7 @@ import (
 	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/dockerhub"
 	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
 	"github.com/portainer/portainer/api/bolt/edgestack"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
@@ -44,6 +45,7 @@ type Store struct {
 	fileService             portainer.FileService
 	DockerHubService        *dockerhub.Service
 	EdgeGroupService        *edgegroup.Service
+	EdgeJobService          *edgejob.Service
 	EdgeStackService        *edgestack.Service
 	EndpointGroupService    *endpointgroup.Service
 	EndpointService         *endpoint.Service
@@ -184,6 +186,12 @@ func (store *Store) initServices() error {
 	}
 	store.EdgeGroupService = edgeGroupService
 
+	edgeJobService, err := edgejob.NewService(store.db)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
 	endpointgroupService, err := endpointgroup.NewService(store.db)
 	if err != nil {
 		return err
@@ -293,6 +301,11 @@ func (store *Store) EdgeGroup() portainer.EdgeGroupService {
 	return store.EdgeGroupService
 }
 
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
 // EdgeStack gives access to the EdgeStack data management layer
 func (store *Store) EdgeStack() portainer.EdgeStackService {
 	return store.EdgeStackService
@@ -333,11 +346,6 @@ func (store *Store) Role() portainer.RoleService {
 	return store.RoleService
 }
 
-// Schedule gives access to the Schedule data management layer
-func (store *Store) Schedule() portainer.ScheduleService {
-	return store.ScheduleService
-}
-
 // Settings gives access to the Settings data management layer
 func (store *Store) Settings() portainer.SettingsService {
 	return store.SettingsService

api/bolt/edgejob/edgejob.go

@@ -0,0 +1,101 @@
+package edgejob
+
+import (
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edgejobs"
+)
+
+// Service represents a service for managing edge jobs data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// EdgeJobs returns a list of Edge jobs
+func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var edgeJob portainer.EdgeJob
+			err := internal.UnmarshalObject(v, &edgeJob)
+			if err != nil {
+				return err
+			}
+			edgeJobs = append(edgeJobs, edgeJob)
+		}
+
+		return nil
+	})
+
+	return edgeJobs, err
+}
+
+// EdgeJob returns an Edge job by ID
+func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.db, BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
+}
+
+// CreateEdgeJob creates a new Edge job
+func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
+	return service.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		if edgeJob.ID == 0 {
+			id, _ := bucket.NextSequence()
+			edgeJob.ID = portainer.EdgeJobID(id)
+		}
+
+		data, err := internal.MarshalObject(edgeJob)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(edgeJob.ID)), data)
+	})
+}
+
+// UpdateEdgeJob updates an Edge job by ID
+func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.db, BucketName, identifier, edgeJob)
+}
+
+// DeleteEdgeJob deletes an Edge job
+func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.db, BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for an endpoint.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.db, BucketName)
+}

api/bolt/migrator/migrate_dbversion19.go

@@ -2,10 +2,10 @@ package migrator
 
 import (
 	"strings"
-
-	portainer "github.com/portainer/portainer/api"
 )
 
+const scheduleScriptExecutionJobType = 1
+
 func (m *Migrator) updateUsersToDBVersion20() error {
 	return m.authorizationService.UpdateUsersAuthorizations()
 }
@@ -28,7 +28,7 @@ func (m *Migrator) updateSchedulesToDBVersion20() error {
 	}
 
 	for _, schedule := range legacySchedules {
-		if schedule.JobType == portainer.ScriptExecutionJobType {
+		if schedule.JobType == scheduleScriptExecutionJobType {
 			if schedule.CronExpression == "0 0 * * *" {
 				schedule.CronExpression = "0 * * * *"
 			} else if schedule.CronExpression == "0 0 0/2 * *" {

api/bolt/migrator/migrate_dbversion23.go

@@ -1,6 +1,8 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	"github.com/portainer/portainer/api"
+)
 
 func (m *Migrator) updateSettingsToDB24() error {
 	legacySettings, err := m.settingsService.Settings()