feat(system): upgrade portainer on kubernetes [EE-4625] (#8448)

api/internal/upgrade/upgrade_kubernetes.go

@@ -0,0 +1,201 @@
+package upgrade
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func ptr[T any](i T) *T { return &i }
+
+func (service *service) upgradeKubernetes(environment *portainer.Endpoint, licenseKey, version string) error {
+	ctx := context.TODO()
+
+	kubeCLI, err := service.kubernetesClientFactory.CreateClient(environment)
+	if err != nil {
+		return errors.WithMessage(err, "failed to get kubernetes client")
+	}
+
+	namespace := "portainer"
+	taskName := fmt.Sprintf("portainer-upgrade-%d", time.Now().Unix())
+
+	jobsCli := kubeCLI.BatchV1().Jobs(namespace)
+
+	updaterImage := os.Getenv(updaterImageEnvVar)
+	if updaterImage == "" {
+		updaterImage = "portainer/portainer-updater:latest"
+	}
+
+	portainerImagePrefix := os.Getenv(portainerImagePrefixEnvVar)
+	if portainerImagePrefix == "" {
+		portainerImagePrefix = "portainer/portainer-ee"
+	}
+
+	image := fmt.Sprintf("%s:%s", portainerImagePrefix, version)
+
+	if err := service.checkImageForKubernetes(ctx, kubeCLI, namespace, image); err != nil {
+		return err
+	}
+
+	job, err := jobsCli.Create(ctx, &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      taskName,
+			Namespace: namespace,
+		},
+
+		Spec: batchv1.JobSpec{
+			TTLSecondsAfterFinished: ptr[int32](5 * 60), // cleanup after 5 minutes
+			BackoffLimit:            ptr[int32](0),
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+
+					RestartPolicy:      "Never",
+					ServiceAccountName: "portainer-sa-clusteradmin",
+					Containers: []corev1.Container{
+						{
+							Name:  taskName,
+							Image: updaterImage,
+							Args: []string{
+								"--pretty-log",
+								"--log-level", "DEBUG",
+								"portainer",
+								"--env-type", "kubernetes",
+								"--image", image,
+								"--license", licenseKey,
+							},
+						},
+					},
+				},
+			},
+		},
+	}, metav1.CreateOptions{})
+
+	if err != nil {
+		return errors.WithMessage(err, "failed to create upgrade job")
+	}
+
+	watcher, err := jobsCli.Watch(ctx, metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + taskName,
+		TimeoutSeconds: ptr[int64](60),
+	})
+	if err != nil {
+		return errors.WithMessage(err, "failed to watch upgrade job")
+	}
+
+	for event := range watcher.ResultChan() {
+		job, ok := event.Object.(*batchv1.Job)
+		if !ok {
+			continue
+		}
+
+		for _, c := range job.Status.Conditions {
+			if c.Type == batchv1.JobComplete {
+				log.Debug().
+					Str("job", job.Name).
+					Msg("Upgrade job completed")
+				return nil
+			}
+
+			if c.Type == batchv1.JobFailed {
+				return fmt.Errorf("upgrade failed: %s", c.Message)
+			}
+		}
+	}
+
+	log.Debug().
+		Str("job", job.Name).
+		Msg("Upgrade job created")
+
+	return errors.New("upgrade failed: server should have been restarted by the updater")
+
+}
+
+func (service *service) checkImageForKubernetes(ctx context.Context, kubeCLI *kubernetes.Clientset, namespace, image string) error {
+	podsCli := kubeCLI.CoreV1().Pods(namespace)
+
+	log.Debug().
+		Str("image", image).
+		Msg("Checking image")
+
+	podName := fmt.Sprintf("portainer-image-check-%d", time.Now().Unix())
+	_, err := podsCli.Create(ctx, &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+		},
+		Spec: corev1.PodSpec{
+			RestartPolicy: "Never",
+
+			Containers: []corev1.Container{
+				{
+					Name:  fmt.Sprint(podName, "-container"),
+					Image: image,
+				},
+			},
+		},
+	}, metav1.CreateOptions{})
+
+	if err != nil {
+		log.Warn().Err(err).Msg("failed to create image check pod")
+		return errors.WithMessage(err, "failed to create image check pod")
+	}
+
+	defer func() {
+		log.Debug().
+			Str("pod", podName).
+			Msg("Deleting image check pod")
+
+		if err := podsCli.Delete(ctx, podName, metav1.DeleteOptions{}); err != nil {
+			log.Warn().Err(err).Msg("failed to delete image check pod")
+		}
+	}()
+
+	i := 0
+	for {
+		time.Sleep(2 * time.Second)
+
+		log.Debug().
+			Str("image", image).
+			Int("try", i).
+			Msg("Checking image")
+
+		i++
+
+		pod, err := podsCli.Get(ctx, podName, metav1.GetOptions{})
+		if err != nil {
+			return errors.WithMessage(err, "failed to get image check pod")
+		}
+
+		for _, containerStatus := range pod.Status.ContainerStatuses {
+			if containerStatus.Ready {
+				log.Debug().
+					Str("image", image).
+					Str("pod", podName).
+					Msg("Image check container ready, assuming image is available")
+
+				return nil
+			}
+
+			if containerStatus.State.Waiting != nil {
+				if containerStatus.State.Waiting.Reason == "ErrImagePull" || containerStatus.State.Waiting.Reason == "ImagePullBackOff" {
+					log.Debug().
+						Str("image", image).
+						Str("pod", podName).
+						Str("reason", containerStatus.State.Waiting.Reason).
+						Str("message", containerStatus.State.Waiting.Message).
+						Str("container", containerStatus.Name).
+						Msg("Image check container failed because of missing image")
+					return fmt.Errorf("image %s not found", image)
+				}
+			}
+		}
+	}
+}