From 32a9a2e46b21320405871f0449e94b0cd560e836 Mon Sep 17 00:00:00 2001 From: Maxime Bajeux Date: Mon, 15 Mar 2021 22:36:14 +0100 Subject: [PATCH] Enable the ability to cordon/uncordon/drain nodes (#4723) * feat(node): Enable the ability to cordon/uncordon/drain nodes * feat(cluster): check if there is a drain operation somewhere * feat(kubernetes): allow to cordon, uncordon, drain nodes * refacto(kubernetes): set a constant for drain label name * fix(node): Relocate the warning message next to the dropdown and change the information message --- app/kubernetes/node/converter.js | 20 +++- app/kubernetes/node/formValues.js | 1 + app/kubernetes/node/models.js | 9 ++ app/kubernetes/node/service.js | 3 +- app/kubernetes/pod/converter.js | 9 +- app/kubernetes/pod/models/index.js | 15 +++ app/kubernetes/pod/service.js | 23 ++++- app/kubernetes/rest/pod.js | 1 + app/kubernetes/views/cluster/node/node.html | 20 ++++ .../views/cluster/node/nodeController.js | 94 +++++++++++++++++-- 10 files changed, 182 insertions(+), 13 deletions(-) diff --git a/app/kubernetes/node/converter.js b/app/kubernetes/node/converter.js index cc57e5c90..a7dc4d313 100644 --- a/app/kubernetes/node/converter.js +++ b/app/kubernetes/node/converter.js @@ -1,6 +1,6 @@ import _ from 'lodash-es'; -import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint } from 'Kubernetes/node/models'; +import { KubernetesNode, KubernetesNodeDetails, KubernetesNodeTaint, KubernetesNodeAvailabilities, KubernetesPortainerNodeDrainLabel } from 'Kubernetes/node/models'; import KubernetesResourceReservationHelper from 'Kubernetes/helpers/resourceReservationHelper'; import { KubernetesNodeFormValues, KubernetesNodeTaintFormValues, KubernetesNodeLabelFormValues } from 'Kubernetes/node/formValues'; import { KubernetesNodeCreatePayload, KubernetesNodeTaintPayload } from 'Kubernetes/node/payload'; @@ -30,6 +30,11 @@ class KubernetesNodeConverter { NetworkUnavailable: networkUnavailable && networkUnavailable.status === 'True', }; + res.Availability = KubernetesNodeAvailabilities.ACTIVE; + if (data.spec.unschedulable === true) { + res.Availability = _.has(data.metadata.labels, KubernetesPortainerNodeDrainLabel) ? KubernetesNodeAvailabilities.DRAIN : KubernetesNodeAvailabilities.PAUSE; + } + if (ready.status === 'False') { res.Status = 'Unhealthy'; } else if (ready.status === 'Unknown' || res.Conditions.MemoryPressure || res.Conditions.PIDPressure || res.Conditions.DiskPressure || res.Conditions.NetworkUnavailable) { @@ -67,6 +72,8 @@ class KubernetesNodeConverter { static nodeToFormValues(node) { const res = new KubernetesNodeFormValues(); + res.Availability = node.Availability; + res.Taints = _.map(node.Taints, (taint) => { const res = new KubernetesNodeTaintFormValues(); res.Key = taint.Key; @@ -92,6 +99,8 @@ class KubernetesNodeConverter { static formValuesToNode(node, formValues) { const res = angular.copy(node); + res.Availability = formValues.Availability; + const filteredTaints = _.filter(formValues.Taints, (taint) => !taint.NeedsDeletion); res.Taints = _.map(filteredTaints, (item) => { const taint = new KubernetesNodeTaint(); @@ -130,6 +139,15 @@ class KubernetesNodeConverter { payload.metadata.labels = node.Labels; + if (node.Availability !== KubernetesNodeAvailabilities.ACTIVE) { + payload.spec.unschedulable = true; + if (node.Availability === KubernetesNodeAvailabilities.DRAIN) { + payload.metadata.labels[KubernetesPortainerNodeDrainLabel] = ''; + } else { + delete payload.metadata.labels[KubernetesPortainerNodeDrainLabel]; + } + } + return payload; } diff --git a/app/kubernetes/node/formValues.js b/app/kubernetes/node/formValues.js index d7227ce70..51e9cb43f 100644 --- a/app/kubernetes/node/formValues.js +++ b/app/kubernetes/node/formValues.js @@ -1,6 +1,7 @@ const _KubernetesNodeFormValues = Object.freeze({ Taints: [], Labels: [], + Availability: '', }); export class KubernetesNodeFormValues { diff --git a/app/kubernetes/node/models.js b/app/kubernetes/node/models.js index e8e54a727..86bf8da4a 100644 --- a/app/kubernetes/node/models.js +++ b/app/kubernetes/node/models.js @@ -1,3 +1,5 @@ +export const KubernetesPortainerNodeDrainLabel = 'io.portainer/node-status-drain'; + /** * KubernetesNode Model */ @@ -14,6 +16,7 @@ const _KubernetesNode = Object.freeze({ Api: false, Taints: [], Port: 0, + Availability: '', }); export class KubernetesNode { @@ -58,6 +61,12 @@ export class KubernetesNodeTaint { } } +export const KubernetesNodeAvailabilities = Object.freeze({ + ACTIVE: 'Active', + PAUSE: 'Pause', + DRAIN: 'Drain', +}); + export const KubernetesNodeTaintEffects = Object.freeze({ NOSCHEDULE: 'NoSchedule', PREFERNOSCHEDULE: 'PreferNoSchedule', diff --git a/app/kubernetes/node/service.js b/app/kubernetes/node/service.js index 36b81e481..e2d207f7b 100644 --- a/app/kubernetes/node/service.js +++ b/app/kubernetes/node/service.js @@ -57,7 +57,8 @@ class KubernetesNodeService { const newNode = KubernetesNodeConverter.formValuesToNode(node, nodeFormValues); const payload = KubernetesNodeConverter.patchPayload(node, newNode); const data = await this.KubernetesNodes().patch(params, payload).$promise; - return data; + const patchedNode = KubernetesNodeConverter.apiToNodeDetails(data); + return patchedNode; } catch (err) { throw { msg: 'Unable to patch node', err: err }; } diff --git a/app/kubernetes/pod/converter.js b/app/kubernetes/pod/converter.js index a3b01b18c..0317b0901 100644 --- a/app/kubernetes/pod/converter.js +++ b/app/kubernetes/pod/converter.js @@ -10,7 +10,7 @@ import { } from 'Kubernetes/models/application/models'; import { createPayloadFactory } from './payloads/create'; -import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes } from './models'; +import { KubernetesPod, KubernetesPodToleration, KubernetesPodAffinity, KubernetesPodContainer, KubernetesPodContainerTypes, KubernetesPodEviction } from 'Kubernetes/pod/models'; function computeStatus(statuses) { const containerStatuses = _.map(statuses, 'state'); @@ -117,6 +117,13 @@ export default class KubernetesPodConverter { return res; } + static evictionPayload(pod) { + const res = new KubernetesPodEviction(); + res.metadata.name = pod.Name; + res.metadata.namespace = pod.Namespace; + return res; + } + static patchPayload(oldPod, newPod) { const oldPayload = createPayload(oldPod); const newPayload = createPayload(newPod); diff --git a/app/kubernetes/pod/models/index.js b/app/kubernetes/pod/models/index.js index a2590610f..d7a4950ea 100644 --- a/app/kubernetes/pod/models/index.js +++ b/app/kubernetes/pod/models/index.js @@ -65,6 +65,21 @@ export class KubernetesPodContainer { } } +const _KubernetesPodEviction = Object.freeze({ + apiVersion: 'policy/v1beta1', + kind: 'Eviction', + metadata: { + name: '', + namespace: '', + }, +}); + +export class KubernetesPodEviction { + constructor() { + Object.assign(this, JSON.parse(JSON.stringify(_KubernetesPodEviction))); + } +} + export const KubernetesPodContainerTypes = { INIT: 1, APP: 2, diff --git a/app/kubernetes/pod/service.js b/app/kubernetes/pod/service.js index af6359fb7..925b09797 100644 --- a/app/kubernetes/pod/service.js +++ b/app/kubernetes/pod/service.js @@ -2,7 +2,7 @@ import angular from 'angular'; import PortainerError from 'Portainer/error'; import { KubernetesCommonParams } from 'Kubernetes/models/common/params'; -import KubernetesPodConverter from './converter'; +import KubernetesPodConverter from 'Kubernetes/pod/converter'; class KubernetesPodService { /* @ngInject */ @@ -15,6 +15,7 @@ class KubernetesPodService { this.logsAsync = this.logsAsync.bind(this); this.deleteAsync = this.deleteAsync.bind(this); this.patchAsync = this.patchAsync.bind(this); + this.evictionAsync = this.evictionAsync.bind(this); } async getAsync(namespace, name) { @@ -116,6 +117,26 @@ class KubernetesPodService { delete(pod) { return this.$async(this.deleteAsync, pod); } + + /** + * EVICT + */ + async evictionAsync(pod) { + try { + const params = new KubernetesCommonParams(); + params.id = pod.Name; + params.action = 'eviction'; + const namespace = pod.Namespace; + const podEvictionPayload = KubernetesPodConverter.evictionPayload(pod); + await this.KubernetesPods(namespace).evict(params, podEvictionPayload).$promise; + } catch (err) { + throw new PortainerError('Unable to evict pod', err); + } + } + + eviction(pod) { + return this.$async(this.evictionAsync, pod); + } } export default KubernetesPodService; diff --git a/app/kubernetes/rest/pod.js b/app/kubernetes/rest/pod.js index 281b698ce..505558d41 100644 --- a/app/kubernetes/rest/pod.js +++ b/app/kubernetes/rest/pod.js @@ -42,6 +42,7 @@ angular.module('portainer.kubernetes').factory('KubernetesPods', [ params: { action: 'log' }, transformResponse: logsHandler, }, + evict: { method: 'POST' }, } ); }; diff --git a/app/kubernetes/views/cluster/node/node.html b/app/kubernetes/views/cluster/node/node.html index 87ae50227..c4c9229c1 100644 --- a/app/kubernetes/views/cluster/node/node.html +++ b/app/kubernetes/views/cluster/node/node.html @@ -52,6 +52,26 @@ + + + Availability + + + + + + Cannot use this action while another node is currently being drained. + + + + Cannot drain a node where this Portainer instance is running. + + + diff --git a/app/kubernetes/views/cluster/node/nodeController.js b/app/kubernetes/views/cluster/node/nodeController.js index cc4f7056b..47058f4c7 100644 --- a/app/kubernetes/views/cluster/node/nodeController.js +++ b/app/kubernetes/views/cluster/node/nodeController.js @@ -5,7 +5,7 @@ import { KubernetesResourceReservation } from 'Kubernetes/models/resource-reserv import KubernetesEventHelper from 'Kubernetes/helpers/eventHelper'; import KubernetesNodeConverter from 'Kubernetes/node/converter'; import { KubernetesNodeLabelFormValues, KubernetesNodeTaintFormValues } from 'Kubernetes/node/formValues'; -import { KubernetesNodeTaintEffects } from 'Kubernetes/node/models'; +import { KubernetesNodeTaintEffects, KubernetesNodeAvailabilities } from 'Kubernetes/node/models'; import KubernetesFormValidationHelper from 'Kubernetes/helpers/formValidationHelper'; import { KubernetesNodeHelper } from 'Kubernetes/node/helper'; @@ -35,12 +35,13 @@ class KubernetesNodeController { this.KubernetesEndpointService = KubernetesEndpointService; this.onInit = this.onInit.bind(this); - this.getNodeAsync = this.getNodeAsync.bind(this); + this.getNodesAsync = this.getNodesAsync.bind(this); this.getEvents = this.getEvents.bind(this); this.getEventsAsync = this.getEventsAsync.bind(this); this.getApplicationsAsync = this.getApplicationsAsync.bind(this); this.getEndpointsAsync = this.getEndpointsAsync.bind(this); this.updateNodeAsync = this.updateNodeAsync.bind(this); + this.drainNodeAsync = this.drainNodeAsync.bind(this); } selectTab(index) { @@ -152,6 +153,47 @@ class KubernetesNodeController { /* #endregion */ + /* #region cordon */ + + computeCordonWarning() { + return this.formValues.Availability === this.availabilities.PAUSE; + } + + /* #endregion */ + + /* #region drain */ + + computeDrainWarning() { + return this.formValues.Availability === this.availabilities.DRAIN; + } + + async drainNodeAsync() { + const pods = _.flatten(_.map(this.applications, (app) => app.Pods)); + let actionCount = pods.length; + for (const pod of pods) { + try { + await this.KubernetesPodService.eviction(pod); + this.Notifications.success('Pod successfully evicted', pod.Name); + } catch (err) { + this.Notifications.error('Failure', err, 'Unable to evict pod'); + this.formValues.Availability = this.availabilities.PAUSE; + await this.KubernetesNodeService.patch(this.node, this.formValues); + } finally { + --actionCount; + if (actionCount === 0) { + this.formValues.Availability = this.availabilities.PAUSE; + await this.KubernetesNodeService.patch(this.node, this.formValues); + } + } + } + } + + drainNode() { + return this.$async(this.drainNodeAsync); + } + + /* #endregion */ + /* #region actions */ isNoChangesMade() { @@ -160,8 +202,12 @@ class KubernetesNodeController { return !payload.length; } + isDrainError() { + return (this.state.isDrainOperation || this.state.isContainPortainer) && this.formValues.Availability === this.availabilities.DRAIN; + } + isFormValid() { - return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade(); + return !this.state.hasDuplicateTaintKeys && !this.state.hasDuplicateLabelKeys && !this.isNoChangesMade() && !this.isDrainError(); } resetFormValues() { @@ -196,7 +242,10 @@ class KubernetesNodeController { async updateNodeAsync() { try { - await this.KubernetesNodeService.patch(this.node, this.formValues); + this.node = await this.KubernetesNodeService.patch(this.node, this.formValues); + if (this.formValues.Availability === 'Drain') { + await this.drainNode(); + } this.Notifications.success('Node updated successfully'); this.$state.reload(); } catch (err) { @@ -207,6 +256,8 @@ class KubernetesNodeController { updateNode() { const taintsWarning = this.computeTaintsWarning(); const labelsWarning = this.computeLabelsWarning(); + const cordonWarning = this.computeCordonWarning(); + const drainWarning = this.computeDrainWarning(); if (taintsWarning && !labelsWarning) { this.ModalService.confirmUpdate( @@ -235,16 +286,36 @@ class KubernetesNodeController { } } ); + } else if (cordonWarning) { + this.ModalService.confirmUpdate( + 'Marking this node as unschedulable will effectively cordon the node and prevent any new workload from being scheduled on that node. Are you sure?', + (confirmed) => { + if (confirmed) { + return this.$async(this.updateNodeAsync); + } + } + ); + } else if (drainWarning) { + this.ModalService.confirmUpdate( + 'Draining this node will cause all workloads to be evicted from that node. This might lead to some service interruption. Are you sure?', + (confirmed) => { + if (confirmed) { + return this.$async(this.updateNodeAsync); + } + } + ); } else { return this.$async(this.updateNodeAsync); } } - async getNodeAsync() { + async getNodesAsync() { try { this.state.dataLoading = true; const nodeName = this.$transition$.params().name; - this.node = await this.KubernetesNodeService.get(nodeName); + this.nodes = await this.KubernetesNodeService.get(); + this.node = _.find(this.nodes, { Name: nodeName }); + this.state.isDrainOperation = _.find(this.nodes, { Availability: this.availabilities.DRAIN }); } catch (err) { this.Notifications.error('Failure', err, 'Unable to retrieve node'); } finally { @@ -252,8 +323,8 @@ class KubernetesNodeController { } } - getNode() { - return this.$async(this.getNodeAsync); + getNodes() { + return this.$async(this.getNodesAsync); } hasEventWarnings() { @@ -303,6 +374,7 @@ class KubernetesNodeController { }); this.resourceReservation.Memory = KubernetesResourceReservationHelper.megaBytesValue(this.resourceReservation.Memory); this.memoryLimit = KubernetesResourceReservationHelper.megaBytesValue(this.node.Memory); + this.state.isContainPortainer = _.find(this.applications, { ApplicationName: 'portainer' }); } catch (err) { this.Notifications.error('Failure', err, 'Unable to retrieve applications'); } finally { @@ -328,11 +400,15 @@ class KubernetesNodeController { hasDuplicateTaintKeys: false, duplicateLabelKeys: [], hasDuplicateLabelKeys: false, + isDrainOperation: false, + isContainPortainer: false, }; + this.availabilities = KubernetesNodeAvailabilities; + this.state.activeTab = this.LocalStorage.getActiveTab('node'); - await this.getNode(); + await this.getNodes(); await this.getEvents(); await this.getApplications(); await this.getEndpoints();