feat(kubernetes/shell): kubectl web shell and kubeconfig functionality EE-448 (#5229)

* feat(kubernetes/shell): backport kubectl shell backend functionality EE-849 (#5168)

* backported core backend kubectl shell functionality

* - backported kubectl shell unit tests
- backported k8s cli interface update
- backported k8s client library fake patch

* refactored backend to match EE

* fixed test error typo

* GetServiceAccountName -> GetServiceAccount - making the function reusable in multiple contexts

* feat(kubernetes/shell): backport kubeconfig generation backend functionality EE-1004 (#5213)

* backported core backend kubectl shell functionality

* refactored backend to match EE

* - backported kubernetes backend handler implementation
- backported kubernetes config endpoint
- backported kubeconfig file generation
- backported kubeconfig and yaml unit tests
- backported updates to kubeclient interfaces

* feat(app): kubectl shell ui backport EE-927 (#5221)

* Kubectl UI backport to CE

* fix authentication redirect issue

* comment out redirect function

* fix shell full width & change name of shell

* disable button when terminal connected

* fixed whitespace changes for css

* fixed whitespace changes for html

* linting fixes

Co-authored-by: zees-dev <dev.786zshan@gmail.com>

* feat(kubernetes/shell): backport of kubeconfig export functionality EE-926 (#5228)

* EE backport of kubeconfig UI functionality

* using angularjs constant instead of hardcoded URL

* updated portainer kubectl shell image

* fix kubectl button position issue in ce

* fix pod keep running when switching page

* feat(app): Kubectl shell ui EE-833 EE-1099 (#5271)

* fix kubectl shell css

* fix mini css issue

* fix tech issue for ui changes from review

* delete unuse file

* - refactored variable names
- restored content-wrapper scroll
- created object to store wrapper css

Co-authored-by: zees-dev <dev.786zshan@gmail.com>

* addressing PR issues

* fix required changes from tech reviews (#5319)

* fix required changes from tech reviews

* remove unuse css variable

* component refactor accoridng to PR and style guidelines

Co-authored-by: zees-dev <dev.786zshan@gmail.com>

* removed redundant dockerhub api endpoint variable

* - autoHeight -> terminal-window
- removed redundant try-catch
- saving config.yaml file as config

* fix(kube/shell): show error on failure

* fixed default https bug

* resolved merge conflicts

Co-authored-by: Richard Wei <54336863+WaysonWei@users.noreply.github.com>
Co-authored-by: richard <richard@richards-iMac-Pro.local>
Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>

api/http/handler/handler.go

@@ -16,6 +16,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
+	"github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -47,6 +48,7 @@ type Handler struct {
 	EndpointHandler        *endpoints.Handler
 	EndpointProxyHandler   *endpointproxy.Handler
 	FileHandler            *file.Handler
+	KubernetesHandler      *kubernetes.Handler
 	MOTDHandler            *motd.Handler
 	RegistryHandler        *registries.Handler
 	ResourceControlHandler *resourcecontrols.Handler
@@ -100,6 +102,8 @@ type Handler struct {
 // @tag.description Manage Docker environments
 // @tag.name endpoint_groups
 // @tag.description Manage endpoint groups
+// @tag.name kubernetes
+// @tag.description Manage Kubernetes cluster
 // @tag.name motd
 // @tag.description Fetch the message of the day
 // @tag.name registries
@@ -156,6 +160,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.StripPrefix("/api", h.EdgeTemplatesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/endpoint_groups"):
 		http.StripPrefix("/api", h.EndpointGroupHandler).ServeHTTP(w, r)
+	case strings.HasPrefix(r.URL.Path, "/api/kubernetes"):
+		http.StripPrefix("/api", h.KubernetesHandler).ServeHTTP(w, r)
 	case strings.HasPrefix(r.URL.Path, "/api/endpoints"):
 		switch {
 		case strings.Contains(r.URL.Path, "/docker/"):

api/http/handler/kubernetes/handler.go

@@ -0,0 +1,28 @@
+package kubernetes
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	httperror "github.com/portainer/libhttp/error"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/security"
+	"github.com/portainer/portainer/api/kubernetes/cli"
+)
+
+// Handler is the HTTP handler which will natively deal with to external endpoints.
+type Handler struct {
+	*mux.Router
+	DataStore               portainer.DataStore
+	KubernetesClientFactory *cli.ClientFactory
+}
+
+// NewHandler creates a handler to process pre-proxied requests to external APIs.
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
+	h := &Handler{
+		Router: mux.NewRouter(),
+	}
+	h.PathPrefix("/kubernetes/{id}/config").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.getKubernetesConfig))).Methods(http.MethodGet)
+	return h
+}

api/http/handler/kubernetes/kubernetes_config.go

@@ -0,0 +1,132 @@
+package kubernetes
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/http/security"
+	kcli "github.com/portainer/portainer/api/kubernetes/cli"
+
+	"net/http"
+)
+
+// @id GetKubernetesConfig
+// @summary Generates kubeconfig file enabling client communication with k8s api server
+// @description Generates kubeconfig file enabling client communication with k8s api server
+// @description **Access policy**: authorized
+// @tags kubernetes
+// @security jwt
+// @accept json
+// @produce json
+// @param id path int true "Endpoint identifier"
+// @success 200 "Success"
+// @failure 400 "Invalid request"
+// @failure 401 "Unauthorized"
+// @failure 403 "Permission denied"
+// @failure 404 "Endpoint or ServiceAccount not found"
+// @failure 500 "Server error"
+// @router /kubernetes/{id}/config [get]
+func (handler *Handler) getKubernetesConfig(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	if r.TLS == nil {
+		return &httperror.HandlerError{
+			StatusCode: http.StatusInternalServerError,
+			Message:    "Kubernetes config generation only supported on portainer instances running with TLS",
+			Err:        errors.New("missing request TLS config"),
+		}
+	}
+
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	bearerToken, err := extractBearerToken(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusUnauthorized, "Unauthorized", err}
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	apiServerURL := getProxyUrl(r, endpointID)
+
+	config, err := cli.GetKubeConfig(r.Context(), apiServerURL, bearerToken, tokenData)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to generate Kubeconfig", err}
+	}
+
+	contentAcceptHeader := r.Header.Get("Accept")
+	if contentAcceptHeader == "text/yaml" {
+		yaml, err := kcli.GenerateYAML(config)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Failed to generate Kubeconfig", err}
+		}
+		w.Header().Set("Content-Disposition", `attachment; filename=config.yaml`)
+		return YAML(w, yaml)
+	}
+
+	w.Header().Set("Content-Disposition", `attachment; filename="config.json"`)
+	return response.JSON(w, config)
+}
+
+// extractBearerToken extracts user's portainer bearer token from request auth header
+func extractBearerToken(r *http.Request) (string, error) {
+	token := ""
+	tokens := r.Header["Authorization"]
+	if len(tokens) >= 1 {
+		token = tokens[0]
+		token = strings.TrimPrefix(token, "Bearer ")
+	}
+	if token == "" {
+		return "", httperrors.ErrUnauthorized
+	}
+	return token, nil
+}
+
+// getProxyUrl generates portainer proxy url which acts as proxy to k8s api server
+func getProxyUrl(r *http.Request, endpointID int) string {
+	return fmt.Sprintf("https://%s/api/endpoints/%d/kubernetes", r.Host, endpointID)
+}
+
+// YAML writes yaml response as string to writer. Returns a pointer to a HandlerError if encoding fails.
+// This could be moved to a more useful place; but that place is most likely not in this project.
+// It should actually go in https://github.com/portainer/libhttp - since that is from where we use response.JSON.
+// We use `data interface{}` as parameter - since im trying to keep it as close to (or the same as) response.JSON method signature:
+// https://github.com/portainer/libhttp/blob/d20481a3da823c619887c440a22fdf4fa8f318f2/response/response.go#L13
+func YAML(rw http.ResponseWriter, data interface{}) *httperror.HandlerError {
+	rw.Header().Set("Content-Type", "text/yaml")
+
+	strData, ok := data.(string)
+	if !ok {
+		return &httperror.HandlerError{
+			StatusCode: http.StatusInternalServerError,
+			Message:    "Unable to write YAML response",
+			Err:        errors.New("failed to convert input to string"),
+		}
+	}
+
+	fmt.Fprint(rw, strData)
+
+	return nil
+}

api/http/handler/websocket/handler.go

@@ -36,5 +36,7 @@ func NewHandler(kubernetesTokenCacheManager *kubernetes.TokenCacheManager, bounc
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketAttach)))
 	h.PathPrefix("/websocket/pod").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketPodExec)))
+	h.PathPrefix("/websocket/kubernetes-shell").Handler(
+		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.websocketShellPodExec)))
 	return h
 }

api/http/handler/websocket/pod.go

@@ -2,12 +2,13 @@ package websocket
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/http/security"
 	"io"
 	"log"
 	"net/http"
 	"strings"
 
+	"github.com/portainer/portainer/api/http/security"
+
 	"github.com/gorilla/websocket"
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
@@ -73,14 +74,14 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
 	}
 
-	token, useAdminToken, err := handler.getToken(r, endpoint, false)
+	serviceAccountToken, isAdminToken, err := handler.getToken(r, endpoint, false)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to get user service account token", err}
 	}
 
 	params := &webSocketRequestParams{
 		endpoint: endpoint,
-		token:    token,
+		token:    serviceAccountToken,
 	}
 
 	r.Header.Del("Origin")
@@ -99,6 +100,28 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 		return nil
 	}
 
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	handlerErr := handler.hijackPodExecStartOperation(w, r, cli, serviceAccountToken, isAdminToken, endpoint, namespace, podName, containerName, command)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
+	return nil
+}
+
+func (handler *Handler) hijackPodExecStartOperation(
+	w http.ResponseWriter,
+	r *http.Request,
+	cli portainer.KubeClient,
+	serviceAccountToken string,
+	isAdminToken bool,
+	endpoint *portainer.Endpoint,
+	namespace, podName, containerName, command string,
+) *httperror.HandlerError {
 	commandArray := strings.Split(command, " ")
 
 	websocketConn, err := handler.connectionUpgrader.Upgrade(w, r, nil)
@@ -116,12 +139,7 @@ func (handler *Handler) websocketPodExec(w http.ResponseWriter, r *http.Request)
 	go streamFromWebsocketToWriter(websocketConn, stdinWriter, errorChan)
 	go streamFromReaderToWebsocket(websocketConn, stdoutReader, errorChan)
 
-	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
-	}
-
-	err = cli.StartExecProcess(token, useAdminToken, namespace, podName, containerName, commandArray, stdinReader, stdoutWriter)
+	err = cli.StartExecProcess(serviceAccountToken, isAdminToken, namespace, podName, containerName, commandArray, stdinReader, stdoutWriter)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start exec process inside container", err}
 	}

api/http/handler/websocket/shell_pod.go

@@ -0,0 +1,111 @@
+package websocket
+
+import (
+	"net/http"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/security"
+)
+
+// websocketShellPodExec handles GET requests on /websocket/pod?token=<token>&endpointId=<endpointID>
+// The request will be upgraded to the websocket protocol.
+// Authentication and access is controlled via the mandatory token query parameter.
+// The request will proxy input from the client to the pod via long-lived websocket connection.
+// The following query parameters are mandatory:
+// * token: JWT token used for authentication against this endpoint
+// * endpointId: endpoint ID of the endpoint where the resource is located
+func (handler *Handler) websocketShellPodExec(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err}
+	}
+
+	tokenData, err := security.RetrieveTokenData(r)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
+	}
+
+	cli, err := handler.KubernetesClientFactory.GetKubeClient(endpoint)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Kubernetes client", err}
+	}
+
+	serviceAccount, err := cli.GetServiceAccount(tokenData)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find serviceaccount associated with user", err}
+	}
+
+	shellPod, err := cli.CreateUserShellPod(r.Context(), serviceAccount.Name)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create user shell", err}
+	}
+
+	// Modifying request params mid-flight before forewarding to K8s API server (websocket)
+	q := r.URL.Query()
+
+	q.Add("namespace", shellPod.Namespace)
+	q.Add("podName", shellPod.PodName)
+	q.Add("containerName", shellPod.ContainerName)
+	q.Add("command", shellPod.ShellExecCommand)
+
+	r.URL.RawQuery = q.Encode()
+
+	// Modify url path mid-flight before forewarding to k8s API server (websocket)
+	r.URL.Path = "/websocket/pod"
+
+	/*
+		Note: The following websocket proxying logic is duplicated from `api/http/handler/websocket/pod.go`
+	*/
+	params := &webSocketRequestParams{
+		endpoint: endpoint,
+	}
+
+	r.Header.Del("Origin")
+
+	if endpoint.Type == portainer.AgentOnKubernetesEnvironment {
+		err := handler.proxyAgentWebsocketRequest(w, r, params)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to proxy websocket request to agent", err}
+		}
+		return nil
+	} else if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		err := handler.proxyEdgeAgentWebsocketRequest(w, r, params)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to proxy websocket request to Edge agent", err}
+		}
+		return nil
+	}
+
+	serviceAccountToken, isAdminToken, err := handler.getToken(r, endpoint, false)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to get user service account token", err}
+	}
+
+	handlerErr := handler.hijackPodExecStartOperation(
+		w,
+		r,
+		cli,
+		serviceAccountToken,
+		isAdminToken,
+		endpoint,
+		shellPod.Namespace,
+		shellPod.PodName,
+		shellPod.ContainerName,
+		shellPod.ShellExecCommand,
+	)
+	if handlerErr != nil {
+		return handlerErr
+	}
+
+	return nil
+}

api/http/server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/portainer/portainer/api/http/handler/endpointproxy"
 	"github.com/portainer/portainer/api/http/handler/endpoints"
 	"github.com/portainer/portainer/api/http/handler/file"
+	kube "github.com/portainer/portainer/api/http/handler/kubernetes"
 	"github.com/portainer/portainer/api/http/handler/motd"
 	"github.com/portainer/portainer/api/http/handler/registries"
 	"github.com/portainer/portainer/api/http/handler/resourcecontrols"
@@ -152,6 +153,10 @@ func (server *Server) Start() error {
 
 	var fileHandler = file.NewHandler(filepath.Join(server.AssetsPath, "public"))
 
+	var kubernetesHandler = kube.NewHandler(requestBouncer)
+	kubernetesHandler.DataStore = server.DataStore
+	kubernetesHandler.KubernetesClientFactory = server.KubernetesClientFactory
+
 	var motdHandler = motd.NewHandler(requestBouncer)
 
 	var registryHandler = registries.NewHandler(requestBouncer)
@@ -226,6 +231,7 @@ func (server *Server) Start() error {
 		EndpointEdgeHandler:    endpointEdgeHandler,
 		EndpointProxyHandler:   endpointProxyHandler,
 		FileHandler:            fileHandler,
+		KubernetesHandler:      kubernetesHandler,
 		MOTDHandler:            motdHandler,
 		RegistryHandler:        registryHandler,
 		ResourceControlHandler: resourceControlHandler,