1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-24 15:59:41 +02:00

feat(app): rework private registries and support private registries in kubernetes EE-30 (#5131)

* feat(app): rework private registries and support private registries in kubernetes

[EE-30]

feat(api): backport private registries backend changes (#5072)

* feat(api/bolt): backport bolt changes

* feat(api/exec): backport exec changes

* feat(api/http): backport http/handler/dockerhub changes

* feat(api/http): backport http/handler/endpoints changes

* feat(api/http): backport http/handler/registries changes

* feat(api/http): backport http/handler/stacks changes

* feat(api/http): backport http/handler changes

* feat(api/http): backport http/proxy/factory/azure changes

* feat(api/http): backport http/proxy/factory/docker changes

* feat(api/http): backport http/proxy/factory/utils changes

* feat(api/http): backport http/proxy/factory/kubernetes changes

* feat(api/http): backport http/proxy/factory changes

* feat(api/http): backport http/security changes

* feat(api/http): backport http changes

* feat(api/internal): backport internal changes

* feat(api): backport api changes

* feat(api/kubernetes): backport kubernetes changes

* fix(api/http): changes on backend following backport

feat(app): backport private registries frontend changes (#5056)

* feat(app/docker): backport docker/components changes

* feat(app/docker): backport docker/helpers changes

* feat(app/docker): backport docker/views/container changes

* feat(app/docker): backport docker/views/images changes

* feat(app/docker): backport docker/views/registries changes

* feat(app/docker): backport docker/views/services changes

* feat(app/docker): backport docker changes

* feat(app/kubernetes): backport kubernetes/components changes

* feat(app/kubernetes): backport kubernetes/converters changes

* feat(app/kubernetes): backport kubernetes/models changes

* feat(app/kubernetes): backport kubernetes/registries changes

* feat(app/kubernetes): backport kubernetes/services changes

* feat(app/kubernetes): backport kubernetes/views/applications changes

* feat(app/kubernetes): backport kubernetes/views/configurations changes

* feat(app/kubernetes): backport kubernetes/views/configure changes

* feat(app/kubernetes): backport kubernetes/views/resource-pools changes

* feat(app/kubernetes): backport kubernetes/views changes

* feat(app/portainer): backport portainer/components/accessManagement changes

* feat(app/portainer): backport portainer/components/datatables changes

* feat(app/portainer): backport portainer/components/forms changes

* feat(app/portainer): backport portainer/components/registry-details changes

* feat(app/portainer): backport portainer/models changes

* feat(app/portainer): backport portainer/rest changes

* feat(app/portainer): backport portainer/services changes

* feat(app/portainer): backport portainer/views changes

* feat(app/portainer): backport portainer changes

* feat(app): backport app changes

* config(project): gitignore + jsconfig changes

gitignore all files under api/cmd/portainer but main.go and enable Code Editor autocomplete on import ... from '@/...'

fix(app): fix pull rate limit checker

fix(app/registries): sidebar menus and registry accesses users filtering

fix(api): add missing kube client factory

fix(kube): fetch dockerhub pull limits (#5133)

fix(app): pre review fixes (#5142)

* fix(app/registries): remove checkbox for endpointRegistries view

* fix(endpoints): allow access to default namespace

* fix(docker): fetch pull limits

* fix(kube/ns): show selected registries for non admin

Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>

chore(webpack): ignore missing sourcemaps

fix(registries): fetch registry config from url

feat(kube/registries): ignore not found when deleting secret

feat(db): move migration to db 31

fix(registries): fix bugs in PR EE-869 (#5169)

* fix(registries): hide role

* fix(endpoints): set empty access policy to edge endpoint

* fix(registry): remove double arguments

* fix(admin): ignore warning

* feat(kube/configurations): tag registry secrets (#5157)

* feat(kube/configurations): tag registry secrets

* feat(kube/secrets): show registry secrets for admins

* fix(registries): move dockerhub to beginning

* refactor(registries): use endpoint scoped registries

feat(registries): filter by namespace if supplied

feat(access-managment): filter users for registry (#5191)

* refactor(access-manage): move users selector to component

* feat(access-managment): filter users for registry

refactor(registries): sync code with CE (#5200)

* refactor(registry): add inspect handler under endpoints

* refactor(endpoint): sync endpoint_registries_list

* refactor(endpoints): sync registry_access

* fix(db): rename migration functions

* fix(registries): show accesses for admin

* fix(kube): set token on transport

* refactor(kube): move secret help to bottom

* fix(kuberentes): remove shouldLog parameter

* style(auth): add description of security.IsAdmin

* feat(security): allow admin access to registry

* feat(edge): connect to edge endpoint when creating client

* style(portainer): change deprecation version

* refactor(sidebar): hide manage

* refactor(containers): revert changes

* style(container): remove whitespace

* fix(endpoint): add handler to registy on endpointService

* refactor(image): use endpointService.registries

* fix(kueb/namespaces): rename resource pool to namespace

* fix(kube/namespace): move selected registries

* fix(api/registries): hide accesses on registry creation

Co-authored-by: LP B <xAt0mZ@users.noreply.github.com>

refactor(api): remove code duplication after rebase

fix(app/registries): replace last registry api usage by endpoint registry api

fix(api/endpoints): update registry access policies on endpoint deletion (#5226)

[EE-1027]

fix(db): update db version

* fix(dockerhub): fetch rate limits

* fix(registry/tests): supply restricred context

* fix(registries): show proget registry only when selected

* fix(registry): create dockerhub registry

* feat(db): move migrations to db 32

Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>
This commit is contained in:
LP B 2021-07-14 11:15:21 +02:00 committed by GitHub
parent 0f5407da40
commit 179df06267
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
175 changed files with 3757 additions and 2544 deletions

View file

@ -0,0 +1,60 @@
package kubernetes
import (
"crypto/tls"
"net/http"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/kubernetes/cli"
)
type agentTransport struct {
*baseTransport
signatureService portainer.DigitalSignatureService
}
// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
func NewAgentTransport(signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager, endpoint *portainer.Endpoint, k8sClientFactory *cli.ClientFactory, dataStore portainer.DataStore) *agentTransport {
transport := &agentTransport{
baseTransport: newBaseTransport(
&http.Transport{
TLSClientConfig: tlsConfig,
},
tokenManager,
endpoint,
k8sClientFactory,
dataStore,
),
signatureService: signatureService,
}
return transport
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
token, err := getRoundTripToken(request, transport.tokenManager)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
if strings.HasPrefix(request.URL.Path, "/v2") {
err := decorateAgentRequest(request, transport.dataStore)
if err != nil {
return nil, err
}
}
signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
return transport.baseTransport.RoundTrip(request)
}

View file

@ -0,0 +1,57 @@
package kubernetes
import (
"net/http"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/kubernetes/cli"
)
type edgeTransport struct {
*baseTransport
reverseTunnelService portainer.ReverseTunnelService
}
// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent
func NewEdgeTransport(reverseTunnelService portainer.ReverseTunnelService, endpoint *portainer.Endpoint, tokenManager *tokenManager, k8sClientFactory *cli.ClientFactory, dataStore portainer.DataStore) *edgeTransport {
transport := &edgeTransport{
baseTransport: newBaseTransport(
&http.Transport{},
tokenManager,
endpoint,
k8sClientFactory,
dataStore,
),
reverseTunnelService: reverseTunnelService,
}
return transport
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *edgeTransport) RoundTrip(request *http.Request) (*http.Response, error) {
token, err := getRoundTripToken(request, transport.tokenManager)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
if strings.HasPrefix(request.URL.Path, "/v2") {
err := decorateAgentRequest(request, transport.dataStore)
if err != nil {
return nil, err
}
}
response, err := transport.baseTransport.RoundTrip(request)
if err == nil {
transport.reverseTunnelService.SetTunnelStatusToActive(transport.endpoint.ID)
} else {
transport.reverseTunnelService.SetTunnelStatusToIdle(transport.endpoint.ID)
}
return response, err
}

View file

@ -0,0 +1,45 @@
package kubernetes
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/kubernetes/cli"
)
type localTransport struct {
*baseTransport
}
// NewLocalTransport returns a new transport that can be used to send requests to the local Kubernetes API
func NewLocalTransport(tokenManager *tokenManager, endpoint *portainer.Endpoint, k8sClientFactory *cli.ClientFactory, dataStore portainer.DataStore) (*localTransport, error) {
config, err := crypto.CreateTLSConfigurationFromBytes(nil, nil, nil, true, true)
if err != nil {
return nil, err
}
transport := &localTransport{
baseTransport: newBaseTransport(
&http.Transport{
TLSClientConfig: config,
},
tokenManager,
endpoint,
k8sClientFactory,
dataStore,
),
}
return transport, nil
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *localTransport) RoundTrip(request *http.Request) (*http.Response, error) {
_, err := transport.prepareRoundTrip(request)
if err != nil {
return nil, err
}
return transport.baseTransport.RoundTrip(request)
}

View file

@ -0,0 +1,45 @@
package kubernetes
import (
"net/http"
portainer "github.com/portainer/portainer/api"
)
func (transport *baseTransport) proxyNamespaceDeleteOperation(request *http.Request, namespace string) (*http.Response, error) {
registries, err := transport.dataStore.Registry().Registries()
if err != nil {
return nil, err
}
for _, registry := range registries {
for endpointID, registryAccessPolicies := range registry.RegistryAccesses {
if endpointID != transport.endpoint.ID {
continue
}
namespaces := []string{}
for _, ns := range registryAccessPolicies.Namespaces {
if ns == namespace {
continue
}
namespaces = append(namespaces, ns)
}
if len(namespaces) != len(registryAccessPolicies.Namespaces) {
updatedAccessPolicies := portainer.RegistryAccessPolicies{
Namespaces: namespaces,
UserAccessPolicies: registryAccessPolicies.UserAccessPolicies,
TeamAccessPolicies: registryAccessPolicies.TeamAccessPolicies,
}
registry.RegistryAccesses[endpointID] = updatedAccessPolicies
err := transport.dataStore.Registry().UpdateRegistry(registry.ID, &registry)
if err != nil {
return nil, err
}
}
}
}
return transport.executeKubernetesRequest(request)
}

View file

@ -0,0 +1,170 @@
package kubernetes
import (
"net/http"
"path"
"github.com/portainer/portainer/api/http/proxy/factory/utils"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/privateregistries"
v1 "k8s.io/api/core/v1"
)
func (transport *baseTransport) proxySecretRequest(request *http.Request, namespace, requestPath string) (*http.Response, error) {
switch request.Method {
case "POST":
return transport.proxySecretCreationOperation(request)
case "GET":
if path.Base(requestPath) == "secrets" {
return transport.proxySecretListOperation(request)
}
return transport.proxySecretInspectOperation(request)
case "PUT":
return transport.proxySecretUpdateOperation(request)
case "DELETE":
return transport.proxySecretDeleteOperation(request, namespace)
default:
return transport.executeKubernetesRequest(request)
}
}
func (transport *baseTransport) proxySecretCreationOperation(request *http.Request) (*http.Response, error) {
body, err := utils.GetRequestAsMap(request)
if err != nil {
return nil, err
}
if isSecretRepresentPrivateRegistry(body) {
return utils.WriteAccessDeniedResponse()
}
err = utils.RewriteRequest(request, body)
if err != nil {
return nil, err
}
return transport.executeKubernetesRequest(request)
}
func (transport *baseTransport) proxySecretListOperation(request *http.Request) (*http.Response, error) {
response, err := transport.executeKubernetesRequest(request)
if err != nil {
return nil, err
}
isAdmin, err := security.IsAdmin(request)
if err != nil {
return nil, err
}
if isAdmin {
return response, nil
}
body, err := utils.GetResponseAsJSONObject(response)
if err != nil {
return nil, err
}
items := utils.GetArrayObject(body, "items")
if items == nil {
utils.RewriteResponse(response, body, response.StatusCode)
return response, nil
}
filteredItems := []interface{}{}
for _, item := range items {
itemObj := item.(map[string]interface{})
if !isSecretRepresentPrivateRegistry(itemObj) {
filteredItems = append(filteredItems, item)
}
}
body["items"] = filteredItems
utils.RewriteResponse(response, body, response.StatusCode)
return response, nil
}
func (transport *baseTransport) proxySecretInspectOperation(request *http.Request) (*http.Response, error) {
response, err := transport.executeKubernetesRequest(request)
if err != nil {
return nil, err
}
isAdmin, err := security.IsAdmin(request)
if err != nil {
return nil, err
}
if isAdmin {
return response, nil
}
body, err := utils.GetResponseAsJSONObject(response)
if err != nil {
return nil, err
}
if isSecretRepresentPrivateRegistry(body) {
return utils.WriteAccessDeniedResponse()
}
err = utils.RewriteResponse(response, body, response.StatusCode)
if err != nil {
return nil, err
}
return response, nil
}
func (transport *baseTransport) proxySecretUpdateOperation(request *http.Request) (*http.Response, error) {
body, err := utils.GetRequestAsMap(request)
if err != nil {
return nil, err
}
if isSecretRepresentPrivateRegistry(body) {
return utils.WriteAccessDeniedResponse()
}
err = utils.RewriteRequest(request, body)
if err != nil {
return nil, err
}
return transport.executeKubernetesRequest(request)
}
func (transport *baseTransport) proxySecretDeleteOperation(request *http.Request, namespace string) (*http.Response, error) {
kcl, err := transport.k8sClientFactory.GetKubeClient(transport.endpoint)
if err != nil {
return nil, err
}
secretName := path.Base(request.RequestURI)
isRegistry, err := kcl.IsRegistrySecret(namespace, secretName)
if err != nil {
return nil, err
}
if isRegistry {
return utils.WriteAccessDeniedResponse()
}
return transport.executeKubernetesRequest(request)
}
func isSecretRepresentPrivateRegistry(secret map[string]interface{}) bool {
if secret["type"].(string) != string(v1.SecretTypeDockerConfigJson) {
return false
}
metadata := utils.GetJSONObject(secret, "metadata")
annotations := utils.GetJSONObject(metadata, "annotations")
_, ok := annotations[privateregistries.RegistryIDLabel]
return ok
}

View file

@ -2,153 +2,107 @@ package kubernetes
import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"path"
"regexp"
"strconv"
"strings"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/crypto"
)
type (
localTransport struct {
httpTransport *http.Transport
tokenManager *tokenManager
endpointIdentifier portainer.EndpointID
}
agentTransport struct {
dataStore portainer.DataStore
httpTransport *http.Transport
tokenManager *tokenManager
signatureService portainer.DigitalSignatureService
endpointIdentifier portainer.EndpointID
}
edgeTransport struct {
dataStore portainer.DataStore
httpTransport *http.Transport
tokenManager *tokenManager
reverseTunnelService portainer.ReverseTunnelService
endpointIdentifier portainer.EndpointID
}
)
// NewLocalTransport returns a new transport that can be used to send requests to the local Kubernetes API
func NewLocalTransport(tokenManager *tokenManager) (*localTransport, error) {
config, err := crypto.CreateTLSConfigurationFromBytes(nil, nil, nil, true, true)
if err != nil {
return nil, err
}
transport := &localTransport{
httpTransport: &http.Transport{
TLSClientConfig: config,
},
tokenManager: tokenManager,
}
return transport, nil
type baseTransport struct {
httpTransport *http.Transport
tokenManager *tokenManager
endpoint *portainer.Endpoint
k8sClientFactory *cli.ClientFactory
dataStore portainer.DataStore
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *localTransport) RoundTrip(request *http.Request) (*http.Response, error) {
token, err := getRoundTripToken(request, transport.tokenManager, transport.endpointIdentifier)
func newBaseTransport(httpTransport *http.Transport, tokenManager *tokenManager, endpoint *portainer.Endpoint, k8sClientFactory *cli.ClientFactory, dataStore portainer.DataStore) *baseTransport {
return &baseTransport{
httpTransport: httpTransport,
tokenManager: tokenManager,
endpoint: endpoint,
k8sClientFactory: k8sClientFactory,
dataStore: dataStore,
}
}
// #region KUBERNETES PROXY
// proxyKubernetesRequest intercepts a Kubernetes API request and apply logic based
// on the requested operation.
func (transport *baseTransport) proxyKubernetesRequest(request *http.Request) (*http.Response, error) {
apiVersionRe := regexp.MustCompile(`^(/kubernetes)?/api/v[0-9](\.[0-9])?`)
requestPath := apiVersionRe.ReplaceAllString(request.URL.Path, "")
switch {
case strings.EqualFold(requestPath, "/namespaces"):
return transport.executeKubernetesRequest(request)
case strings.HasPrefix(requestPath, "/namespaces"):
return transport.proxyNamespacedRequest(request, requestPath)
default:
return transport.executeKubernetesRequest(request)
}
}
func (transport *baseTransport) proxyNamespacedRequest(request *http.Request, fullRequestPath string) (*http.Response, error) {
requestPath := strings.TrimPrefix(fullRequestPath, "/namespaces/")
split := strings.SplitN(requestPath, "/", 2)
namespace := split[0]
requestPath = ""
if len(split) > 1 {
requestPath = split[1]
}
switch {
case strings.HasPrefix(requestPath, "secrets"):
return transport.proxySecretRequest(request, namespace, requestPath)
case requestPath == "" && request.Method == "DELETE":
return transport.proxyNamespaceDeleteOperation(request, namespace)
default:
return transport.executeKubernetesRequest(request)
}
}
func (transport *baseTransport) executeKubernetesRequest(request *http.Request) (*http.Response, error) {
resp, err := transport.httpTransport.RoundTrip(request)
return resp, err
}
// #endregion
// #region ROUND TRIP
func (transport *baseTransport) prepareRoundTrip(request *http.Request) (string, error) {
token, err := getRoundTripToken(request, transport.tokenManager)
if err != nil {
return nil, err
return "", err
}
request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
return transport.httpTransport.RoundTrip(request)
}
// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
func NewAgentTransport(datastore portainer.DataStore, signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager) *agentTransport {
transport := &agentTransport{
dataStore: datastore,
httpTransport: &http.Transport{
TLSClientConfig: tlsConfig,
},
tokenManager: tokenManager,
signatureService: signatureService,
}
return transport
return token, nil
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Response, error) {
token, err := getRoundTripToken(request, transport.tokenManager, transport.endpointIdentifier)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
if strings.HasPrefix(request.URL.Path, "/v2") {
decorateAgentRequest(request, transport.dataStore)
}
signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentPublicKeyHeader, transport.signatureService.EncodedPublicKey())
request.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
return transport.httpTransport.RoundTrip(request)
func (transport *baseTransport) RoundTrip(request *http.Request) (*http.Response, error) {
return transport.proxyKubernetesRequest(request)
}
// NewEdgeTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent
func NewEdgeTransport(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, endpointIdentifier portainer.EndpointID, tokenManager *tokenManager) *edgeTransport {
transport := &edgeTransport{
dataStore: datastore,
httpTransport: &http.Transport{},
tokenManager: tokenManager,
reverseTunnelService: reverseTunnelService,
endpointIdentifier: endpointIdentifier,
}
return transport
}
// RoundTrip is the implementation of the the http.RoundTripper interface
func (transport *edgeTransport) RoundTrip(request *http.Request) (*http.Response, error) {
token, err := getRoundTripToken(request, transport.tokenManager, transport.endpointIdentifier)
if err != nil {
return nil, err
}
request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
if strings.HasPrefix(request.URL.Path, "/v2") {
decorateAgentRequest(request, transport.dataStore)
}
response, err := transport.httpTransport.RoundTrip(request)
if err == nil {
transport.reverseTunnelService.SetTunnelStatusToActive(transport.endpointIdentifier)
} else {
transport.reverseTunnelService.SetTunnelStatusToIdle(transport.endpointIdentifier)
}
return response, err
}
func getRoundTripToken(
request *http.Request,
tokenManager *tokenManager,
endpointIdentifier portainer.EndpointID,
) (string, error) {
func getRoundTripToken(request *http.Request, tokenManager *tokenManager) (string, error) {
tokenData, err := security.RetrieveTokenData(request)
if err != nil {
return "", err
@ -168,32 +122,56 @@ func getRoundTripToken(
return token, nil
}
// #endregion
// #region DECORATE FUNCTIONS
func decorateAgentRequest(r *http.Request, dataStore portainer.DataStore) error {
requestPath := strings.TrimPrefix(r.URL.Path, "/v2")
switch {
case strings.HasPrefix(requestPath, "/dockerhub"):
decorateAgentDockerHubRequest(r, dataStore)
return decorateAgentDockerHubRequest(r, dataStore)
}
return nil
}
func decorateAgentDockerHubRequest(r *http.Request, dataStore portainer.DataStore) error {
dockerhub, err := dataStore.DockerHub().DockerHub()
requestPath, registryIdString := path.Split(r.URL.Path)
registryID, err := strconv.Atoi(registryIdString)
if err != nil {
return err
return fmt.Errorf("missing registry id: %w", err)
}
newBody, err := json.Marshal(dockerhub)
r.URL.Path = strings.TrimSuffix(requestPath, "/")
registry := &portainer.Registry{
Type: portainer.DockerHubRegistry,
}
if registryID != 0 {
registry, err = dataStore.Registry().Registry(portainer.RegistryID(registryID))
if err != nil {
return fmt.Errorf("failed fetching registry: %w", err)
}
}
if registry.Type != portainer.DockerHubRegistry {
return errors.New("invalid registry type")
}
newBody, err := json.Marshal(registry)
if err != nil {
return err
return fmt.Errorf("failed marshaling registry: %w", err)
}
r.Method = http.MethodPost
r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
r.ContentLength = int64(len(newBody))
return nil
}
// #endregion