1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-25 00:09:40 +02:00

feat(app): rework private registries and support private registries in kubernetes EE-30 (#5131)

* feat(app): rework private registries and support private registries in kubernetes

[EE-30]

feat(api): backport private registries backend changes (#5072)

* feat(api/bolt): backport bolt changes

* feat(api/exec): backport exec changes

* feat(api/http): backport http/handler/dockerhub changes

* feat(api/http): backport http/handler/endpoints changes

* feat(api/http): backport http/handler/registries changes

* feat(api/http): backport http/handler/stacks changes

* feat(api/http): backport http/handler changes

* feat(api/http): backport http/proxy/factory/azure changes

* feat(api/http): backport http/proxy/factory/docker changes

* feat(api/http): backport http/proxy/factory/utils changes

* feat(api/http): backport http/proxy/factory/kubernetes changes

* feat(api/http): backport http/proxy/factory changes

* feat(api/http): backport http/security changes

* feat(api/http): backport http changes

* feat(api/internal): backport internal changes

* feat(api): backport api changes

* feat(api/kubernetes): backport kubernetes changes

* fix(api/http): changes on backend following backport

feat(app): backport private registries frontend changes (#5056)

* feat(app/docker): backport docker/components changes

* feat(app/docker): backport docker/helpers changes

* feat(app/docker): backport docker/views/container changes

* feat(app/docker): backport docker/views/images changes

* feat(app/docker): backport docker/views/registries changes

* feat(app/docker): backport docker/views/services changes

* feat(app/docker): backport docker changes

* feat(app/kubernetes): backport kubernetes/components changes

* feat(app/kubernetes): backport kubernetes/converters changes

* feat(app/kubernetes): backport kubernetes/models changes

* feat(app/kubernetes): backport kubernetes/registries changes

* feat(app/kubernetes): backport kubernetes/services changes

* feat(app/kubernetes): backport kubernetes/views/applications changes

* feat(app/kubernetes): backport kubernetes/views/configurations changes

* feat(app/kubernetes): backport kubernetes/views/configure changes

* feat(app/kubernetes): backport kubernetes/views/resource-pools changes

* feat(app/kubernetes): backport kubernetes/views changes

* feat(app/portainer): backport portainer/components/accessManagement changes

* feat(app/portainer): backport portainer/components/datatables changes

* feat(app/portainer): backport portainer/components/forms changes

* feat(app/portainer): backport portainer/components/registry-details changes

* feat(app/portainer): backport portainer/models changes

* feat(app/portainer): backport portainer/rest changes

* feat(app/portainer): backport portainer/services changes

* feat(app/portainer): backport portainer/views changes

* feat(app/portainer): backport portainer changes

* feat(app): backport app changes

* config(project): gitignore + jsconfig changes

gitignore all files under api/cmd/portainer but main.go and enable Code Editor autocomplete on import ... from '@/...'

fix(app): fix pull rate limit checker

fix(app/registries): sidebar menus and registry accesses users filtering

fix(api): add missing kube client factory

fix(kube): fetch dockerhub pull limits (#5133)

fix(app): pre review fixes (#5142)

* fix(app/registries): remove checkbox for endpointRegistries view

* fix(endpoints): allow access to default namespace

* fix(docker): fetch pull limits

* fix(kube/ns): show selected registries for non admin

Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>

chore(webpack): ignore missing sourcemaps

fix(registries): fetch registry config from url

feat(kube/registries): ignore not found when deleting secret

feat(db): move migration to db 31

fix(registries): fix bugs in PR EE-869 (#5169)

* fix(registries): hide role

* fix(endpoints): set empty access policy to edge endpoint

* fix(registry): remove double arguments

* fix(admin): ignore warning

* feat(kube/configurations): tag registry secrets (#5157)

* feat(kube/configurations): tag registry secrets

* feat(kube/secrets): show registry secrets for admins

* fix(registries): move dockerhub to beginning

* refactor(registries): use endpoint scoped registries

feat(registries): filter by namespace if supplied

feat(access-managment): filter users for registry (#5191)

* refactor(access-manage): move users selector to component

* feat(access-managment): filter users for registry

refactor(registries): sync code with CE (#5200)

* refactor(registry): add inspect handler under endpoints

* refactor(endpoint): sync endpoint_registries_list

* refactor(endpoints): sync registry_access

* fix(db): rename migration functions

* fix(registries): show accesses for admin

* fix(kube): set token on transport

* refactor(kube): move secret help to bottom

* fix(kuberentes): remove shouldLog parameter

* style(auth): add description of security.IsAdmin

* feat(security): allow admin access to registry

* feat(edge): connect to edge endpoint when creating client

* style(portainer): change deprecation version

* refactor(sidebar): hide manage

* refactor(containers): revert changes

* style(container): remove whitespace

* fix(endpoint): add handler to registy on endpointService

* refactor(image): use endpointService.registries

* fix(kueb/namespaces): rename resource pool to namespace

* fix(kube/namespace): move selected registries

* fix(api/registries): hide accesses on registry creation

Co-authored-by: LP B <xAt0mZ@users.noreply.github.com>

refactor(api): remove code duplication after rebase

fix(app/registries): replace last registry api usage by endpoint registry api

fix(api/endpoints): update registry access policies on endpoint deletion (#5226)

[EE-1027]

fix(db): update db version

* fix(dockerhub): fetch rate limits

* fix(registry/tests): supply restricred context

* fix(registries): show proget registry only when selected

* fix(registry): create dockerhub registry

* feat(db): move migrations to db 32

Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>
This commit is contained in:
LP B 2021-07-14 11:15:21 +02:00 committed by GitHub
parent 0f5407da40
commit 179df06267
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
175 changed files with 3757 additions and 2544 deletions

View file

@ -322,8 +322,8 @@ func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload)
TLSConfig: portainer.TLSConfiguration{
TLS: false,
},
AuthorizedUsers: []portainer.UserID{},
AuthorizedTeams: []portainer.TeamID{},
UserAccessPolicies: portainer.UserAccessPolicies{},
TeamAccessPolicies: portainer.TeamAccessPolicies{},
Extensions: []portainer.EndpointExtension{},
TagIDs: payload.TagIDs,
Status: portainer.EndpointStatusUp,

View file

@ -103,6 +103,22 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
}
}
registries, err := handler.DataStore.Registry().Registries()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
}
for idx := range registries {
registry := &registries[idx]
if _, ok := registry.RegistryAccesses[endpoint.ID]; ok {
delete(registry.RegistryAccesses, endpoint.ID)
err = handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update registry accesses", Err: err}
}
}
}
return response.Empty(w)
}

View file

@ -22,7 +22,7 @@ type dockerhubStatusResponse struct {
Limit int `json:"limit"`
}
// GET request on /api/endpoints/{id}/dockerhub/status
// GET request on /api/endpoints/{id}/dockerhub/{registryId}
func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
@ -40,13 +40,30 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")}
}
dockerhub, err := handler.DataStore.DockerHub().DockerHub()
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
}
var registry *portainer.Registry
if registryID == 0 {
registry = &portainer.Registry{}
} else {
registry, err = handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
} else if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
}
if registry.Type != portainer.DockerHubRegistry {
return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry type", errors.New("Invalid registry type")}
}
}
httpClient := client.NewHTTPClient()
token, err := getDockerHubToken(httpClient, dockerhub)
token, err := getDockerHubToken(httpClient, registry)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err}
}
@ -59,7 +76,7 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
return response.JSON(w, resp)
}
func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) {
func getDockerHubToken(httpClient *client.HTTPClient, registry *portainer.Registry) (string, error) {
type dockerhubTokenResponse struct {
Token string `json:"token"`
}
@ -71,8 +88,8 @@ func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.Docke
return "", err
}
if dockerhub.Authentication {
req.SetBasicAuth(dockerhub.Username, dockerhub.Password)
if registry.Authentication {
req.SetBasicAuth(registry.Username, registry.Password)
}
resp, err := httpClient.Do(req)

View file

@ -0,0 +1,50 @@
package endpoints
import (
"net/http"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
bolterrors "github.com/portainer/portainer/api/bolt/errors"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
)
// GET request on /endpoints/{id}/registries/{registryId}
func (handler *Handler) endpointRegistryInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
}
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid registry identifier route variable", Err: err}
}
registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a registry with the specified identifier inside the database", Err: err}
} else if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a registry with the specified identifier inside the database", Err: err}
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err}
}
user, err := handler.DataStore.User().User(securityContext.UserID)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve user from the database", Err: err}
}
if !security.AuthorizedRegistryAccess(registry, user, securityContext.UserMemberships, portainer.EndpointID(endpointID)) {
return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied}
}
hideRegistryFields(registry, !securityContext.IsAdmin)
return response.JSON(w, registry)
}

View file

@ -0,0 +1,130 @@
package endpoints
import (
"net/http"
"github.com/pkg/errors"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
bolterrors "github.com/portainer/portainer/api/bolt/errors"
"github.com/portainer/portainer/api/http/security"
endpointutils "github.com/portainer/portainer/api/internal/endpoint"
)
// GET request on /endpoints/{id}/registries?namespace
func (handler *Handler) endpointRegistriesList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
}
user, err := handler.DataStore.User().User(securityContext.UserID)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve user from the database", err}
}
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
} else if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
}
isAdmin := securityContext.IsAdmin
registries, err := handler.DataStore.Registry().Registries()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
}
if endpointutils.IsKubernetesEndpoint(endpoint) {
namespace, _ := request.RetrieveQueryParameter(r, "namespace", true)
if namespace == "" && !isAdmin {
return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Missing namespace query parameter", Err: errors.New("missing namespace query parameter")}
}
if namespace != "" {
authorized, err := handler.isNamespaceAuthorized(endpoint, namespace, user.ID, securityContext.UserMemberships, isAdmin)
if err != nil {
return &httperror.HandlerError{http.StatusNotFound, "Unable to check for namespace authorization", err}
}
if !authorized {
return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "User is not authorized to use namespace", Err: errors.New("user is not authorized to use namespace")}
}
registries = filterRegistriesByNamespace(registries, endpoint.ID, namespace)
}
} else if !isAdmin {
registries = security.FilterRegistries(registries, user, securityContext.UserMemberships, endpoint.ID)
}
for idx := range registries {
hideRegistryFields(&registries[idx], !isAdmin)
}
return response.JSON(w, registries)
}
func (handler *Handler) isNamespaceAuthorized(endpoint *portainer.Endpoint, namespace string, userId portainer.UserID, memberships []portainer.TeamMembership, isAdmin bool) (bool, error) {
if isAdmin || namespace == "" {
return true, nil
}
if namespace == "default" {
return true, nil
}
kcl, err := handler.K8sClientFactory.GetKubeClient(endpoint)
if err != nil {
return false, errors.Wrap(err, "unable to retrieve kubernetes client")
}
accessPolicies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return false, errors.Wrap(err, "unable to retrieve endpoint's namespaces policies")
}
namespacePolicy, ok := accessPolicies[namespace]
if !ok {
return false, nil
}
return security.AuthorizedAccess(userId, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies), nil
}
func filterRegistriesByNamespace(registries []portainer.Registry, endpointId portainer.EndpointID, namespace string) []portainer.Registry {
if namespace == "" {
return registries
}
filteredRegistries := []portainer.Registry{}
for _, registry := range registries {
for _, authorizedNamespace := range registry.RegistryAccesses[endpointId].Namespaces {
if authorizedNamespace == namespace {
filteredRegistries = append(filteredRegistries, registry)
}
}
}
return filteredRegistries
}
func hideRegistryFields(registry *portainer.Registry, hideAccesses bool) {
registry.Password = ""
registry.ManagementConfiguration = nil
if hideAccesses {
registry.RegistryAccesses = nil
}
}

View file

@ -0,0 +1,149 @@
package endpoints
import (
"net/http"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
bolterrors "github.com/portainer/portainer/api/bolt/errors"
"github.com/portainer/portainer/api/http/security"
)
type registryAccessPayload struct {
UserAccessPolicies portainer.UserAccessPolicies
TeamAccessPolicies portainer.TeamAccessPolicies
Namespaces []string
}
func (payload *registryAccessPayload) Validate(r *http.Request) error {
return nil
}
// PUT request on /endpoints/{id}/registries/{registryId}
func (handler *Handler) endpointRegistryAccess(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid endpoint identifier route variable", Err: err}
}
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid registry identifier route variable", Err: err}
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
} else if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
}
err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint)
if err != nil {
return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
}
if !securityContext.IsAdmin {
return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "User is not authorized", Err: err}
}
registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
} else if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find an endpoint with the specified identifier inside the database", Err: err}
}
var payload registryAccessPayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
}
if registry.RegistryAccesses == nil {
registry.RegistryAccesses = portainer.RegistryAccesses{}
}
if _, ok := registry.RegistryAccesses[endpoint.ID]; !ok {
registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{}
}
registryAccess := registry.RegistryAccesses[endpoint.ID]
if endpoint.Type == portainer.KubernetesLocalEnvironment || endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
err := handler.updateKubeAccess(endpoint, registry, registryAccess.Namespaces, payload.Namespaces)
if err != nil {
return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to update kube access policies", Err: err}
}
registryAccess.Namespaces = payload.Namespaces
} else {
registryAccess.UserAccessPolicies = payload.UserAccessPolicies
registryAccess.TeamAccessPolicies = payload.TeamAccessPolicies
}
registry.RegistryAccesses[portainer.EndpointID(endpointID)] = registryAccess
handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
return response.Empty(w)
}
func (handler *Handler) updateKubeAccess(endpoint *portainer.Endpoint, registry *portainer.Registry, oldNamespaces, newNamespaces []string) error {
oldNamespacesSet := toSet(oldNamespaces)
newNamespacesSet := toSet(newNamespaces)
namespacesToRemove := setDifference(oldNamespacesSet, newNamespacesSet)
namespacesToAdd := setDifference(newNamespacesSet, oldNamespacesSet)
cli, err := handler.K8sClientFactory.GetKubeClient(endpoint)
if err != nil {
return err
}
for namespace := range namespacesToRemove {
err := cli.DeleteRegistrySecret(registry, namespace)
if err != nil {
return err
}
}
for namespace := range namespacesToAdd {
err := cli.CreateRegistrySecret(registry, namespace)
if err != nil {
return err
}
}
return nil
}
type stringSet map[string]bool
func toSet(list []string) stringSet {
set := stringSet{}
for _, el := range list {
set[el] = true
}
return set
}
// setDifference returns the set difference tagsA - tagsB
func setDifference(setA stringSet, setB stringSet) stringSet {
set := stringSet{}
for el := range setA {
if !setB[el] {
set[el] = true
}
}
return set
}

View file

@ -6,6 +6,7 @@ import (
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/kubernetes/cli"
"net/http"
@ -28,6 +29,7 @@ type Handler struct {
ProxyManager *proxy.Manager
ReverseTunnelService portainer.ReverseTunnelService
SnapshotService portainer.SnapshotService
K8sClientFactory *cli.ClientFactory
ComposeStackManager portainer.ComposeStackManager
AuthorizationService *authorization.Service
}
@ -53,7 +55,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
h.Handle("/endpoints/{id}",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
h.Handle("/endpoints/{id}/dockerhub",
h.Handle("/endpoints/{id}/dockerhub/{registryId}",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/extensions",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)
@ -63,5 +65,11 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointSnapshot))).Methods(http.MethodPost)
h.Handle("/endpoints/{id}/status",
bouncer.PublicAccess(httperror.LoggerHandler(h.endpointStatusInspect))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/registries",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistriesList))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/registries/{registryId}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryInspect))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/registries/{registryId}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.endpointRegistryAccess))).Methods(http.MethodPut)
return h
}