diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 031482b8d..2b7afe20c 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -719,6 +719,23 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { log.Fatal().Msg("failed to fetch SSL settings from DB") } + // FIXME: In 2.16 we changed the way ingress controller permissions are + // stored. Instead of being stored as annotation on an ingress rule, we keep + // them in our database. However, in order to run the migration we need an + // admin kube client to run lookup the old ingress rules and compare them + // with the current existing ingress classes. + // + // Unfortunately, our migrations run as part of the database initialization + // and our kubeclients require an initialized database. So it is not + // possible to do this migration as part of our normal flow. We DO have a + // migration which toggles a boolean in kubernetes configuration that + // indicated that this "post init" migration should be run. If/when this is + // resolved we can remove this function. + err = kubernetesClientFactory.PostInitMigrateIngresses() + if err != nil { + log.Fatal().Err(err).Msg("failure during creation of new database") + } + return &http.Server{ AuthorizationService: authorizationService, ReverseTunnelService: reverseTunnelService, diff --git a/api/datastore/migrator/migrate_dbversion70.go b/api/datastore/migrator/migrate_dbversion70.go index 5aadff56c..fc8e3febb 100644 --- a/api/datastore/migrator/migrate_dbversion70.go +++ b/api/datastore/migrator/migrate_dbversion70.go @@ -60,6 +60,7 @@ func (m *Migrator) updateIngressFieldsForEnvDB70() error { for _, endpoint := range endpoints { endpoint.Kubernetes.Configuration.IngressAvailabilityPerNamespace = true endpoint.Kubernetes.Configuration.AllowNoneIngressClass = false + endpoint.PostInitMigrations.MigrateIngresses = true err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint) if err != nil { diff --git a/api/datastore/test_data/output_24_to_latest.json b/api/datastore/test_data/output_24_to_latest.json index d21359eb6..a751c0889 100644 --- a/api/datastore/test_data/output_24_to_latest.json +++ b/api/datastore/test_data/output_24_to_latest.json @@ -66,6 +66,9 @@ }, "LastCheckInDate": 0, "Name": "local", + "PostInitMigrations": { + "MigrateIngresses": true + }, "PublicURL": "", "QueryDate": 0, "SecuritySettings": { diff --git a/api/http/handler/kubernetes/ingresses.go b/api/http/handler/kubernetes/ingresses.go index 8af98df90..292295cf3 100644 --- a/api/http/handler/kubernetes/ingresses.go +++ b/api/http/handler/kubernetes/ingresses.go @@ -279,6 +279,7 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter err, ) } + // Add none controller if "AllowNone" is set for endpoint. if endpoint.Kubernetes.Configuration.AllowNoneIngressClass { controllers = append(controllers, models.K8sIngressController{ @@ -287,6 +288,7 @@ func (handler *Handler) updateKubernetesIngressControllers(w http.ResponseWriter Type: "custom", }) } + var updatedClasses []portainer.KubernetesIngressClassConfig for i := range controllers { controllers[i].Availability = true diff --git a/api/kubernetes/cli/client.go b/api/kubernetes/cli/client.go index 821a17a0e..d15d486bb 100644 --- a/api/kubernetes/cli/client.go +++ b/api/kubernetes/cli/client.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/dataservices" + "github.com/rs/zerolog/log" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -219,3 +220,127 @@ func buildLocalClient() (*kubernetes.Clientset, error) { return kubernetes.NewForConfig(config) } + +func (factory *ClientFactory) PostInitMigrateIngresses() error { + endpoints, err := factory.dataStore.Endpoint().Endpoints() + if err != nil { + return err + } + for i := range endpoints { + // Early exit if we do not need to migrate! + if endpoints[i].PostInitMigrations.MigrateIngresses == false { + return nil + } + + err := factory.migrateEndpointIngresses(&endpoints[i]) + if err != nil { + log.Debug().Err(err).Msg("failure migrating endpoint ingresses") + } + } + + return nil +} + +func (factory *ClientFactory) migrateEndpointIngresses(e *portainer.Endpoint) error { + // classes is a list of controllers which have been manually added to the + // cluster setup view. These need to all be allowed globally, but then + // blocked in specific namespaces which they were not previously allowed in. + classes := e.Kubernetes.Configuration.IngressClasses + + // We need a kube client to gather namespace level permissions. In pre-2.16 + // versions of portainer, the namespace level permissions were stored by + // creating an actual ingress rule in the cluster with a particular + // annotation indicating that it's name (the class name) should be allowed. + cli, err := factory.GetKubeClient(e) + if err != nil { + return err + } + + detected, err := cli.GetIngressControllers() + if err != nil { + return err + } + + // newControllers is a set of all currently detected controllers. + newControllers := make(map[string]struct{}) + for _, controller := range detected { + newControllers[controller.ClassName] = struct{}{} + } + + namespaces, err := cli.GetNamespaces() + if err != nil { + return err + } + + // Set of namespaces, if any, in which "allow none" should be true. + allow := make(map[string]map[string]struct{}) + for _, c := range classes { + allow[c.Name] = make(map[string]struct{}) + } + allow["none"] = make(map[string]struct{}) + + for namespace := range namespaces { + // Compare old annotations with currently detected controllers. + ingresses, err := cli.GetIngresses(namespace) + if err != nil { + return fmt.Errorf("failure getting ingresses during migration") + } + for _, ingress := range ingresses { + oldController, ok := ingress.Annotations["ingress.portainer.io/ingress-type"] + if !ok { + // Skip rules without our old annotation. + continue + } + + if _, ok := newControllers[oldController]; ok { + // Skip rules which match a detected controller. + // TODO: Allow this particular controller. + allow[oldController][ingress.Namespace] = struct{}{} + continue + } + + allow["none"][ingress.Namespace] = struct{}{} + } + } + + // Locally, disable "allow none" for namespaces not inside shouldAllowNone. + var newClasses []portainer.KubernetesIngressClassConfig + for _, c := range classes { + var blocked []string + for namespace := range namespaces { + if _, ok := allow[c.Name][namespace]; ok { + continue + } + blocked = append(blocked, namespace) + } + + newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{ + Name: c.Name, + Type: c.Type, + GloballyBlocked: false, + BlockedNamespaces: blocked, + }) + } + + // Handle "none". + if len(allow["none"]) != 0 { + e.Kubernetes.Configuration.AllowNoneIngressClass = true + var disallowNone []string + for namespace := range namespaces { + if _, ok := allow["none"][namespace]; ok { + continue + } + disallowNone = append(disallowNone, namespace) + } + newClasses = append(newClasses, portainer.KubernetesIngressClassConfig{ + Name: "none", + Type: "custom", + GloballyBlocked: false, + BlockedNamespaces: disallowNone, + }) + } + + e.Kubernetes.Configuration.IngressClasses = newClasses + e.PostInitMigrations.MigrateIngresses = false + return factory.dataStore.Endpoint().UpdateEndpoint(e.ID, e) +} diff --git a/api/portainer.go b/api/portainer.go index 3afa7322a..dcd6a3717 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -352,6 +352,9 @@ type ( // Whether the device has been trusted or not by the user UserTrusted bool + // Whether we need to run any "post init migrations". + PostInitMigrations EndpointPostInitMigrations `json:"PostInitMigrations"` + Edge struct { // Whether the device has been started in edge async mode AsyncMode bool @@ -453,6 +456,11 @@ type ( EdgeStacks map[EdgeStackID]bool } + // EndpointPostInitMigrations + EndpointPostInitMigrations struct { + MigrateIngresses bool `json:"MigrateIngresses"` + } + // Extension represents a deprecated Portainer extension Extension struct { // Extension Identifier