fix(container/network): recreate container changes static IP [EE-5448] (#8960)

Co-authored-by: Chaim Lev-Ari <chaim.levi-ari@portainer.io>

api/docker/client/client.go

@@ -1,4 +1,4 @@
-package docker
+package client
 
 import (
 	"errors"

api/docker/consts/labels.go

@@ -0,0 +1,8 @@
+package consts
+
+const (
+	ComposeStackNameLabel = "com.docker.compose.project"
+	SwarmStackNameLabel   = "com.docker.stack.namespace"
+	SwarmServiceIdLabel   = "com.docker.swarm.service.id"
+	SwarmNodeIdLabel      = "com.docker.swarm.node.id"
+)

api/docker/container.go

@@ -0,0 +1,232 @@
+package docker
+
+import (
+	"context"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	dockercontainer "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/docker/images"
+	"github.com/rs/zerolog/log"
+)
+
+type ContainerService struct {
+	factory   *dockerclient.ClientFactory
+	dataStore dataservices.DataStore
+	sr        *serviceRestore
+}
+
+func NewContainerService(factory *dockerclient.ClientFactory, dataStore dataservices.DataStore) *ContainerService {
+	return &ContainerService{
+		factory:   factory,
+		dataStore: dataStore,
+		sr:        &serviceRestore{},
+	}
+}
+
+// Recreate a container
+func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerId string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error) {
+	cli, err := c.factory.CreateClient(endpoint, nodeName, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "create client error")
+	}
+
+	defer func(cli *client.Client) {
+		cli.Close()
+	}(cli)
+
+	log.Debug().Str("container_id", containerId).Msg("starting to fetch container information")
+
+	container, _, err := cli.ContainerInspectWithRaw(ctx, containerId, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch container information error")
+	}
+
+	log.Debug().Str("image", container.Config.Image).Msg("starting to parse image")
+	img, err := images.ParseImage(images.ParseImageOptions{
+		Name: container.Config.Image,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "parse image error")
+	}
+
+	if imageTag != "" {
+		err = img.WithTag(imageTag)
+		if err != nil {
+			return nil, errors.Wrapf(err, "set image tag error %s", imageTag)
+		}
+
+		log.Debug().Str("image", container.Config.Image).Msg("new image with tag")
+
+		container.Config.Image = img.FullName()
+	}
+
+	// 1. pull image if you need force pull
+	if forcePullImage {
+		puller := images.NewPuller(cli, images.NewRegistryClient(c.dataStore), c.dataStore)
+		err = puller.Pull(ctx, img)
+		if err != nil {
+			return nil, errors.Wrapf(err, "pull image error %s", img.FullName())
+		}
+	}
+
+	// 2. stop the current container
+	log.Debug().Str("container_id", containerId).Msg("starting to stop the container")
+	err = cli.ContainerStop(ctx, containerId, dockercontainer.StopOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "stop container error")
+	}
+
+	// 3. rename the current container
+	log.Debug().Str("container_id", containerId).Msg("starting to rename the container")
+	err = cli.ContainerRename(ctx, containerId, container.Name+"-old")
+	if err != nil {
+		return nil, errors.Wrap(err, "rename container error")
+	}
+
+	networkWithCreation := network.NetworkingConfig{
+		EndpointsConfig: make(map[string]*network.EndpointSettings),
+	}
+
+	// 4. disconnect all networks from the current container
+	for name, network := range container.NetworkSettings.Networks {
+		// This allows new container to use the same IP address if specified
+		err = cli.NetworkDisconnect(ctx, network.NetworkID, containerId, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "disconnect network from old container error")
+		}
+
+		// 5. get the first network attached to the current container
+		if len(networkWithCreation.EndpointsConfig) == 0 {
+			// Retrieve the first network that is linked to the present container, which
+			// will be utilized when creating the container.
+			networkWithCreation.EndpointsConfig[name] = network
+		}
+	}
+	c.sr.enable()
+	defer c.sr.close()
+	defer c.sr.restore()
+
+	c.sr.push(func() {
+		log.Debug().Str("container_id", containerId).Str("container", container.Name).Msg("restoring the container")
+		cli.ContainerRename(ctx, containerId, container.Name)
+		for _, network := range container.NetworkSettings.Networks {
+			cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
+		}
+		cli.ContainerStart(ctx, containerId, types.ContainerStartOptions{})
+	})
+
+	log.Debug().Str("container", strings.Split(container.Name, "/")[1]).Msg("starting to create a new container")
+
+	// 6. create a new container
+	// when a container is created without a network, docker connected it by default to the
+	// bridge network with a random IP, also it can only connect to one network on creation.
+	// to retain the same network settings we have to connect on creation to one of the old
+	// container's networks, and connect to the other networks after creation.
+	// see: https://portainer.atlassian.net/browse/EE-5448
+	create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &networkWithCreation, nil, container.Name)
+
+	c.sr.push(func() {
+		log.Debug().Str("container_id", create.ID).Msg("removing the new container")
+		cli.ContainerStop(ctx, create.ID, dockercontainer.StopOptions{})
+		cli.ContainerRemove(ctx, create.ID, types.ContainerRemoveOptions{})
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "create container error")
+	}
+
+	newContainerId := create.ID
+
+	// 7. connect to networks
+	// docker can connect to only one network at creation, so we need to connect to networks after creation
+	// see https://github.com/moby/moby/issues/17750
+	log.Debug().Str("container_id", newContainerId).Msg("connecting networks to container")
+	networks := container.NetworkSettings.Networks
+	for key, network := range networks {
+		_, ok := networkWithCreation.EndpointsConfig[key]
+		if ok {
+			// skip the network that is used during container creation
+			continue
+		}
+
+		err = cli.NetworkConnect(ctx, network.NetworkID, newContainerId, network)
+		if err != nil {
+			return nil, errors.Wrap(err, "connect container network error")
+		}
+	}
+
+	// 8. start the new container
+	log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
+	err = cli.ContainerStart(ctx, newContainerId, types.ContainerStartOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "start container error")
+	}
+
+	// 9. delete the old container
+	log.Debug().Str("container_id", containerId).Msg("starting to remove the old container")
+	_ = cli.ContainerRemove(ctx, containerId, types.ContainerRemoveOptions{})
+
+	c.sr.disable()
+
+	newContainer, _, err := cli.ContainerInspectWithRaw(ctx, newContainerId, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch container information error")
+	}
+
+	return &newContainer, nil
+}
+
+type serviceRestore struct {
+	restoreC chan struct{}
+	fs       []func()
+}
+
+func (sr *serviceRestore) enable() {
+	sr.restoreC = make(chan struct{}, 1)
+	sr.fs = make([]func(), 0)
+	sr.restoreC <- struct{}{}
+}
+
+func (sr *serviceRestore) disable() {
+	select {
+	case <-sr.restoreC:
+	default:
+	}
+}
+
+func (sr *serviceRestore) push(f func()) {
+	sr.fs = append(sr.fs, f)
+}
+
+func (sr *serviceRestore) restore() {
+	select {
+	case <-sr.restoreC:
+		l := len(sr.fs)
+		if l > 0 {
+			for i := l - 1; i >= 0; i-- {
+				sr.fs[i]()
+			}
+		}
+	default:
+	}
+}
+
+func (sr *serviceRestore) close() {
+	if sr == nil || sr.restoreC == nil {
+		return
+	}
+
+	select {
+	case <-sr.restoreC:
+	default:
+	}
+
+	close(sr.restoreC)
+}

api/docker/images/digest.go

@@ -0,0 +1,184 @@
+package images
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+
+	"github.com/containers/image/v5/docker"
+	imagetypes "github.com/containers/image/v5/types"
+	"github.com/docker/docker/api/types"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+// Options holds docker registry object options
+type Options struct {
+	Auth    imagetypes.DockerAuthConfig
+	Timeout time.Duration
+}
+
+type DigestClient struct {
+	clientFactory  *dockerclient.ClientFactory
+	opts           Options
+	sysCtx         *imagetypes.SystemContext
+	registryClient *RegistryClient
+}
+
+func NewClientWithRegistry(registryClient *RegistryClient, clientFactory *dockerclient.ClientFactory) *DigestClient {
+	return &DigestClient{
+		clientFactory:  clientFactory,
+		registryClient: registryClient,
+	}
+}
+
+func (c *DigestClient) RemoteDigest(image Image) (digest.Digest, error) {
+	ctx, cancel := c.timeoutContext()
+	defer cancel()
+	// Docker references with both a tag and digest are currently not supported
+	if image.Tag != "" && image.Digest != "" {
+		err := image.trimDigest()
+		if err != nil {
+			return "", err
+		}
+	}
+
+	rmRef, err := ParseReference(image.String())
+	if err != nil {
+		return "", errors.Wrap(err, "Cannot parse the image reference")
+	}
+
+	sysCtx := c.sysCtx
+	if c.registryClient != nil {
+		username, password, err := c.registryClient.RegistryAuth(image)
+		if err != nil {
+			log.Info().Str("image up to date indicator", image.String()).Msg("No environment registry credentials found, using anonymous access")
+		} else {
+			sysCtx = &imagetypes.SystemContext{
+				DockerAuthConfig: &imagetypes.DockerAuthConfig{
+					Username: username,
+					Password: password,
+				},
+			}
+		}
+	}
+
+	// Retrieve remote digest through HEAD request
+	rmDigest, err := docker.GetDigest(ctx, sysCtx, rmRef)
+	if err != nil {
+		// fallback to public registry for hub
+		if image.HubLink != "" {
+			rmDigest, err = docker.GetDigest(ctx, c.sysCtx, rmRef)
+			if err == nil {
+				return rmDigest, nil
+			}
+		}
+
+		log.Debug().Err(err).Msg("get remote digest error")
+
+		return "", errors.Wrap(err, "Cannot get image digest from HEAD request")
+	}
+
+	return rmDigest, nil
+}
+
+func ParseLocalImage(inspect types.ImageInspect) (*Image, error) {
+	if IsLocalImage(inspect) || IsDanglingImage(inspect) {
+		return nil, errors.New("the image is not regular")
+	}
+
+	fromRepoDigests, err := ParseImage(ParseImageOptions{
+		// including image name but no tag
+		Name: inspect.RepoDigests[0],
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if IsNoTagImage(inspect) {
+		return &fromRepoDigests, nil
+	}
+
+	fromRepoTags, err := ParseImage(ParseImageOptions{
+		Name: inspect.RepoTags[0],
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	fromRepoDigests.Tag = fromRepoTags.Tag
+
+	return &fromRepoDigests, nil
+}
+
+func ParseRepoDigests(repoDigests []string) []digest.Digest {
+	digests := make([]digest.Digest, 0)
+	for _, repoDigest := range repoDigests {
+		d := ParseRepoDigest(repoDigest)
+		if d == "" {
+			continue
+		}
+
+		digests = append(digests, d)
+	}
+
+	return digests
+}
+
+func ParseRepoTags(repoTags []string) []*Image {
+	images := make([]*Image, 0)
+	for _, repoTag := range repoTags {
+		image := ParseRepoTag(repoTag)
+		if image != nil {
+			images = append(images, image)
+		}
+	}
+
+	return images
+}
+
+func ParseRepoDigest(repoDigest string) digest.Digest {
+	if !strings.ContainsAny(repoDigest, "@") {
+		return ""
+	}
+
+	d, err := digest.Parse(strings.Split(repoDigest, "@")[1])
+	if err != nil {
+		log.Warn().Msgf("Skip invalid repo digest item: %s [error: %v]", repoDigest, err)
+
+		return ""
+	}
+
+	return d
+}
+
+func ParseRepoTag(repoTag string) *Image {
+	if repoTag == "" {
+		return nil
+	}
+
+	image, err := ParseImage(ParseImageOptions{
+		Name: repoTag,
+	})
+	if err != nil {
+		log.Warn().Err(err).Str("repoTag", repoTag).Msg("RepoTag cannot be parsed.")
+
+		return nil
+	}
+
+	return &image
+}
+
+func (c *DigestClient) timeoutContext() (context.Context, context.CancelFunc) {
+	ctx := context.Background()
+	var cancel context.CancelFunc = func() {}
+
+	if c.opts.Timeout > 0 {
+		ctx, cancel = context.WithTimeout(ctx, c.opts.Timeout)
+	}
+
+	return ctx, cancel
+}

api/docker/images/image.go

@@ -0,0 +1,182 @@
+package images
+
+import (
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/docker/docker/api/types"
+
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+)
+
+type ImageID string
+
+// Image holds information about an image.
+type Image struct {
+	// Domain is the registry host of this image
+	Domain string
+	// Path may include username like portainer/portainer-ee, no Tag or Digest
+	Path    string
+	Tag     string
+	Digest  digest.Digest
+	HubLink string
+	named   reference.Named
+	opts    ParseImageOptions
+}
+
+// ParseImageOptions holds image options for parsing.
+type ParseImageOptions struct {
+	Name   string
+	HubTpl string
+}
+
+// Name returns the full name representation of an image but no Tag or Digest.
+func (i *Image) Name() string {
+	return i.named.Name()
+}
+
+// FullName return the real full name may include Tag or Digest of the image, Tag first.
+func (i *Image) FullName() string {
+	if i.Tag == "" {
+		return fmt.Sprintf("%s@%s", i.Name(), i.Digest)
+	}
+	return fmt.Sprintf("%s:%s", i.Name(), i.Tag)
+}
+
+// String returns the string representation of an image, including Tag and Digest if existed.
+func (i *Image) String() string {
+	return i.named.String()
+}
+
+// Reference returns either the digest if it is non-empty or the tag for the image.
+func (i *Image) Reference() string {
+	if len(i.Digest.String()) > 1 {
+		return i.Digest.String()
+	}
+
+	return i.Tag
+}
+
+// WithDigest sets the digest for an image.
+func (i *Image) WithDigest(digest digest.Digest) (err error) {
+	i.Digest = digest
+	i.named, err = reference.WithDigest(i.named, digest)
+	return err
+}
+
+func (i *Image) WithTag(tag string) (err error) {
+	i.Tag = tag
+	i.named, err = reference.WithTag(i.named, tag)
+	return err
+}
+
+func (i *Image) trimDigest() error {
+	i.Digest = ""
+	named, err := ParseImage(ParseImageOptions{Name: i.FullName()})
+	if err != nil {
+		return err
+	}
+	i.named = &named
+	return nil
+}
+
+// ParseImage returns an Image struct with all the values filled in for a given image.
+func ParseImage(parseOpts ParseImageOptions) (Image, error) {
+	// Parse the image name and tag.
+	named, err := reference.ParseNormalizedNamed(parseOpts.Name)
+	if err != nil {
+		return Image{}, errors.Wrapf(err, "parsing image %s failed", parseOpts.Name)
+	}
+	// Add the latest lag if they did not provide one.
+	named = reference.TagNameOnly(named)
+
+	i := Image{
+		opts:   parseOpts,
+		named:  named,
+		Domain: reference.Domain(named),
+		Path:   reference.Path(named),
+	}
+
+	// Hub link
+	i.HubLink, err = i.hubLink()
+	if err != nil {
+		return Image{}, errors.Wrap(err, fmt.Sprintf("resolving hub link for image %s failed", parseOpts.Name))
+	}
+
+	// Add the tag if there was one.
+	if tagged, ok := named.(reference.Tagged); ok {
+		i.Tag = tagged.Tag()
+	}
+
+	// Add the digest if there was one.
+	if canonical, ok := named.(reference.Canonical); ok {
+		i.Digest = canonical.Digest()
+	}
+
+	return i, nil
+}
+
+func (i *Image) hubLink() (string, error) {
+	if i.opts.HubTpl != "" {
+		var out bytes.Buffer
+		tmpl, err := template.New("tmpl").
+			Option("missingkey=error").
+			Parse(i.opts.HubTpl)
+		if err != nil {
+			return "", err
+		}
+		err = tmpl.Execute(&out, i)
+		return out.String(), err
+	}
+
+	switch i.Domain {
+	case "docker.io":
+		prefix := "r"
+		path := i.Path
+		if strings.HasPrefix(i.Path, "library/") {
+			prefix = "_"
+			path = strings.Replace(i.Path, "library/", "", 1)
+		}
+		return fmt.Sprintf("https://hub.docker.com/%s/%s", prefix, path), nil
+	case "docker.bintray.io", "jfrog-docker-reg2.bintray.io":
+		return fmt.Sprintf("https://bintray.com/jfrog/reg2/%s", strings.ReplaceAll(i.Path, "/", "%3A")), nil
+	case "docker.pkg.github.com":
+		return fmt.Sprintf("https://github.com/%s/packages", filepath.ToSlash(filepath.Dir(i.Path))), nil
+	case "gcr.io":
+		return fmt.Sprintf("https://%s/%s", i.Domain, i.Path), nil
+	case "ghcr.io":
+		ref := strings.Split(i.Path, "/")
+		ghUser, ghPackage := ref[0], ref[1]
+		return fmt.Sprintf("https://github.com/users/%s/packages/container/package/%s", ghUser, ghPackage), nil
+	case "quay.io":
+		return fmt.Sprintf("https://quay.io/repository/%s", i.Path), nil
+	case "registry.access.redhat.com":
+		return fmt.Sprintf("https://access.redhat.com/containers/#/registry.access.redhat.com/%s", i.Path), nil
+	case "registry.gitlab.com":
+		return fmt.Sprintf("https://gitlab.com/%s/container_registry", i.Path), nil
+	default:
+		return "", nil
+	}
+}
+
+// IsLocalImage checks if the image has been built locally
+func IsLocalImage(image types.ImageInspect) bool {
+	return len(image.RepoDigests) == 0
+}
+
+// IsDanglingImage returns whether the given image is "dangling" which means
+// that there are no repository references to the given image and it has no
+// child images
+func IsDanglingImage(image types.ImageInspect) bool {
+	return len(image.RepoTags) == 1 && image.RepoTags[0] == "<none>:<none>" && len(image.RepoDigests) == 1 && image.RepoDigests[0] == "<none>@<none>"
+}
+
+// IsNoTagImage returns whether the given image is damaged, has no tags
+func IsNoTagImage(image types.ImageInspect) bool {
+	return len(image.RepoTags) == 0
+}

api/docker/images/image_test.go

@@ -0,0 +1,119 @@
+package images
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestImageParser(t *testing.T) {
+	is := assert.New(t)
+
+	// portainer/portainer-ee
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "portainer/portainer-ee",
+		})
+		is.NoError(err, "")
+		is.Equal("docker.io/portainer/portainer-ee:latest", image.FullName())
+		is.Equal("portainer/portainer-ee", image.opts.Name)
+		is.Equal("latest", image.Tag)
+		is.Equal("portainer/portainer-ee", image.Path)
+		is.Equal("docker.io", image.Domain)
+		is.Equal("docker.io/portainer/portainer-ee", image.Name())
+		is.Equal("latest", image.Reference())
+		is.Equal("docker.io/portainer/portainer-ee:latest", image.String())
+
+	})
+	// gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+}
+
+func TestUpdateParsedImage(t *testing.T) {
+	is := assert.New(t)
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.WithTag("v0.0.31")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.31", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.31", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.31@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.WithDigest("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3")
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b3", image.String())
+	})
+
+	// gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2
+	t.Run("", func(t *testing.T) {
+		image, err := ParseImage(ParseImageOptions{
+			Name: "gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2",
+		})
+		is.NoError(err, "")
+		_ = image.trimDigest()
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.FullName())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30@sha256:02c921df998f95e849058af14de7045efc3954d90320967418a0d1f182bbc0b2", image.opts.Name)
+		is.Equal("v0.0.30", image.Tag)
+		is.Equal("k8s-minikube/kicbase", image.Path)
+		is.Equal("gcr.io", image.Domain)
+		is.Equal("https://gcr.io/k8s-minikube/kicbase", image.HubLink)
+		is.Equal("gcr.io/k8s-minikube/kicbase", image.Name())
+		is.Equal("v0.0.30", image.Reference())
+		is.Equal("gcr.io/k8s-minikube/kicbase:v0.0.30", image.String())
+	})
+}

api/docker/images/puller.go

@@ -0,0 +1,50 @@
+package images
+
+import (
+	"context"
+	"io"
+
+	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/rs/zerolog/log"
+)
+
+type Puller struct {
+	client         *client.Client
+	registryClient *RegistryClient
+	dataStore      dataservices.DataStore
+}
+
+func NewPuller(client *client.Client, registryClient *RegistryClient, dataStore dataservices.DataStore) *Puller {
+	return &Puller{
+		client:         client,
+		registryClient: registryClient,
+		dataStore:      dataStore,
+	}
+}
+
+func (puller *Puller) Pull(ctx context.Context, image Image) error {
+	log.Debug().Str("image", image.FullName()).Msg("starting to pull the image")
+
+	registryAuth, err := puller.registryClient.EncodedRegistryAuth(image)
+	if err != nil {
+		log.Debug().
+			Str("image", image.FullName()).
+			Err(err).
+			Msg("failed to get an encoded registry auth via image, try to pull image without registry auth")
+	}
+
+	out, err := puller.client.ImagePull(ctx, image.FullName(), types.ImagePullOptions{
+		RegistryAuth: registryAuth,
+	})
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.ReadAll(out)
+
+	return err
+}

api/docker/images/ref.go

@@ -0,0 +1,16 @@
+package images
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/types"
+)
+
+func ParseReference(imageStr string) (types.ImageReference, error) {
+	if !strings.HasPrefix(imageStr, "//") {
+		imageStr = fmt.Sprintf("//%s", imageStr)
+	}
+	return docker.ParseReference(imageStr)
+}

api/docker/images/registry.go

@@ -0,0 +1,147 @@
+package images
+
+import (
+	"strings"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/internal/registryutils"
+)
+
+var (
+	_registriesCache = cache.New(5*time.Minute, 5*time.Minute)
+)
+
+type (
+	RegistryClient struct {
+		dataStore dataservices.DataStore
+	}
+)
+
+func NewRegistryClient(dataStore dataservices.DataStore) *RegistryClient {
+	return &RegistryClient{dataStore: dataStore}
+}
+
+func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
+	registries, err := c.dataStore.Registry().Registries()
+	if err != nil {
+		return "", "", err
+	}
+
+	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	if err != nil {
+		return "", "", err
+	}
+
+	if !registry.Authentication {
+		return "", "", errors.New("authentication is disabled")
+	}
+
+	return c.CertainRegistryAuth(registry)
+}
+
+func (c *RegistryClient) CertainRegistryAuth(registry *portainer.Registry) (string, string, error) {
+	err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
+	if err != nil {
+		return "", "", err
+	}
+
+	if !registry.Authentication {
+		return "", "", errors.New("authentication is disabled")
+	}
+
+	return registryutils.GetRegEffectiveCredential(registry)
+}
+
+func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
+	registries, err := c.dataStore.Registry().Registries()
+	if err != nil {
+		return "", err
+	}
+
+	registry, err := findBestMatchRegistry(image.opts.Name, registries)
+	if err != nil {
+		return "", err
+	}
+
+	if !registry.Authentication {
+		return "", errors.New("authentication is disabled")
+	}
+
+	return c.EncodedCertainRegistryAuth(registry)
+}
+
+func (c *RegistryClient) EncodedCertainRegistryAuth(registry *portainer.Registry) (string, error) {
+	err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
+	if err != nil {
+		return "", err
+	}
+
+	return registryutils.GetRegistryAuthHeader(registry)
+}
+
+// findBestMatchRegistry finds out the best match registry for repository @Meng
+// matching precedence:
+// 1. both domain name and username matched (for dockerhub only)
+// 2. only URL matched
+// 3. pick up the first dockerhub registry
+func findBestMatchRegistry(repository string, registries []portainer.Registry) (*portainer.Registry, error) {
+	cachedRegistry, err := cachedRegistry(repository)
+	if err == nil {
+		return cachedRegistry, nil
+	}
+
+	var match1, match2, match3 *portainer.Registry
+	for i := 0; i < len(registries); i++ {
+		registry := registries[i]
+		if registry.Type == portainer.DockerHubRegistry {
+
+			// try to match repository examples:
+			//   <USERNAME>/nginx:latest
+			//   docker.io/<USERNAME>/nginx:latest
+			if strings.HasPrefix(repository, registry.Username+"/") || strings.HasPrefix(repository, registry.URL+"/"+registry.Username+"/") {
+				match1 = &registry
+			}
+
+			// try to match repository examples:
+			//   portainer/portainer-ee:latest
+			//   <NON-USERNAME>/portainer-ee:latest
+			if match3 == nil {
+				match3 = &registry
+			}
+		}
+
+		if strings.Contains(repository, registry.URL) {
+			match2 = &registry
+		}
+	}
+
+	match := match1
+	if match == nil {
+		match = match2
+	}
+	if match == nil {
+		match = match3
+	}
+
+	if match == nil {
+		return nil, errors.New("no registries matched")
+	}
+	_registriesCache.Set(repository, match, 0)
+	return match, nil
+}
+
+func cachedRegistry(cacheKey string) (*portainer.Registry, error) {
+	r, ok := _registriesCache.Get(cacheKey)
+	if ok {
+		registry, ok := r.(portainer.Registry)
+		if ok {
+			return &registry, nil
+		}
+	}
+
+	return nil, errors.Errorf("no registry found in cache: %s", cacheKey)
+}

api/docker/images/registry_test.go

@@ -0,0 +1,69 @@
+package images
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindBestMatchNeedAuthRegistry(t *testing.T) {
+	is := assert.New(t)
+
+	t.Run("", func(t *testing.T) {
+		image := "USERNAME/nginx:latest"
+		registries := []portainer.Registry{createNewRegistry("docker.io", "USERNAME", false),
+			createNewRegistry("hub-mirror.c.163.com", "", false)}
+		r, err := findBestMatchRegistry(image, registries)
+		is.NoError(err, "")
+		is.NotNil(r, "")
+		is.False(r.Authentication, "")
+		is.Equal("docker.io", r.URL)
+	})
+
+	t.Run("", func(t *testing.T) {
+		image := "USERNAME/nginx:latest"
+		registries := []portainer.Registry{createNewRegistry("docker.io", "", false),
+			createNewRegistry("hub-mirror.c.163.com", "USERNAME", false)}
+		r, err := findBestMatchRegistry(image, registries)
+		is.NoError(err, "")
+		is.NotNil(r, "")
+		is.False(r.Authentication, "")
+		is.Equal("docker.io", r.URL)
+	})
+
+	t.Run("", func(t *testing.T) {
+		image := "docker.io/<USERNAME>/nginx:latest"
+		registries := []portainer.Registry{createNewRegistry("docker.io", "USERNAME", true),
+			createNewRegistry("hub-mirror.c.163.com", "", false)}
+		r, err := findBestMatchRegistry(image, registries)
+		is.NoError(err, "")
+		is.NotNil(r, "")
+		is.True(r.Authentication, "")
+		is.Equal("docker.io", r.URL)
+	})
+
+	t.Run("", func(t *testing.T) {
+		image := "portainer/portainer-ee:latest"
+		registries := []portainer.Registry{createNewRegistry("docker.io", "", true)}
+		r, err := findBestMatchRegistry(image, registries)
+		is.NoError(err, "")
+		is.NotNil(r, "")
+		is.True(r.Authentication, "")
+		is.Equal("docker.io", r.URL)
+	})
+}
+
+func createNewRegistry(domain, username string, auth bool) portainer.Registry {
+	registry := portainer.Registry{
+		URL:            domain,
+		Authentication: auth,
+		Username:       username,
+	}
+
+	if domain == "docker.io" {
+		registry.Type = portainer.DockerHubRegistry
+	}
+
+	return registry
+}

api/docker/images/status.go

@@ -0,0 +1,302 @@
+package images
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/opencontainers/go-digest"
+	"github.com/patrickmn/go-cache"
+	portainer "github.com/portainer/portainer/api"
+	consts "github.com/portainer/portainer/api/docker/consts"
+	"github.com/portainer/portainer/api/internal/slices"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
+
+// Status constants
+const (
+	Processing = Status("processing")
+	Outdated   = Status("outdated")
+	Updated    = Status("updated")
+	Skipped    = Status("skipped")
+	Preparing  = Status("preparing")
+	Error      = Status("error")
+)
+
+var (
+	statusCache       = cache.New(24*time.Hour, 24*time.Hour)
+	remoteDigestCache = cache.New(5*time.Second, 5*time.Second)
+	swarmID2NameCache = cache.New(5*time.Second, 5*time.Second)
+)
+
+// Status holds Docker image  analysis
+type Status string
+
+func (c *DigestClient) ContainersImageStatus(ctx context.Context, containers []types.Container, endpoint *portainer.Endpoint) Status {
+	cli, err := c.clientFactory.CreateClient(endpoint, "", nil)
+	if err != nil {
+		log.Error().Err(err).Msg("cannot create docker client")
+		return Error
+	}
+
+	statuses := make([]Status, len(containers))
+	for i, ct := range containers {
+		var nodeName string
+		if swarmNodeId := ct.Labels[consts.SwarmNodeIdLabel]; swarmNodeId != "" {
+			if swarmNodeName, ok := swarmID2NameCache.Get(swarmNodeId); ok {
+				nodeName, _ = swarmNodeName.(string)
+			} else {
+				node, _, err := cli.NodeInspectWithRaw(ctx, ct.Labels[consts.SwarmNodeIdLabel])
+				if err != nil {
+					return Error
+				}
+
+				nodeName = node.Description.Hostname
+				swarmID2NameCache.Set(swarmNodeId, nodeName, 0)
+			}
+		}
+
+		s, err := c.ContainerImageStatus(ctx, ct.ID, endpoint, nodeName)
+		if err != nil {
+			statuses[i] = Error
+			log.Warn().Str("containerId", ct.ID).Err(err).Msg("error when fetching image status for container")
+
+			continue
+		}
+
+		statuses[i] = s
+
+		if s == Outdated || s == Processing {
+			break
+		}
+	}
+
+	return FigureOut(statuses)
+}
+
+func FigureOut(statuses []Status) Status {
+	if allMatch(statuses, Skipped) {
+		return Skipped
+	}
+
+	if allMatch(statuses, Preparing) {
+		return Preparing
+	}
+
+	if contains(statuses, Outdated) {
+		return Outdated
+	} else if contains(statuses, Processing) {
+		return Processing
+	} else if contains(statuses, Error) {
+		return Error
+	}
+
+	return Updated
+}
+
+func (c *DigestClient) ContainerImageStatus(ctx context.Context, containerID string, endpoint *portainer.Endpoint, nodeName string) (Status, error) {
+	cli, err := c.clientFactory.CreateClient(endpoint, nodeName, nil)
+	if err != nil {
+		log.Warn().Str("swarmNodeId", nodeName).Msg("Cannot create new docker client.")
+	}
+
+	container, err := cli.ContainerInspect(ctx, containerID)
+	if err != nil {
+		log.Warn().Err(err).Str("containerID", containerID).Msg("Inspect container error.")
+		return Skipped, nil
+	}
+
+	var imageID string
+	if strings.Contains(container.Image, "sha256") {
+		imageID = container.Image[strings.Index(container.Image, "sha256"):]
+	}
+
+	if imageID == "" {
+		return Skipped, nil
+	}
+
+	digs := make([]digest.Digest, 0)
+	images := make([]*Image, 0)
+	if i, err := ParseImage(ParseImageOptions{Name: container.Config.Image}); err == nil {
+		images = append(images, &i)
+	}
+
+	imageInspect, _, err := cli.ImageInspectWithRaw(ctx, imageID)
+	if err != nil {
+		log.Debug().Str("imageID", imageID).Msg("inspect failed")
+		return Error, err
+	}
+
+	if len(imageInspect.RepoDigests) > 0 {
+		digs = append(digs, ParseRepoDigests(imageInspect.RepoDigests)...)
+	}
+
+	if len(imageInspect.RepoTags) > 0 {
+		images = append(images, ParseRepoTags(imageInspect.RepoTags)...)
+	}
+
+	s, err := c.checkStatus(images, digs)
+	if err != nil {
+		log.Debug().Str("image", container.Image).Err(err).Msg("fetching a certain image status")
+		return Error, err
+	}
+
+	statusCache.Set(imageID, s, 0)
+
+	return s, err
+}
+
+func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string, endpoint *portainer.Endpoint) (Status, error) {
+	cli, err := c.clientFactory.CreateClient(endpoint, "", nil)
+	if err != nil {
+		return Error, nil
+	}
+
+	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
+		All:     true,
+		Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+serviceID)),
+	})
+	if err != nil {
+		log.Warn().Err(err).Str("serviceID", serviceID).Msg("cannot list container for the service")
+		return Error, err
+	}
+
+	nonExistedOrStoppedContainers := make([]types.Container, 0)
+	for _, container := range containers {
+		if container.State == "exited" || container.State == "stopped" {
+			continue
+		}
+
+		// When there is a container with the state "Created" under the service, it
+		// indicates that the Docker Swarm is replacing the existing task with
+		// a new task. At the moment, the state of the new task is "Created", and
+		// the state of the old task is "Running".
+		// Until the new task runs up, the image status should be set "Preparing"
+		if container.State == "created" {
+			return Preparing, nil
+		}
+		nonExistedOrStoppedContainers = append(nonExistedOrStoppedContainers, container)
+	}
+
+	if len(nonExistedOrStoppedContainers) == 0 {
+		return Preparing, nil
+	}
+
+	return c.ContainersImageStatus(ctx, nonExistedOrStoppedContainers, endpoint), nil
+}
+
+func (c *DigestClient) checkStatus(images []*Image, digests []digest.Digest) (Status, error) {
+	if digests == nil {
+		digests = make([]digest.Digest, 0)
+	}
+
+	for _, img := range images {
+		if img.Digest != "" && !slices.Contains(digests, img.Digest) {
+			log.Info().Str("localDigest", img.Domain).Msg("incoming local digest is not nil")
+			digests = append([]digest.Digest{img.Digest}, digests...)
+		}
+	}
+
+	if len(digests) == 0 {
+		return Skipped, nil
+	}
+
+	var imageStatus Status
+
+	for _, img := range images {
+		var remoteDigest digest.Digest
+		var err error
+		if rd, ok := remoteDigestCache.Get(img.FullName()); ok {
+			remoteDigest, _ = rd.(digest.Digest)
+		}
+		if remoteDigest == "" {
+			remoteDigest, err = c.RemoteDigest(*img)
+			if err != nil {
+				log.Error().Str("image", img.String()).Msg("error when fetch remote digest for image")
+				return Error, err
+			}
+		}
+		remoteDigestCache.Set(img.FullName(), remoteDigest, 0)
+
+		log.Debug().Str("image", img.FullName()).Stringer("remote_digest", remoteDigest).
+			Int("local_digest_size", len(digests)).
+			Msg("Digests")
+
+		// final locals vs remote one
+		for _, dig := range digests {
+			log.Debug().
+				Str("image", img.FullName()).
+				Stringer("remote_digest", remoteDigest).
+				Stringer("local_digest", dig).
+				Msg("Comparing")
+
+			if dig == remoteDigest {
+				log.Debug().Str("image", img.FullName()).
+					Stringer("remote_digest", remoteDigest).
+					Stringer("local_digest", dig).
+					Msg("Found a match")
+				return Updated, nil
+			}
+		}
+	}
+
+	imageStatus = Outdated
+
+	return imageStatus, nil
+}
+
+func CachedResourceImageStatus(resourceID string) (Status, error) {
+	if s, ok := statusCache.Get(resourceID); ok {
+		return s.(Status), nil
+	}
+
+	return "", errors.Errorf("no image found in cache: %s", resourceID)
+}
+
+func CacheResourceImageStatus(resourceID string, status Status) {
+	statusCache.Set(resourceID, status, 0)
+}
+
+func CachedImageDigest(resourceID string) (Status, error) {
+	if s, ok := statusCache.Get(resourceID); ok {
+		return s.(Status), nil
+	}
+
+	return "", errors.Errorf("no image found in cache: %s", resourceID)
+}
+
+func EvictImageStatus(resourceID string) {
+	statusCache.Delete(resourceID)
+}
+
+func contains(statuses []Status, status Status) bool {
+	if len(statuses) == 0 {
+		return false
+	}
+
+	for _, s := range statuses {
+		if s == status {
+			return true
+		}
+	}
+
+	return false
+}
+
+func allMatch(statuses []Status, status Status) bool {
+	if len(statuses) == 0 {
+		return false
+	}
+
+	for _, s := range statuses {
+		if s != status {
+			return false
+		}
+	}
+
+	return true
+}

api/docker/labels.go

@@ -1,6 +0,0 @@
-package docker
-
-const (
-	ComposeStackNameLabel = "com.docker.compose.project"
-	SwarmStackNameLabel   = "com.docker.stack.namespace"
-)

api/docker/snapshot.go

@@ -5,22 +5,23 @@ import (
 	"strings"
 	"time"
 
-	portainer "github.com/portainer/portainer/api"
-
 	"github.com/docker/docker/api/types"
 	_container "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
+	portainer "github.com/portainer/portainer/api"
+	dockerclient "github.com/portainer/portainer/api/docker/client"
+	"github.com/portainer/portainer/api/docker/consts"
 	"github.com/rs/zerolog/log"
 )
 
 // Snapshotter represents a service used to create environment(endpoint) snapshots
 type Snapshotter struct {
-	clientFactory *ClientFactory
+	clientFactory *dockerclient.ClientFactory
 }
 
 // NewSnapshotter returns a new Snapshotter instance
-func NewSnapshotter(clientFactory *ClientFactory) *Snapshotter {
+func NewSnapshotter(clientFactory *dockerclient.ClientFactory) *Snapshotter {
 	return &Snapshotter{
 		clientFactory: clientFactory,
 	}
@@ -202,7 +203,7 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
 		}
 
 		for k, v := range container.Labels {
-			if k == ComposeStackNameLabel {
+			if k == consts.ComposeStackNameLabel {
 				stacks[v] = struct{}{}
 			}
 		}