feat(docker): show docker pull rate limits (#4666)

* feat(dockerhub): introduce local status endpoint

* feat(proxy): rewrite request with dockerhub credentials

* feat(endpoint): check env type

* feat(endpoint): check for local endpoint

* feat(docker): introduce client side service to get limits

* feat(container): add info about rate limits in container

* feat(dockerhub): load rate limits just for specific endpoints

* feat(images): show specific dockerhub messages for admin

* feat(service-create): show docker rate limits

* feat(service-edit): show rate limit messages

* fix(images): fix loading of page

* refactor(images): move rate limits check to container

* feat(kubernetes): proxy agent requests

* feat(kubernetes/apps): show pull limits in application creation

* refactor(image-registry): move warning to end of field

* fix(image-registry): show right message for admin

* fix(images): silently fail when loading rate limits

* fix(kube/apps): use new rate limits comp

* fix(images): move rate warning to end

* fix(registry): move search to right place

* fix(service): remove service warning

* fix(endpoints): check if kube endpoint is local

api/http/handler/endpointproxy/proxy_kubernetes.go

@@ -3,11 +3,12 @@ package endpointproxy
 import (
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/request"
-	"github.com/portainer/portainer/api"
+	portainer "github.com/portainer/portainer/api"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 
 	"net/http"
@@ -66,9 +67,15 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h
 
 	requestPrefix := fmt.Sprintf("/%d/kubernetes", endpointID)
 	if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
-		requestPrefix = fmt.Sprintf("/%d", endpointID)
+		if isKubernetesRequest(strings.TrimPrefix(r.URL.String(), requestPrefix)) {
+			requestPrefix = fmt.Sprintf("/%d", endpointID)
+		}
 	}
 
 	http.StripPrefix(requestPrefix, proxy).ServeHTTP(w, r)
 	return nil
 }
+
+func isKubernetesRequest(requestURL string) bool {
+	return strings.HasPrefix(requestURL, "/api")
+}

api/http/handler/endpoints/endpoint_dockerhub_status.go

@@ -0,0 +1,140 @@
+package endpoints
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	httperror "github.com/portainer/libhttp/error"
+	"github.com/portainer/libhttp/request"
+	"github.com/portainer/libhttp/response"
+	portainer "github.com/portainer/portainer/api"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/http/client"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+)
+
+type dockerhubStatusResponse struct {
+	Remaining int `json:"remaining"`
+	Limit     int `json:"limit"`
+}
+
+// GET request on /api/endpoints/{id}/dockerhub/status
+func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
+	if err != nil {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
+	}
+
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
+	if err == bolterrors.ErrObjectNotFound {
+		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
+	} else if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
+	}
+
+	if !endpointutils.IsLocalEndpoint(endpoint) {
+		return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")}
+	}
+
+	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
+	}
+
+	httpClient := client.NewHTTPClient()
+	token, err := getDockerHubToken(httpClient, dockerhub)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err}
+	}
+
+	resp, err := getDockerHubLimits(httpClient, token)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub rate limits from DockerHub", err}
+	}
+
+	return response.JSON(w, resp)
+}
+
+func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) {
+	type dockerhubTokenResponse struct {
+		Token string `json:"token"`
+	}
+
+	requestURL := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull"
+
+	req, err := http.NewRequest(http.MethodGet, requestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	if dockerhub.Authentication {
+		req.SetBasicAuth(dockerhub.Username, dockerhub.Password)
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.New("failed fetching dockerhub token")
+	}
+
+	var data dockerhubTokenResponse
+	err = json.NewDecoder(resp.Body).Decode(&data)
+	if err != nil {
+		return "", err
+	}
+
+	return data.Token, nil
+}
+
+func getDockerHubLimits(httpClient *client.HTTPClient, token string) (*dockerhubStatusResponse, error) {
+
+	requestURL := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest"
+
+	req, err := http.NewRequest(http.MethodHead, requestURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New("failed fetching dockerhub limits")
+	}
+
+	rateLimit, err := parseRateLimitHeader(resp.Header, "RateLimit-Limit")
+	rateLimitRemaining, err := parseRateLimitHeader(resp.Header, "RateLimit-Remaining")
+
+	return &dockerhubStatusResponse{
+		Limit:     rateLimit,
+		Remaining: rateLimitRemaining,
+	}, nil
+}
+
+func parseRateLimitHeader(headers http.Header, headerKey string) (int, error) {
+	headerValue := headers.Get(headerKey)
+	if headerValue == "" {
+		return 0, fmt.Errorf("Missing %s header", headerKey)
+	}
+
+	matches := strings.Split(headerValue, ";")
+	value, err := strconv.Atoi(matches[0])
+	if err != nil {
+		return 0, err
+	}
+
+	return value, nil
+}

api/http/handler/endpoints/handler.go

@@ -51,6 +51,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
 	h.Handle("/endpoints/{id}",
 		bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
+	h.Handle("/endpoints/{id}/dockerhub",
+		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
 	h.Handle("/endpoints/{id}/extensions",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)
 	h.Handle("/endpoints/{id}/extensions/{extensionType}",

api/http/proxy/factory/docker/transport.go

@@ -1,9 +1,11 @@
 package docker
 
 import (
+	"bytes"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"path"
@@ -163,6 +165,21 @@ func (transport *Transport) proxyAgentRequest(r *http.Request) (*http.Response,
 
 		// volume browser request
 		return transport.restrictedResourceOperation(r, resourceID, portainer.VolumeResourceControl, true)
+	case strings.HasPrefix(requestPath, "/dockerhub"):
+		dockerhub, err := transport.dataStore.DockerHub().DockerHub()
+		if err != nil {
+			return nil, err
+		}
+
+		newBody, err := json.Marshal(dockerhub)
+		if err != nil {
+			return nil, err
+		}
+
+		r.Method = http.MethodPost
+
+		r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
+		r.ContentLength = int64(len(newBody))
 	}
 
 	return transport.executeDockerRequest(r)

api/http/proxy/factory/kubernetes.go

@@ -72,7 +72,7 @@ func (factory *ProxyFactory) newKubernetesEdgeHTTPProxy(endpoint *portainer.Endp
 
 	endpointURL.Scheme = "http"
 	proxy := newSingleHostReverseProxyWithHostHeader(endpointURL)
-	proxy.Transport = kubernetes.NewEdgeTransport(factory.reverseTunnelService, endpoint.ID, tokenManager)
+	proxy.Transport = kubernetes.NewEdgeTransport(factory.dataStore, factory.reverseTunnelService, endpoint.ID, tokenManager)
 
 	return proxy, nil
 }
@@ -103,7 +103,7 @@ func (factory *ProxyFactory) newKubernetesAgentHTTPSProxy(endpoint *portainer.En
 	}
 
 	proxy := newSingleHostReverseProxyWithHostHeader(remoteURL)
-	proxy.Transport = kubernetes.NewAgentTransport(factory.signatureService, tlsConfig, tokenManager)
+	proxy.Transport = kubernetes.NewAgentTransport(factory.dataStore, factory.signatureService, tlsConfig, tokenManager)
 
 	return proxy, nil
 }

api/http/proxy/factory/kubernetes/transport.go

@@ -1,10 +1,14 @@
 package kubernetes
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"strings"
 
 	"github.com/portainer/portainer/api/http/security"
 
@@ -20,6 +24,7 @@ type (
 	}
 
 	agentTransport struct {
+		dataStore          portainer.DataStore
 		httpTransport      *http.Transport
 		tokenManager       *tokenManager
 		signatureService   portainer.DigitalSignatureService
@@ -27,6 +32,7 @@ type (
 	}
 
 	edgeTransport struct {
+		dataStore            portainer.DataStore
 		httpTransport        *http.Transport
 		tokenManager         *tokenManager
 		reverseTunnelService portainer.ReverseTunnelService
@@ -64,8 +70,9 @@ func (transport *localTransport) RoundTrip(request *http.Request) (*http.Respons
 }
 
 // NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent
-func NewAgentTransport(signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager) *agentTransport {
+func NewAgentTransport(datastore portainer.DataStore, signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager) *agentTransport {
 	transport := &agentTransport{
+		dataStore: datastore,
 		httpTransport: &http.Transport{
 			TLSClientConfig: tlsConfig,
 		},
@@ -85,6 +92,10 @@ func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Respons
 
 	request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
 
+	if strings.HasPrefix(request.URL.Path, "/v2") {
+		decorateAgentRequest(request, transport.dataStore)
+	}
+
 	signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
 	if err != nil {
 		return nil, err
@@ -96,9 +107,10 @@ func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Respons
 	return transport.httpTransport.RoundTrip(request)
 }
 
-// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent
-func NewEdgeTransport(reverseTunnelService portainer.ReverseTunnelService, endpointIdentifier portainer.EndpointID, tokenManager *tokenManager) *edgeTransport {
+// NewEdgeTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent
+func NewEdgeTransport(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, endpointIdentifier portainer.EndpointID, tokenManager *tokenManager) *edgeTransport {
 	transport := &edgeTransport{
+		dataStore:            datastore,
 		httpTransport:        &http.Transport{},
 		tokenManager:         tokenManager,
 		reverseTunnelService: reverseTunnelService,
@@ -117,6 +129,10 @@ func (transport *edgeTransport) RoundTrip(request *http.Request) (*http.Response
 
 	request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token)
 
+	if strings.HasPrefix(request.URL.Path, "/v2") {
+		decorateAgentRequest(request, transport.dataStore)
+	}
+
 	response, err := transport.httpTransport.RoundTrip(request)
 
 	if err == nil {
@@ -151,3 +167,33 @@ func getRoundTripToken(
 
 	return token, nil
 }
+
+func decorateAgentRequest(r *http.Request, dataStore portainer.DataStore) error {
+	requestPath := strings.TrimPrefix(r.URL.Path, "/v2")
+
+	switch {
+	case strings.HasPrefix(requestPath, "/dockerhub"):
+		decorateAgentDockerHubRequest(r, dataStore)
+	}
+
+	return nil
+}
+
+func decorateAgentDockerHubRequest(r *http.Request, dataStore portainer.DataStore) error {
+	dockerhub, err := dataStore.DockerHub().DockerHub()
+	if err != nil {
+		return err
+	}
+
+	newBody, err := json.Marshal(dockerhub)
+	if err != nil {
+		return err
+	}
+
+	r.Method = http.MethodPost
+
+	r.Body = ioutil.NopCloser(bytes.NewReader(newBody))
+	r.ContentLength = int64(len(newBody))
+
+	return nil
+}