diff --git a/api/http/handler/endpointproxy/proxy_kubernetes.go b/api/http/handler/endpointproxy/proxy_kubernetes.go index 7a4e9dc01..ed5b1a28b 100644 --- a/api/http/handler/endpointproxy/proxy_kubernetes.go +++ b/api/http/handler/endpointproxy/proxy_kubernetes.go @@ -3,11 +3,12 @@ package endpointproxy import ( "errors" "fmt" + "strings" "time" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" - "github.com/portainer/portainer/api" + portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" "net/http" @@ -66,9 +67,15 @@ func (handler *Handler) proxyRequestsToKubernetesAPI(w http.ResponseWriter, r *h requestPrefix := fmt.Sprintf("/%d/kubernetes", endpointID) if endpoint.Type == portainer.AgentOnKubernetesEnvironment || endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment { - requestPrefix = fmt.Sprintf("/%d", endpointID) + if isKubernetesRequest(strings.TrimPrefix(r.URL.String(), requestPrefix)) { + requestPrefix = fmt.Sprintf("/%d", endpointID) + } } http.StripPrefix(requestPrefix, proxy).ServeHTTP(w, r) return nil } + +func isKubernetesRequest(requestURL string) bool { + return strings.HasPrefix(requestURL, "/api") +} diff --git a/api/http/handler/endpoints/endpoint_dockerhub_status.go b/api/http/handler/endpoints/endpoint_dockerhub_status.go new file mode 100644 index 000000000..793b85715 --- /dev/null +++ b/api/http/handler/endpoints/endpoint_dockerhub_status.go @@ -0,0 +1,140 @@ +package endpoints + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/http/client" + "github.com/portainer/portainer/api/internal/endpointutils" +) + +type dockerhubStatusResponse struct { + Remaining int `json:"remaining"` + Limit int `json:"limit"` +} + +// GET request on /api/endpoints/{id}/dockerhub/status +func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err} + } + + endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID)) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err} + } else if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err} + } + + if !endpointutils.IsLocalEndpoint(endpoint) { + return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")} + } + + dockerhub, err := handler.DataStore.DockerHub().DockerHub() + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err} + } + + httpClient := client.NewHTTPClient() + token, err := getDockerHubToken(httpClient, dockerhub) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err} + } + + resp, err := getDockerHubLimits(httpClient, token) + if err != nil { + return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub rate limits from DockerHub", err} + } + + return response.JSON(w, resp) +} + +func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) { + type dockerhubTokenResponse struct { + Token string `json:"token"` + } + + requestURL := "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" + + req, err := http.NewRequest(http.MethodGet, requestURL, nil) + if err != nil { + return "", err + } + + if dockerhub.Authentication { + req.SetBasicAuth(dockerhub.Username, dockerhub.Password) + } + + resp, err := httpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New("failed fetching dockerhub token") + } + + var data dockerhubTokenResponse + err = json.NewDecoder(resp.Body).Decode(&data) + if err != nil { + return "", err + } + + return data.Token, nil +} + +func getDockerHubLimits(httpClient *client.HTTPClient, token string) (*dockerhubStatusResponse, error) { + + requestURL := "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" + + req, err := http.NewRequest(http.MethodHead, requestURL, nil) + if err != nil { + return nil, err + } + + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("failed fetching dockerhub limits") + } + + rateLimit, err := parseRateLimitHeader(resp.Header, "RateLimit-Limit") + rateLimitRemaining, err := parseRateLimitHeader(resp.Header, "RateLimit-Remaining") + + return &dockerhubStatusResponse{ + Limit: rateLimit, + Remaining: rateLimitRemaining, + }, nil +} + +func parseRateLimitHeader(headers http.Header, headerKey string) (int, error) { + headerValue := headers.Get(headerKey) + if headerValue == "" { + return 0, fmt.Errorf("Missing %s header", headerKey) + } + + matches := strings.Split(headerValue, ";") + value, err := strconv.Atoi(matches[0]) + if err != nil { + return 0, err + } + + return value, nil +} diff --git a/api/http/handler/endpoints/handler.go b/api/http/handler/endpoints/handler.go index ae23f5c9a..e7a41b8c8 100644 --- a/api/http/handler/endpoints/handler.go +++ b/api/http/handler/endpoints/handler.go @@ -51,6 +51,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut) h.Handle("/endpoints/{id}", bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete) + h.Handle("/endpoints/{id}/dockerhub", + bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet) h.Handle("/endpoints/{id}/extensions", bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost) h.Handle("/endpoints/{id}/extensions/{extensionType}", diff --git a/api/http/proxy/factory/docker/transport.go b/api/http/proxy/factory/docker/transport.go index 5ac4efb4f..f68da56a6 100644 --- a/api/http/proxy/factory/docker/transport.go +++ b/api/http/proxy/factory/docker/transport.go @@ -1,9 +1,11 @@ package docker import ( + "bytes" "encoding/base64" "encoding/json" "errors" + "io/ioutil" "log" "net/http" "path" @@ -163,6 +165,21 @@ func (transport *Transport) proxyAgentRequest(r *http.Request) (*http.Response, // volume browser request return transport.restrictedResourceOperation(r, resourceID, portainer.VolumeResourceControl, true) + case strings.HasPrefix(requestPath, "/dockerhub"): + dockerhub, err := transport.dataStore.DockerHub().DockerHub() + if err != nil { + return nil, err + } + + newBody, err := json.Marshal(dockerhub) + if err != nil { + return nil, err + } + + r.Method = http.MethodPost + + r.Body = ioutil.NopCloser(bytes.NewReader(newBody)) + r.ContentLength = int64(len(newBody)) } return transport.executeDockerRequest(r) diff --git a/api/http/proxy/factory/kubernetes.go b/api/http/proxy/factory/kubernetes.go index 2cb09dc62..a1774c0d2 100644 --- a/api/http/proxy/factory/kubernetes.go +++ b/api/http/proxy/factory/kubernetes.go @@ -72,7 +72,7 @@ func (factory *ProxyFactory) newKubernetesEdgeHTTPProxy(endpoint *portainer.Endp endpointURL.Scheme = "http" proxy := newSingleHostReverseProxyWithHostHeader(endpointURL) - proxy.Transport = kubernetes.NewEdgeTransport(factory.reverseTunnelService, endpoint.ID, tokenManager) + proxy.Transport = kubernetes.NewEdgeTransport(factory.dataStore, factory.reverseTunnelService, endpoint.ID, tokenManager) return proxy, nil } @@ -103,7 +103,7 @@ func (factory *ProxyFactory) newKubernetesAgentHTTPSProxy(endpoint *portainer.En } proxy := newSingleHostReverseProxyWithHostHeader(remoteURL) - proxy.Transport = kubernetes.NewAgentTransport(factory.signatureService, tlsConfig, tokenManager) + proxy.Transport = kubernetes.NewAgentTransport(factory.dataStore, factory.signatureService, tlsConfig, tokenManager) return proxy, nil } diff --git a/api/http/proxy/factory/kubernetes/transport.go b/api/http/proxy/factory/kubernetes/transport.go index c1d0de13b..377c7a3dc 100644 --- a/api/http/proxy/factory/kubernetes/transport.go +++ b/api/http/proxy/factory/kubernetes/transport.go @@ -1,10 +1,14 @@ package kubernetes import ( + "bytes" "crypto/tls" + "encoding/json" "fmt" + "io/ioutil" "log" "net/http" + "strings" "github.com/portainer/portainer/api/http/security" @@ -20,6 +24,7 @@ type ( } agentTransport struct { + dataStore portainer.DataStore httpTransport *http.Transport tokenManager *tokenManager signatureService portainer.DigitalSignatureService @@ -27,6 +32,7 @@ type ( } edgeTransport struct { + dataStore portainer.DataStore httpTransport *http.Transport tokenManager *tokenManager reverseTunnelService portainer.ReverseTunnelService @@ -64,8 +70,9 @@ func (transport *localTransport) RoundTrip(request *http.Request) (*http.Respons } // NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer agent -func NewAgentTransport(signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager) *agentTransport { +func NewAgentTransport(datastore portainer.DataStore, signatureService portainer.DigitalSignatureService, tlsConfig *tls.Config, tokenManager *tokenManager) *agentTransport { transport := &agentTransport{ + dataStore: datastore, httpTransport: &http.Transport{ TLSClientConfig: tlsConfig, }, @@ -85,6 +92,10 @@ func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Respons request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token) + if strings.HasPrefix(request.URL.Path, "/v2") { + decorateAgentRequest(request, transport.dataStore) + } + signature, err := transport.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage) if err != nil { return nil, err @@ -96,9 +107,10 @@ func (transport *agentTransport) RoundTrip(request *http.Request) (*http.Respons return transport.httpTransport.RoundTrip(request) } -// NewAgentTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent -func NewEdgeTransport(reverseTunnelService portainer.ReverseTunnelService, endpointIdentifier portainer.EndpointID, tokenManager *tokenManager) *edgeTransport { +// NewEdgeTransport returns a new transport that can be used to send signed requests to a Portainer Edge agent +func NewEdgeTransport(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, endpointIdentifier portainer.EndpointID, tokenManager *tokenManager) *edgeTransport { transport := &edgeTransport{ + dataStore: datastore, httpTransport: &http.Transport{}, tokenManager: tokenManager, reverseTunnelService: reverseTunnelService, @@ -117,6 +129,10 @@ func (transport *edgeTransport) RoundTrip(request *http.Request) (*http.Response request.Header.Set(portainer.PortainerAgentKubernetesSATokenHeader, token) + if strings.HasPrefix(request.URL.Path, "/v2") { + decorateAgentRequest(request, transport.dataStore) + } + response, err := transport.httpTransport.RoundTrip(request) if err == nil { @@ -151,3 +167,33 @@ func getRoundTripToken( return token, nil } + +func decorateAgentRequest(r *http.Request, dataStore portainer.DataStore) error { + requestPath := strings.TrimPrefix(r.URL.Path, "/v2") + + switch { + case strings.HasPrefix(requestPath, "/dockerhub"): + decorateAgentDockerHubRequest(r, dataStore) + } + + return nil +} + +func decorateAgentDockerHubRequest(r *http.Request, dataStore portainer.DataStore) error { + dockerhub, err := dataStore.DockerHub().DockerHub() + if err != nil { + return err + } + + newBody, err := json.Marshal(dockerhub) + if err != nil { + return err + } + + r.Method = http.MethodPost + + r.Body = ioutil.NopCloser(bytes.NewReader(newBody)) + r.ContentLength = int64(len(newBody)) + + return nil +} diff --git a/api/internal/endpointutils/endpointutils.go b/api/internal/endpointutils/endpointutils.go new file mode 100644 index 000000000..249ee11cb --- /dev/null +++ b/api/internal/endpointutils/endpointutils.go @@ -0,0 +1,11 @@ +package endpointutils + +import ( + "strings" + + portainer "github.com/portainer/portainer/api" +) + +func IsLocalEndpoint(endpoint *portainer.Endpoint) bool { + return strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") || endpoint.Type == 5 +} diff --git a/app/agent/rest/dockerhub.js b/app/agent/rest/dockerhub.js new file mode 100644 index 000000000..b48481e8a --- /dev/null +++ b/app/agent/rest/dockerhub.js @@ -0,0 +1,13 @@ +import angular from 'angular'; + +angular.module('portainer.agent').factory('AgentDockerhub', AgentDockerhub); + +function AgentDockerhub($resource, API_ENDPOINT_ENDPOINTS) { + return $resource( + `${API_ENDPOINT_ENDPOINTS}/:endpointId/:endpointType/v2/dockerhub`, + {}, + { + limits: { method: 'GET' }, + } + ); +} diff --git a/app/docker/components/imageRegistry/por-image-registry-rate-limits.controller.js b/app/docker/components/imageRegistry/por-image-registry-rate-limits.controller.js new file mode 100644 index 000000000..83e9876e4 --- /dev/null +++ b/app/docker/components/imageRegistry/por-image-registry-rate-limits.controller.js @@ -0,0 +1,31 @@ +export default class porImageRegistryContainerController { + /* @ngInject */ + constructor(EndpointHelper, DockerHubService, Notifications) { + this.EndpointHelper = EndpointHelper; + this.DockerHubService = DockerHubService; + this.Notifications = Notifications; + + this.pullRateLimits = null; + } + + $onChanges({ isDockerHubRegistry }) { + if (isDockerHubRegistry && isDockerHubRegistry.currentValue) { + this.fetchRateLimits(); + } + } + + async fetchRateLimits() { + this.pullRateLimits = null; + if (this.EndpointHelper.isAgentEndpoint(this.endpoint) || this.EndpointHelper.isLocalEndpoint(this.endpoint)) { + try { + this.pullRateLimits = await this.DockerHubService.checkRateLimits(this.endpoint); + this.setValidity(this.pullRateLimits.remaining >= 0); + } catch (e) { + // eslint-disable-next-line no-console + console.error('Failed loading DockerHub pull rate limits', e); + } + } else { + this.setValidity(true); + } + } +} diff --git a/app/docker/components/imageRegistry/por-image-registry-rate-limits.html b/app/docker/components/imageRegistry/por-image-registry-rate-limits.html new file mode 100644 index 000000000..31d7eaa82 --- /dev/null +++ b/app/docker/components/imageRegistry/por-image-registry-rate-limits.html @@ -0,0 +1,34 @@ +
+ + When using advanced mode, image and repository must be publicly available. +
+ + ++ Image name is required. + Tag must be specified otherwise Portainer will pull all tags associated to the image. +
+- - When using advanced mode, image and repository must be publicly available. -
- - -Image name is required. - Tag must be specified otherwise Portainer will pull all tags associated to the image.
-{{ config.Image }}
image is not registered inside Portainer, you will not be able to create a container. Please register
- that registry first.
+
+ The Docker registry for the {{ config.Image }}
image is not registered inside Portainer, you will not be able to create a container. Please register that
+ registry first.
+