From be236f9d0941aff39d13da6f8e25817ab7cc8cd7 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Mon, 14 May 2018 21:40:50 +0200 Subject: [PATCH 01/15] fix(api): fix default group for endpoint declared via -H --- api/cmd/portainer/main.go | 7 ++++--- gruntfile.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 0e30cf276..63563d596 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -256,9 +256,10 @@ func main() { } if len(endpoints) == 0 { endpoint := &portainer.Endpoint{ - Name: "primary", - URL: *flags.Endpoint, - Type: portainer.DockerEnvironment, + Name: "primary", + URL: *flags.Endpoint, + GroupID: portainer.EndpointGroupID(1), + Type: portainer.DockerEnvironment, TLSConfig: portainer.TLSConfiguration{ TLS: *flags.TLSVerify, TLSSkipVerify: *flags.TLSSkipVerify, diff --git a/gruntfile.js b/gruntfile.js index a3b79f67f..b5f0e5574 100644 --- a/gruntfile.js +++ b/gruntfile.js @@ -269,7 +269,7 @@ function shell_buildBinary(p, a) { function shell_run(arch) { return [ 'docker rm -f portainer', - 'docker run -d -p 9000:9000 -v $(pwd)/dist:/app -v /tmp/portainer:/data -v /var/run/docker.sock:/var/run/docker.sock:z --name portainer portainer/base /app/portainer-linux-' + arch + ' --no-analytics' + 'docker run -d -p 9000:9000 -v $(pwd)/dist:/app -v /tmp/portainer:/data -v /var/run/docker.sock:/var/run/docker.sock:z --name portainer portainer/base /app/portainer-linux-' + arch + ' --no-analytics -H unix:///var/run/docker.sock' ].join(';'); } From a09af01e17e50337d402ace9754662f84137a91b Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Mon, 14 May 2018 21:41:24 +0200 Subject: [PATCH 02/15] chore(build-system): update gruntfile --- gruntfile.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gruntfile.js b/gruntfile.js index b5f0e5574..a3b79f67f 100644 --- a/gruntfile.js +++ b/gruntfile.js @@ -269,7 +269,7 @@ function shell_buildBinary(p, a) { function shell_run(arch) { return [ 'docker rm -f portainer', - 'docker run -d -p 9000:9000 -v $(pwd)/dist:/app -v /tmp/portainer:/data -v /var/run/docker.sock:/var/run/docker.sock:z --name portainer portainer/base /app/portainer-linux-' + arch + ' --no-analytics -H unix:///var/run/docker.sock' + 'docker run -d -p 9000:9000 -v $(pwd)/dist:/app -v /tmp/portainer:/data -v /var/run/docker.sock:/var/run/docker.sock:z --name portainer portainer/base /app/portainer-linux-' + arch + ' --no-analytics' ].join(';'); } From 2c0595f5ed743ca1e5b111527b1e9b4eaf3d2669 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 15 May 2018 14:12:49 +0200 Subject: [PATCH 03/15] feat(exec): relocate config.json to data folder and re-use existing content (#1898) --- api/cmd/portainer/main.go | 6 ++-- api/exec/stack_manager.go | 64 +++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 63563d596..80b62457c 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -62,8 +62,8 @@ func initStore(dataStorePath string) *bolt.Store { return store } -func initStackManager(assetsPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (portainer.StackManager, error) { - return exec.NewStackManager(assetsPath, signatureService, fileService) +func initStackManager(assetsPath string, dataStorePath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (portainer.StackManager, error) { + return exec.NewStackManager(assetsPath, dataStorePath, signatureService, fileService) } func initJWTService(authenticationEnabled bool) portainer.JWTService { @@ -232,7 +232,7 @@ func main() { log.Fatal(err) } - stackManager, err := initStackManager(*flags.Assets, digitalSignatureService, fileService) + stackManager, err := initStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService) if err != nil { log.Fatal(err) } diff --git a/api/exec/stack_manager.go b/api/exec/stack_manager.go index d8da84df1..cf0f976b4 100644 --- a/api/exec/stack_manager.go +++ b/api/exec/stack_manager.go @@ -2,6 +2,7 @@ package exec import ( "bytes" + "encoding/json" "os" "os/exec" "path" @@ -13,28 +14,22 @@ import ( // StackManager represents a service for managing stacks. type StackManager struct { binaryPath string + dataPath string signatureService portainer.DigitalSignatureService fileService portainer.FileService } -type dockerCLIConfiguration struct { - HTTPHeaders struct { - ManagerOperationHeader string `json:"X-PortainerAgent-ManagerOperation"` - SignatureHeader string `json:"X-PortainerAgent-Signature"` - PublicKey string `json:"X-PortainerAgent-PublicKey"` - } `json:"HttpHeaders"` -} - // NewStackManager initializes a new StackManager service. // It also updates the configuration of the Docker CLI binary. -func NewStackManager(binaryPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (*StackManager, error) { +func NewStackManager(binaryPath, dataPath string, signatureService portainer.DigitalSignatureService, fileService portainer.FileService) (*StackManager, error) { manager := &StackManager{ binaryPath: binaryPath, + dataPath: dataPath, signatureService: signatureService, fileService: fileService, } - err := manager.updateDockerCLIConfiguration(binaryPath) + err := manager.updateDockerCLIConfiguration(dataPath) if err != nil { return nil, err } @@ -44,7 +39,7 @@ func NewStackManager(binaryPath string, signatureService portainer.DigitalSignat // Login executes the docker login command against a list of registries (including DockerHub). func (manager *StackManager) Login(dockerhub *portainer.DockerHub, registries []portainer.Registry, endpoint *portainer.Endpoint) { - command, args := prepareDockerCommandAndArgs(manager.binaryPath, endpoint) + command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) for _, registry := range registries { if registry.Authentication { registryArgs := append(args, "login", "--username", registry.Username, "--password", registry.Password, registry.URL) @@ -60,7 +55,7 @@ func (manager *StackManager) Login(dockerhub *portainer.DockerHub, registries [] // Logout executes the docker logout command. func (manager *StackManager) Logout(endpoint *portainer.Endpoint) error { - command, args := prepareDockerCommandAndArgs(manager.binaryPath, endpoint) + command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) args = append(args, "logout") return runCommandAndCaptureStdErr(command, args, nil, "") } @@ -68,7 +63,7 @@ func (manager *StackManager) Logout(endpoint *portainer.Endpoint) error { // Deploy executes the docker stack deploy command. func (manager *StackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error { stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) - command, args := prepareDockerCommandAndArgs(manager.binaryPath, endpoint) + command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) if prune { args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) @@ -87,7 +82,7 @@ func (manager *StackManager) Deploy(stack *portainer.Stack, prune bool, endpoint // Remove executes the docker stack rm command. func (manager *StackManager) Remove(stack *portainer.Stack, endpoint *portainer.Endpoint) error { - command, args := prepareDockerCommandAndArgs(manager.binaryPath, endpoint) + command, args := prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) args = append(args, "stack", "rm", stack.Name) return runCommandAndCaptureStdErr(command, args, nil, "") } @@ -111,7 +106,7 @@ func runCommandAndCaptureStdErr(command string, args []string, env []string, wor return nil } -func prepareDockerCommandAndArgs(binaryPath string, endpoint *portainer.Endpoint) (string, []string) { +func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portainer.Endpoint) (string, []string) { // Assume Linux as a default command := path.Join(binaryPath, "docker") @@ -120,7 +115,7 @@ func prepareDockerCommandAndArgs(binaryPath string, endpoint *portainer.Endpoint } args := make([]string, 0) - args = append(args, "--config", binaryPath) + args = append(args, "--config", dataPath) args = append(args, "-H", endpoint.URL) if !endpoint.TLSConfig.TLS && endpoint.TLSConfig.TLSSkipVerify { @@ -140,21 +135,46 @@ func prepareDockerCommandAndArgs(binaryPath string, endpoint *portainer.Endpoint return command, args } -func (manager *StackManager) updateDockerCLIConfiguration(binaryPath string) error { - config := dockerCLIConfiguration{} - config.HTTPHeaders.ManagerOperationHeader = "1" +func (manager *StackManager) updateDockerCLIConfiguration(dataPath string) error { + configFilePath := path.Join(dataPath, "config.json") + config, err := manager.retrieveConfigurationFromDisk(configFilePath) + if err != nil { + return err + } signature, err := manager.signatureService.Sign(portainer.PortainerAgentSignatureMessage) if err != nil { return err } - config.HTTPHeaders.SignatureHeader = signature - config.HTTPHeaders.PublicKey = manager.signatureService.EncodedPublicKey() - err = manager.fileService.WriteJSONToFile(path.Join(binaryPath, "config.json"), config) + if config["HttpHeaders"] == nil { + config["HttpHeaders"] = make(map[string]interface{}) + } + headersObject := config["HttpHeaders"].(map[string]interface{}) + headersObject["X-PortainerAgent-ManagerOperation"] = "1" + headersObject["X-PortainerAgent-Signature"] = signature + headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey() + + err = manager.fileService.WriteJSONToFile(configFilePath, config) if err != nil { return err } return nil } + +func (manager *StackManager) retrieveConfigurationFromDisk(path string) (map[string]interface{}, error) { + var config map[string]interface{} + + raw, err := manager.fileService.GetFileContent(path) + if err != nil { + return make(map[string]interface{}), nil + } + + err = json.Unmarshal([]byte(raw), &config) + if err != nil { + return nil, err + } + + return config, nil +} From 79f4c20c255d7b72cc26e1db2aefc5fef6ac34fd Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 15 May 2018 18:24:54 +0200 Subject: [PATCH 04/15] fix(endpoints): set TLSSkipVerify to false when TLS is not enabled during update (#1896) --- api/http/handler/endpoint.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/http/handler/endpoint.go b/api/http/handler/endpoint.go index 141f8d29d..497ee9cae 100644 --- a/api/http/handler/endpoint.go +++ b/api/http/handler/endpoint.go @@ -472,7 +472,7 @@ func (handler *EndpointHandler) handlePutEndpoint(w http.ResponseWriter, r *http } } else { endpoint.TLSConfig.TLS = false - endpoint.TLSConfig.TLSSkipVerify = true + endpoint.TLSConfig.TLSSkipVerify = false endpoint.TLSConfig.TLSCACertPath = "" endpoint.TLSConfig.TLSCertPath = "" endpoint.TLSConfig.TLSKeyPath = "" From 5df09923b63973eec41fc3199e8df94da5015fbe Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Tue, 15 May 2018 19:13:27 +0200 Subject: [PATCH 05/15] feat(api): add debug statements in response handling --- api/http/proxy/response.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/http/proxy/response.go b/api/http/proxy/response.go index 751e3df31..3be65bbc6 100644 --- a/api/http/proxy/response.go +++ b/api/http/proxy/response.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "io/ioutil" + "log" "net/http" "strconv" @@ -48,8 +49,10 @@ func getResponseAsJSONArray(response *http.Response) ([]interface{}, error) { if responseObject["message"] != nil { return nil, portainer.Error(responseObject["message"].(string)) } + log.Printf("Response: %+v\n", responseObject) return nil, ErrInvalidResponseContent default: + log.Printf("Response: %+v\n", responseObject) return nil, ErrInvalidResponseContent } } From 32800a843a9c06d8918b86040faa64fe65c81a41 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Wed, 16 May 2018 08:49:14 +0200 Subject: [PATCH 06/15] feat(sidebar): update endpoint selection UX (#1902) * style(sidebar): update selected endpoint name color * feat(sidebar): sort groups/endpoints alphabetically --- app/portainer/views/sidebar/sidebar.html | 2 +- app/portainer/views/sidebar/sidebarController.js | 4 ++-- assets/css/app.css | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/portainer/views/sidebar/sidebar.html b/app/portainer/views/sidebar/sidebar.html index ca809fb75..e182e5a01 100644 --- a/app/portainer/views/sidebar/sidebar.html +++ b/app/portainer/views/sidebar/sidebar.html @@ -14,7 +14,7 @@ groups="groups" select-endpoint="switchEndpoint" > -
  • {{ activeEndpoint.Name }}
    • +
  • {{ activeEndpoint.Name }}
    • Date: Wed, 16 May 2018 09:13:46 +0200 Subject: [PATCH 07/15] fix(websocket): feat(websocket): remove Origin header before handling request (#1901) --- api/http/handler/websocket.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/http/handler/websocket.go b/api/http/handler/websocket.go index 3e624d3b4..01172306d 100644 --- a/api/http/handler/websocket.go +++ b/api/http/handler/websocket.go @@ -94,6 +94,8 @@ func (handler *WebSocketHandler) handleWebsocketExec(w http.ResponseWriter, r *h } func (handler *WebSocketHandler) handleRequest(w http.ResponseWriter, r *http.Request, params *webSocketExecRequestParams) error { + r.Header.Del("Origin") + if params.nodeName != "" { return handler.proxyWebsocketRequest(w, r, params) } @@ -135,7 +137,6 @@ func (handler *WebSocketHandler) proxyWebsocketRequest(w http.ResponseWriter, r out.Set(portainer.PortainerAgentTargetHeader, params.nodeName) } - r.Header.Del("Origin") proxy.ServeHTTP(w, r) return nil From 034fde6d1acbb6f6a80d896160c4b58c294da6e0 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 18 May 2018 10:07:25 +0200 Subject: [PATCH 08/15] chore(codefresh): add branch pipeline --- .codefresh/codefresh_branch.yml | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .codefresh/codefresh_branch.yml diff --git a/.codefresh/codefresh_branch.yml b/.codefresh/codefresh_branch.yml new file mode 100644 index 000000000..4c5fb1222 --- /dev/null +++ b/.codefresh/codefresh_branch.yml @@ -0,0 +1,46 @@ +version: '1.0' +steps: + + build_backend: + image: portainer/golang-builder:ci + working_directory: ${{main_clone}} + commands: + - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}} + - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}} + - /build.sh api/cmd/portainer + + build_frontend: + image: portainer/angular-builder:latest + working_directory: ${{build_backend}} + commands: + - yarn + - yarn grunt build-webapp + - mv api/cmd/portainer/portainer dist/ + + get_docker_version: + image: alpine + working_directory: ${{build_frontend}} + commands: + - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2` + + download_docker_binary: + image: busybox + working_directory: ${{build_frontend}} + commands: + - echo ${{DOCKER_VERSION}} + - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz + - tar -xf /tmp/docker-binaries.tgz -C /tmp + - mv /tmp/docker/docker dist/ + + build_image: + type: build + working_directory: ${{download_docker_binary}} + dockerfile: ./build/linux/Dockerfile + image_name: portainer/portainer + tag: ${{CF_BRANCH}} + + push_image: + type: push + candidate: '${{build_image}}' + tag: '${{CF_BRANCH}}' + registry: dockerhub From 0ea91f7185bf44461e6c8ff52232707c8a9caeb2 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 18 May 2018 10:15:56 +0200 Subject: [PATCH 09/15] chore(codefresh): remove develop pipeline --- .codefresh/codefresh_develop.yml | 50 -------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .codefresh/codefresh_develop.yml diff --git a/.codefresh/codefresh_develop.yml b/.codefresh/codefresh_develop.yml deleted file mode 100644 index 7a17aee3e..000000000 --- a/.codefresh/codefresh_develop.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: '1.0' -steps: - - build_backend: - image: portainer/golang-builder:ci - working_directory: ${{main_clone}} - commands: - - mkdir -p /go/src/github.com/${{CF_REPO_OWNER}} - - ln -s /codefresh/volume/${{CF_REPO_NAME}}/api /go/src/github.com/${{CF_REPO_OWNER}}/${{CF_REPO_NAME}} - - /build.sh api/cmd/portainer - - build_frontend: - image: portainer/angular-builder:latest - working_directory: ${{build_backend}} - commands: - - yarn - - yarn grunt build-webapp - - mv api/cmd/portainer/portainer dist/ - - get_docker_version: - image: alpine - working_directory: ${{build_frontend}} - commands: - - cf_export DOCKER_VERSION=`cat gruntfile.js | grep -m 1 'shippedDockerVersion' | cut -d\' -f2` - - download_docker_binary: - image: busybox - working_directory: ${{build_frontend}} - commands: - - echo ${{DOCKER_VERSION}} - - wget -O /tmp/docker-binaries.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${{DOCKER_VERSION}}.tgz - - tar -xf /tmp/docker-binaries.tgz -C /tmp - - mv /tmp/docker/docker dist/ - - build_image: - type: build - working_directory: ${{download_docker_binary}} - dockerfile: ./build/linux/Dockerfile - image_name: portainer/portainer - tag: ${{CF_BRANCH}} - - push_image: - type: push - candidate: '${{build_image}}' - tag: '${{CF_BRANCH}}' - registry: dockerhub - when: - branch: - only: - - develop From e7e7d73f20fa3beb2f15736c8d9e92038c604ee1 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Fri, 18 May 2018 10:58:16 +0200 Subject: [PATCH 10/15] docs(api): update swagger.yml --- api/swagger.yaml | 102 +++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/api/swagger.yaml b/api/swagger.yaml index fea616110..0fb676c7a 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -228,12 +228,45 @@ paths: produces: - "application/json" parameters: - - in: "body" - name: "body" - description: "Endpoint details" + - name: "Name" + in: "formData" + type: "string" + description: "Name that will be used to identify this endpoint (example: my-endpoint)" required: true - schema: - $ref: "#/definitions/EndpointCreateRequest" + - name: "URL" + in: "formData" + type: "string" + description: "URL or IP address of a Docker host (example: docker.mydomain.tld:2375)" + required: true + - name: "PublicURL" + in: "formData" + type: "string" + description: "URL or IP address where exposed containers will be reachable.\ + \ Defaults to URL if not specified (example: docker.mydomain.tld:2375)" + - name: "GroupID" + in: "formData" + type: "string" + description: "Endpoint group identifier. If not specified will default to 1 (unassigned)." + - name: "TLS" + in: "formData" + type: "string" + description: "Require TLS to connect against this endpoint (example: true)" + - name: "TLSSkipVerify" + in: "formData" + type: "string" + description: "Skip server verification when using TLS" (example: false) + - name: "TLSCACertFile" + in: "formData" + type: "file" + description: "TLS CA certificate file" + - name: "TLSCertFile" + in: "formData" + type: "file" + description: "TLS client certificate file" + - name: "TLSKeyFile" + in: "formData" + type: "file" + description: "TLS client key file" responses: 200: description: "Success" @@ -2344,10 +2377,22 @@ definitions: type: "string" example: "my-endpoint" description: "Endpoint name" + Type: + type: "integer" + example: 1 + description: "Endpoint environment type. 1 for a Docker environment, 2 for an agent on Docker environment." URL: type: "string" example: "docker.mydomain.tld:2375" description: "URL or IP address of the Docker host associated to this endpoint" + PublicURL: + type: "string" + example: "docker.mydomain.tld:2375" + description: "URL or IP address where exposed containers will be reachable" + GroupID: + type: "integer" + example: 1 + description: "Endpoint group identifier" AuthorizedUsers: type: "array" description: "List of user identifiers authorized to connect to this endpoint" @@ -2424,53 +2469,6 @@ definitions: type: "string" example: "hub_password" description: "Password used to authenticate against the DockerHub" - EndpointCreateRequest: - type: "object" - required: - - "Name" - - "URL" - properties: - Name: - type: "string" - example: "my-endpoint" - description: "Name that will be used to identify this endpoint" - URL: - type: "string" - example: "docker.mydomain.tld:2375" - description: "URL or IP address of a Docker host" - PublicURL: - type: "string" - example: "docker.mydomain.tld:2375" - description: "URL or IP address where exposed containers will be reachable.\ - \ Defaults to URL if not specified" - TLS: - type: "boolean" - example: true - description: "Require TLS to connect against this endpoint" - TLSSkipVerify: - type: "boolean" - example: false - description: "Skip server verification when using TLS" - TLSSkipClientVerify: - type: "boolean" - example: false - description: "Skip client verification when using TLS" - TLSCACertFile: - type: "file" - description: "TLS CA certificate file" - TLSCertFile: - type: "file" - description: "TLS client certificate file" - TLSKeyFile: - type: "file" - description: "TLS client key file" - EndpointCreateResponse: - type: "object" - properties: - Id: - type: "integer" - example: 1 - description: "Id of the endpoint" EndpointListResponse: type: "array" items: From 5d3f438288f886f379dad0fb3dd33070cace0a66 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Sat, 19 May 2018 10:47:58 +0200 Subject: [PATCH 11/15] fix(tasks): fix an issue when filtering tasks (#1913) --- .../datatables/tasks-datatable/tasksDatatable.html | 8 ++++---- app/docker/helpers/serviceHelper.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/docker/components/datatables/tasks-datatable/tasksDatatable.html b/app/docker/components/datatables/tasks-datatable/tasksDatatable.html index dc151b5ca..abe642d60 100644 --- a/app/docker/components/datatables/tasks-datatable/tasksDatatable.html +++ b/app/docker/components/datatables/tasks-datatable/tasksDatatable.html @@ -60,8 +60,8 @@ - {{ item.Service.Name }}{{ item.Slot ? '.' + item.Slot : '' }}{{ '.' + item.Id }} - {{ item.Service.Name }}{{ item.Slot ? '.' + item.Slot : '' }}{{ '.' + item.Id }} + {{ item.ServiceName }}{{ item.Slot ? '.' + item.Slot : '' }}{{ '.' + item.Id }} + {{ item.ServiceName }}{{ item.Slot ? '.' + item.Slot : '' }}{{ '.' + item.Id }} {{ item.Status.State }} {{ item.Slot ? item.Slot : '-' }} @@ -77,10 +77,10 @@ - Loading... + Loading... - No task available. + No task available. diff --git a/app/docker/helpers/serviceHelper.js b/app/docker/helpers/serviceHelper.js index 3a170556a..3411a6848 100644 --- a/app/docker/helpers/serviceHelper.js +++ b/app/docker/helpers/serviceHelper.js @@ -11,7 +11,7 @@ angular.module('portainer.docker') var task = tasks[i]; if (task.ServiceId === service.Id) { service.Tasks.push(task); - task.Service = service; + task.ServiceName = service.Name; } else { otherServicesTasks.push(task); } From 63d338c4da071356c022aebc196fb7998c2036e0 Mon Sep 17 00:00:00 2001 From: Anthony Lapenna Date: Sat, 19 May 2018 16:25:11 +0200 Subject: [PATCH 12/15] fix(api): refactor TLS support (#1909) * refactor(api): refactor TLS support * feat(api): migrate endpoint data * refactor(api): remove unused code and rename functions * refactor(app): remove console.log statement --- api/bolt/migrate_dbversion10.go | 28 +++++++ api/bolt/migrator.go | 8 ++ api/cli/cli.go | 18 ++--- api/cli/defaults.go | 2 +- api/cli/defaults_windows.go | 2 +- api/cmd/portainer/main.go | 131 ++++++++++++++++++++++---------- api/crypto/tls.go | 31 ++++---- api/exec/stack_manager.go | 4 +- api/http/client/client.go | 31 -------- api/http/handler/endpoint.go | 2 +- api/http/handler/websocket.go | 2 +- api/http/proxy/factory.go | 2 +- api/http/proxy/manager.go | 2 +- api/ldap/ldap.go | 2 +- api/portainer.go | 6 +- 15 files changed, 163 insertions(+), 108 deletions(-) create mode 100644 api/bolt/migrate_dbversion10.go diff --git a/api/bolt/migrate_dbversion10.go b/api/bolt/migrate_dbversion10.go new file mode 100644 index 000000000..211d2497a --- /dev/null +++ b/api/bolt/migrate_dbversion10.go @@ -0,0 +1,28 @@ +package bolt + +import "github.com/portainer/portainer" + +func (m *Migrator) updateEndpointsToVersion11() error { + legacyEndpoints, err := m.EndpointService.Endpoints() + if err != nil { + return err + } + + for _, endpoint := range legacyEndpoints { + if endpoint.Type == portainer.AgentOnDockerEnvironment { + endpoint.TLSConfig.TLS = true + endpoint.TLSConfig.TLSSkipVerify = true + } else { + if endpoint.TLSConfig.TLSSkipVerify && !endpoint.TLSConfig.TLS { + endpoint.TLSConfig.TLSSkipVerify = false + } + } + + err = m.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint) + if err != nil { + return err + } + } + + return nil +} diff --git a/api/bolt/migrator.go b/api/bolt/migrator.go index 66566ae8d..48242adaf 100644 --- a/api/bolt/migrator.go +++ b/api/bolt/migrator.go @@ -112,6 +112,14 @@ func (m *Migrator) Migrate() error { } } + // https://github.com/portainer/portainer/issues/1906 + if m.CurrentDBVersion < 11 { + err := m.updateEndpointsToVersion11() + if err != nil { + return err + } + } + err := m.VersionService.StoreDBVersion(portainer.DBVersion) if err != nil { return err diff --git a/api/cli/cli.go b/api/cli/cli.go index f2307718b..b9e00ca66 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -33,11 +33,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(), Assets: kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(), Data: kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(), - Endpoint: kingpin.Flag("host", "Dockerd endpoint").Short('H').String(), + EndpointURL: kingpin.Flag("host", "Endpoint URL").Short('H').String(), ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints").String(), NoAuth: kingpin.Flag("no-auth", "Disable authentication").Default(defaultNoAuth).Bool(), NoAnalytics: kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAnalytics).Bool(), - TLSVerify: kingpin.Flag("tlsverify", "TLS support").Default(defaultTLSVerify).Bool(), + TLS: kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(), TLSSkipVerify: kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(), TLSCacert: kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(), TLSCert: kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(), @@ -69,11 +69,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { // ValidateFlags validates the values of the flags. func (*Service) ValidateFlags(flags *portainer.CLIFlags) error { - if *flags.Endpoint != "" && *flags.ExternalEndpoints != "" { + if *flags.EndpointURL != "" && *flags.ExternalEndpoints != "" { return errEndpointExcludeExternal } - err := validateEndpoint(*flags.Endpoint) + err := validateEndpointURL(*flags.EndpointURL) if err != nil { return err } @@ -99,14 +99,14 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error { return nil } -func validateEndpoint(endpoint string) error { - if endpoint != "" { - if !strings.HasPrefix(endpoint, "unix://") && !strings.HasPrefix(endpoint, "tcp://") { +func validateEndpointURL(endpointURL string) error { + if endpointURL != "" { + if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") { return errInvalidEndpointProtocol } - if strings.HasPrefix(endpoint, "unix://") { - socketPath := strings.TrimPrefix(endpoint, "unix://") + if strings.HasPrefix(endpointURL, "unix://") { + socketPath := strings.TrimPrefix(endpointURL, "unix://") if _, err := os.Stat(socketPath); err != nil { if os.IsNotExist(err) { return errSocketNotFound diff --git a/api/cli/defaults.go b/api/cli/defaults.go index b2e40c969..419e5fd81 100644 --- a/api/cli/defaults.go +++ b/api/cli/defaults.go @@ -8,7 +8,7 @@ const ( defaultAssetsDirectory = "./" defaultNoAuth = "false" defaultNoAnalytics = "false" - defaultTLSVerify = "false" + defaultTLS = "false" defaultTLSSkipVerify = "false" defaultTLSCACertPath = "/certs/ca.pem" defaultTLSCertPath = "/certs/cert.pem" diff --git a/api/cli/defaults_windows.go b/api/cli/defaults_windows.go index 6fead127e..2bd909c22 100644 --- a/api/cli/defaults_windows.go +++ b/api/cli/defaults_windows.go @@ -6,7 +6,7 @@ const ( defaultAssetsDirectory = "./" defaultNoAuth = "false" defaultNoAnalytics = "false" - defaultTLSVerify = "false" + defaultTLS = "false" defaultTLSSkipVerify = "false" defaultTLSCACertPath = "C:\\certs\\ca.pem" defaultTLSCertPath = "C:\\certs\\cert.pem" diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 80b62457c..3c2dba7c6 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -1,6 +1,8 @@ package main // import "github.com/portainer/portainer" import ( + "strings" + "github.com/portainer/portainer" "github.com/portainer/portainer/bolt" "github.com/portainer/portainer/cli" @@ -207,6 +209,93 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D return generateAndStoreKeyPair(fileService, signatureService) } +func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error { + tlsConfiguration := portainer.TLSConfiguration{ + TLS: *flags.TLS, + TLSSkipVerify: *flags.TLSSkipVerify, + } + + if *flags.TLS { + tlsConfiguration.TLSCACertPath = *flags.TLSCacert + tlsConfiguration.TLSCertPath = *flags.TLSCert + tlsConfiguration.TLSKeyPath = *flags.TLSKey + } else if !*flags.TLS && *flags.TLSSkipVerify { + tlsConfiguration.TLS = true + } + + endpoint := &portainer.Endpoint{ + Name: "primary", + URL: *flags.EndpointURL, + GroupID: portainer.EndpointGroupID(1), + Type: portainer.DockerEnvironment, + TLSConfig: tlsConfiguration, + AuthorizedUsers: []portainer.UserID{}, + AuthorizedTeams: []portainer.TeamID{}, + Extensions: []portainer.EndpointExtension{}, + } + + if strings.HasPrefix(endpoint.URL, "tcp://") { + tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify) + if err != nil { + return err + } + + agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig) + if err != nil { + return err + } + + if agentOnDockerEnvironment { + endpoint.Type = portainer.AgentOnDockerEnvironment + } + } + + return endpointService.CreateEndpoint(endpoint) +} + +func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService) error { + if strings.HasPrefix(endpointURL, "tcp://") { + _, err := client.ExecutePingOperation(endpointURL, nil) + if err != nil { + return err + } + } + + endpoint := &portainer.Endpoint{ + Name: "primary", + URL: endpointURL, + GroupID: portainer.EndpointGroupID(1), + Type: portainer.DockerEnvironment, + TLSConfig: portainer.TLSConfiguration{}, + AuthorizedUsers: []portainer.UserID{}, + AuthorizedTeams: []portainer.TeamID{}, + Extensions: []portainer.EndpointExtension{}, + } + + return endpointService.CreateEndpoint(endpoint) +} + +func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error { + if *flags.EndpointURL == "" { + return nil + } + + endpoints, err := endpointService.Endpoints() + if err != nil { + return err + } + + if len(endpoints) > 0 { + log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.") + return nil + } + + if *flags.TLS || *flags.TLSSkipVerify { + return createTLSSecuredEndpoint(flags, endpointService) + } + return createUnsecuredEndpoint(*flags.EndpointURL, endpointService) +} + func main() { flags := initCLI() @@ -249,45 +338,9 @@ func main() { applicationStatus := initStatus(authorizeEndpointMgmt, flags) - if *flags.Endpoint != "" { - endpoints, err := store.EndpointService.Endpoints() - if err != nil { - log.Fatal(err) - } - if len(endpoints) == 0 { - endpoint := &portainer.Endpoint{ - Name: "primary", - URL: *flags.Endpoint, - GroupID: portainer.EndpointGroupID(1), - Type: portainer.DockerEnvironment, - TLSConfig: portainer.TLSConfiguration{ - TLS: *flags.TLSVerify, - TLSSkipVerify: *flags.TLSSkipVerify, - TLSCACertPath: *flags.TLSCacert, - TLSCertPath: *flags.TLSCert, - TLSKeyPath: *flags.TLSKey, - }, - AuthorizedUsers: []portainer.UserID{}, - AuthorizedTeams: []portainer.TeamID{}, - Extensions: []portainer.EndpointExtension{}, - } - - agentOnDockerEnvironment, err := client.ExecutePingOperationFromEndpoint(endpoint) - if err != nil { - log.Fatal(err) - } - - if agentOnDockerEnvironment { - endpoint.Type = portainer.AgentOnDockerEnvironment - } - - err = store.EndpointService.CreateEndpoint(endpoint) - if err != nil { - log.Fatal(err) - } - } else { - log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.") - } + err = initEndpoint(flags, store.EndpointService) + if err != nil { + log.Fatal(err) } adminPasswordHash := "" diff --git a/api/crypto/tls.go b/api/crypto/tls.go index ae6bac867..641aed142 100644 --- a/api/crypto/tls.go +++ b/api/crypto/tls.go @@ -4,11 +4,11 @@ import ( "crypto/tls" "crypto/x509" "io/ioutil" - - "github.com/portainer/portainer" ) -func CreateTLSConfig(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) { +// CreateTLSConfigurationFromBytes initializes a tls.Config using a CA certificate, a certificate and a key +// loaded from memory. +func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) { config := &tls.Config{} config.InsecureSkipVerify = skipServerVerification @@ -29,32 +29,31 @@ func CreateTLSConfig(caCert, cert, key []byte, skipClientVerification, skipServe return config, nil } -// CreateTLSConfiguration initializes a tls.Config using a CA certificate, a certificate and a key -func CreateTLSConfiguration(config *portainer.TLSConfiguration) (*tls.Config, error) { - TLSConfig := &tls.Config{} +// CreateTLSConfigurationFromDisk initializes a tls.Config using a CA certificate, a certificate and a key +// loaded from disk. +func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipServerVerification bool) (*tls.Config, error) { + config := &tls.Config{} + config.InsecureSkipVerify = skipServerVerification - if config.TLS && config.TLSCertPath != "" && config.TLSKeyPath != "" { - cert, err := tls.LoadX509KeyPair(config.TLSCertPath, config.TLSKeyPath) + if certPath != "" && keyPath != "" { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) if err != nil { return nil, err } - TLSConfig.Certificates = []tls.Certificate{cert} + config.Certificates = []tls.Certificate{cert} } - if config.TLS && !config.TLSSkipVerify { - caCert, err := ioutil.ReadFile(config.TLSCACertPath) + if !skipServerVerification && caCertPath != "" { + caCert, err := ioutil.ReadFile(caCertPath) if err != nil { return nil, err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - - TLSConfig.RootCAs = caCertPool + config.RootCAs = caCertPool } - TLSConfig.InsecureSkipVerify = config.TLSSkipVerify - - return TLSConfig, nil + return config, nil } diff --git a/api/exec/stack_manager.go b/api/exec/stack_manager.go index cf0f976b4..d8c2d9438 100644 --- a/api/exec/stack_manager.go +++ b/api/exec/stack_manager.go @@ -118,9 +118,7 @@ func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portaine args = append(args, "--config", dataPath) args = append(args, "-H", endpoint.URL) - if !endpoint.TLSConfig.TLS && endpoint.TLSConfig.TLSSkipVerify { - args = append(args, "--tls") - } else if endpoint.TLSConfig.TLS { + if endpoint.TLSConfig.TLS { args = append(args, "--tls") if !endpoint.TLSConfig.TLSSkipVerify { diff --git a/api/http/client/client.go b/api/http/client/client.go index b9e3f318b..438be12ad 100644 --- a/api/http/client/client.go +++ b/api/http/client/client.go @@ -7,39 +7,8 @@ import ( "time" "github.com/portainer/portainer" - "github.com/portainer/portainer/crypto" ) -// ExecutePingOperationFromEndpoint will send a SystemPing operation HTTP request to a Docker environment -// using the specified endpoint configuration. It is used exclusively when -// specifying an endpoint from the CLI via the -H flag. -func ExecutePingOperationFromEndpoint(endpoint *portainer.Endpoint) (bool, error) { - if strings.HasPrefix(endpoint.URL, "unix://") { - return false, nil - } - - transport := &http.Transport{} - - scheme := "http" - - if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify { - tlsConfig, err := crypto.CreateTLSConfiguration(&endpoint.TLSConfig) - if err != nil { - return false, err - } - scheme = "https" - transport.TLSClientConfig = tlsConfig - } - - client := &http.Client{ - Timeout: time.Second * 3, - Transport: transport, - } - - target := strings.Replace(endpoint.URL, "tcp://", scheme+"://", 1) - return pingOperation(client, target) -} - // ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment // using the specified host and optional TLS configuration. func ExecutePingOperation(host string, tlsConfig *tls.Config) (bool, error) { diff --git a/api/http/handler/endpoint.go b/api/http/handler/endpoint.go index 497ee9cae..23c867303 100644 --- a/api/http/handler/endpoint.go +++ b/api/http/handler/endpoint.go @@ -121,7 +121,7 @@ func (handler *EndpointHandler) handleGetEndpoints(w http.ResponseWriter, r *htt } func (handler *EndpointHandler) createTLSSecuredEndpoint(payload *postEndpointPayload) (*portainer.Endpoint, error) { - tlsConfig, err := crypto.CreateTLSConfig(payload.caCert, payload.cert, payload.key, payload.skipTLSClientVerification, payload.skipTLSServerVerification) + tlsConfig, err := crypto.CreateTLSConfigurationFromBytes(payload.caCert, payload.cert, payload.key, payload.skipTLSClientVerification, payload.skipTLSServerVerification) if err != nil { return nil, err } diff --git a/api/http/handler/websocket.go b/api/http/handler/websocket.go index 01172306d..629e60390 100644 --- a/api/http/handler/websocket.go +++ b/api/http/handler/websocket.go @@ -188,7 +188,7 @@ func createDial(endpoint *portainer.Endpoint) (net.Conn, error) { } if endpoint.TLSConfig.TLS { - tlsConfig, err := crypto.CreateTLSConfiguration(&endpoint.TLSConfig) + tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify) if err != nil { return nil, err } diff --git a/api/http/proxy/factory.go b/api/http/proxy/factory.go index 9df363332..8f952f2dc 100644 --- a/api/http/proxy/factory.go +++ b/api/http/proxy/factory.go @@ -29,7 +29,7 @@ func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portaine u.Scheme = "https" proxy := factory.createDockerReverseProxy(u, enableSignature) - config, err := crypto.CreateTLSConfiguration(tlsConfig) + config, err := crypto.CreateTLSConfigurationFromDisk(tlsConfig.TLSCACertPath, tlsConfig.TLSCertPath, tlsConfig.TLSKeyPath, tlsConfig.TLSSkipVerify) if err != nil { return nil, err } diff --git a/api/http/proxy/manager.go b/api/http/proxy/manager.go index c1906e45b..2a9018102 100644 --- a/api/http/proxy/manager.go +++ b/api/http/proxy/manager.go @@ -60,7 +60,7 @@ func (manager *Manager) CreateAndRegisterProxy(endpoint *portainer.Endpoint) (ht } if endpointURL.Scheme == "tcp" { - if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify { + if endpoint.TLSConfig.TLS { proxy, err = manager.proxyFactory.newDockerHTTPSProxy(endpointURL, &endpoint.TLSConfig, enableSignature) if err != nil { return nil, err diff --git a/api/ldap/ldap.go b/api/ldap/ldap.go index 3ad222d05..7b72b8930 100644 --- a/api/ldap/ldap.go +++ b/api/ldap/ldap.go @@ -55,7 +55,7 @@ func searchUser(username string, conn *ldap.Conn, settings []portainer.LDAPSearc func createConnection(settings *portainer.LDAPSettings) (*ldap.Conn, error) { if settings.TLSConfig.TLS || settings.StartTLS { - config, err := crypto.CreateTLSConfiguration(&settings.TLSConfig) + config, err := crypto.CreateTLSConfigurationFromDisk(settings.TLSConfig.TLSCACertPath, settings.TLSConfig.TLSCertPath, settings.TLSConfig.TLSKeyPath, settings.TLSConfig.TLSSkipVerify) if err != nil { return nil, err } diff --git a/api/portainer.go b/api/portainer.go index c502a5b15..a8eb55445 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -16,14 +16,14 @@ type ( AdminPasswordFile *string Assets *string Data *string - Endpoint *string + EndpointURL *string ExternalEndpoints *string Labels *[]Pair Logo *string NoAuth *bool NoAnalytics *bool Templates *string - TLSVerify *bool + TLS *bool TLSSkipVerify *bool TLSCacert *string TLSCert *string @@ -445,7 +445,7 @@ const ( // APIVersion is the version number of the Portainer API. APIVersion = "1.17.0" // DBVersion is the version number of the Portainer database. - DBVersion = 10 + DBVersion = 11 // DefaultTemplatesURL represents the default URL for the templates definitions. DefaultTemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates.json" // PortainerAgentHeader represents the name of the header available in any agent response From 19c3fa276b8e900197a2cccd91b646c512fc8026 Mon Sep 17 00:00:00 2001 From: kirdia Date: Mon, 21 May 2018 11:51:56 +0300 Subject: [PATCH 13/15] feat(log-viewer): Add the ability to specify displayed line count (#1914) --- app/docker/components/log-viewer/log-viewer.js | 3 ++- app/docker/components/log-viewer/logViewer.html | 8 ++++++++ app/docker/views/containers/logs/containerlogs.html | 2 +- app/docker/views/services/logs/servicelogs.html | 2 +- app/docker/views/tasks/logs/tasklogs.html | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/docker/components/log-viewer/log-viewer.js b/app/docker/components/log-viewer/log-viewer.js index 3788cf2be..070567b9e 100644 --- a/app/docker/components/log-viewer/log-viewer.js +++ b/app/docker/components/log-viewer/log-viewer.js @@ -4,6 +4,7 @@ angular.module('portainer.docker').component('logViewer', { bindings: { data: '=', displayTimestamps: '=', - logCollectionChange: '<' + logCollectionChange: '<', + lineCount: '=' } }); diff --git a/app/docker/components/log-viewer/logViewer.html b/app/docker/components/log-viewer/logViewer.html index 996bf5e8d..6f6140d4f 100644 --- a/app/docker/components/log-viewer/logViewer.html +++ b/app/docker/components/log-viewer/logViewer.html @@ -33,6 +33,14 @@ +
    + +
    + +
    +