1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-08-09 15:55:23 +02:00

fix(build) cleanup build process [EE-5555] (#9027)

* bring release branch build scripts and workflow into line with develop

* make makefile align with develop

* copy pr-security from develop

* copy workflows from develop
This commit is contained in:
Matt Hook 2023-06-01 15:33:13 +12:00 committed by GitHub
parent b63f869cbc
commit 430691afd8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 136 additions and 161 deletions

View file

@ -11,5 +11,5 @@ jobs:
with:
CONFLICT_LABEL_NAME: 'has conflicts'
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MAX_RETRIES: 5
WAIT_MS: 5000
MAX_RETRIES: 10
WAIT_MS: 60000

View file

@ -21,13 +21,12 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
with:
node-version: '14'
node-version: '18'
cache: 'yarn'
- uses: actions/setup-go@v3
- uses: actions/setup-go@v4
with:
go-version: 1.19.4
go-version: 1.19.5
- run: yarn --frozen-lockfile
- name: Run linters
uses: wearerequired/lint-action@v1
with:
@ -44,4 +43,4 @@ jobs:
with:
version: latest
working-directory: api
args: -c .golangci.yaml
args: --timeout=10m -c .golangci.yaml

View file

@ -2,21 +2,22 @@ name: Nightly Code Security Scan
on:
schedule:
- cron: '0 8 * * *'
- cron: '0 20 * * *'
workflow_dispatch:
jobs:
client-dependencies:
name: Client dependency check
name: Client Dependency Check
runs-on: ubuntu-latest
if: >- # only run for develop branch
github.ref == 'refs/heads/develop'
outputs:
js: ${{ steps.set-matrix.outputs.js_result }}
steps:
- uses: actions/checkout@master
- name: checkout repository
uses: actions/checkout@master
- name: Run Snyk to check for vulnerabilities
- name: scan vulnerabilities by Snyk
uses: snyk/actions/node@master
continue-on-error: true # To make sure that artifact upload gets called
env:
@ -24,46 +25,48 @@ jobs:
with:
json: true
- name: Upload js security scan result as artifact
- name: upload scan result as develop artifact
uses: actions/upload-artifact@v3
with:
name: js-security-scan-develop-result
path: snyk.json
- name: Export scan result to html file
- name: develop scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/js-result")
- name: Upload js result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-js-result-${{github.run_id}}
path: js-result.html
- name: Analyse the js result
- name: analyse vulnerabilities
id: set-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
echo "::set-output name=js_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
echo "js_result=${result}" >> $GITHUB_OUTPUT
server-dependencies:
name: Server dependency check
name: Server Dependency Check
runs-on: ubuntu-latest
if: >- # only run for develop branch
github.ref == 'refs/heads/develop'
outputs:
go: ${{ steps.set-matrix.outputs.go_result }}
steps:
- uses: actions/checkout@master
- name: checkout repository
uses: actions/checkout@master
- uses: actions/setup-go@v3
- name: install Go
uses: actions/setup-go@v3
with:
go-version: '1.19.4'
go-version: '1.19.5'
- name: Download go modules
- name: download Go modules
run: cd ./api && go get -t -v -d ./...
- name: Run Snyk to check for vulnerabilities
- name: scan vulnerabilities by Snyk
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
@ -71,100 +74,66 @@ jobs:
yarn global add snyk
snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
- name: Upload go security scan result as artifact
- name: upload scan result as develop artifact
uses: actions/upload-artifact@v3
with:
name: go-security-scan-develop-result
path: snyk.json
- name: Export scan result to html file
- name: develop scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=table --export --export-filename="/data/go-result")
- name: Upload go result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-go-result-${{github.run_id}}
path: go-result.html
- name: Analyse the go result
- name: analyse vulnerabilities
id: set-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
echo "::set-output name=go_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=snyk --path="/data/snyk.json" --output-type=matrix)
echo "go_result=${result}" >> $GITHUB_OUTPUT
image-vulnerability:
name: Build docker image and Image vulnerability check
name: Image Vulnerability Check
runs-on: ubuntu-latest
if: >-
github.ref == 'refs/heads/develop'
outputs:
image: ${{ steps.set-matrix.outputs.image_result }}
steps:
- name: Checkout code
uses: actions/checkout@master
- name: Use golang 1.19.4
uses: actions/setup-go@v3
with:
go-version: '1.19.4'
- name: Use Node.js 18.x
uses: actions/setup-node@v1
with:
node-version: 18.x
- name: Install packages
run: yarn --frozen-lockfile
- name: build
run: make build
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
file: build/linux/Dockerfile
tags: trivy-portainer:${{ github.sha }}
outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
- name: Load docker image
run: |
docker load --input /tmp/trivy-portainer-image.tar
- name: Run Trivy vulnerability scanner
- name: scan vulnerabilities by Trivy
uses: docker://docker.io/aquasec/trivy:latest
continue-on-error: true
with:
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
- name: Upload image security scan result as artifact
- name: upload image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-develop-result
path: image-trivy.json
- name: Export scan result to html file
- name: develop scan report export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=table --export --export-filename="/data/image-result")
- name: Upload go result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-${{github.run_id}}
path: image-result.html
- name: Analyse the trivy result
- name: analyse vulnerabilities
id: set-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
echo "::set-output name=image_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest summary --report-type=trivy --path="/data/image-trivy.json" --output-type=matrix)
echo "image_result=${result}" >> $GITHUB_OUTPUT
result-analysis:
name: Analyse scan result
name: Analyse Scan Results
needs: [client-dependencies, server-dependencies, image-vulnerability]
runs-on: ubuntu-latest
if: >-
@ -175,21 +144,21 @@ jobs:
go: ${{fromJson(needs.server-dependencies.outputs.go)}}
image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
steps:
- name: Display the results of js, go and image
- name: display the results of js, Go, and image scan
run: |
echo ${{ matrix.js.status }}
echo ${{ matrix.go.status }}
echo ${{ matrix.image.status }}
echo ${{ matrix.js.summary }}
echo ${{ matrix.go.summary }}
echo ${{ matrix.image.summary }}
echo "${{ matrix.js.status }}"
echo "${{ matrix.go.status }}"
echo "${{ matrix.image.status }}"
echo "${{ matrix.js.summary }}"
echo "${{ matrix.go.summary }}"
echo "${{ matrix.image.summary }}"
- name: Send Slack message
if: >-
- name: send message to Slack
if: >-
matrix.js.status == 'failure' ||
matrix.go.status == 'failure' ||
matrix.image.status == 'failure'
uses: slackapi/slack-github-action@v1.18.0
uses: slackapi/slack-github-action@v1.23.0
with:
payload: |
{

View file

@ -12,10 +12,11 @@ on:
- 'build/linux/Dockerfile'
- 'build/linux/alpine.Dockerfile'
- 'build/windows/Dockerfile'
- '.github/workflows/pr-security.yml'
jobs:
client-dependencies:
name: Client dependency check
name: Client Dependency Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
@ -23,9 +24,10 @@ jobs:
outputs:
jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
steps:
- uses: actions/checkout@master
- name: checkout repository
uses: actions/checkout@master
- name: Run Snyk to check for vulnerabilities
- name: scan vulnerabilities by Snyk
uses: snyk/actions/node@master
continue-on-error: true # To make sure that artifact upload gets called
env:
@ -33,13 +35,13 @@ jobs:
with:
json: true
- name: Upload js security scan result as artifact
- name: upload scan result as pull-request artifact
uses: actions/upload-artifact@v3
with:
name: js-security-scan-feat-result
path: snyk.json
- name: Download artifacts from develop branch
- name: download artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@ -51,24 +53,24 @@ jobs:
echo "null" > ./js-snyk-develop.json
fi
- name: Export scan result to html file
- name: pr vs develop scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=table --export --export-filename="/data/js-result")
- name: Upload js result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-js-result-compare-to-develop-${{github.run_id}}
path: js-result.html
- name: Analyse the js diff result
- name: analyse different vulnerabilities against develop branch
id: set-diff-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
echo "::set-output name=js_diff_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/js-snyk-feature.json" --compare-to="/data/js-snyk-develop.json" --output-type=matrix)
echo "js_diff_result=${result}" >> $GITHUB_OUTPUT
server-dependencies:
name: Server dependency check
name: Server Dependency Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
@ -76,16 +78,18 @@ jobs:
outputs:
godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
steps:
- uses: actions/checkout@master
- name: checkout repository
uses: actions/checkout@master
- uses: actions/setup-go@v3
- name: install Go
uses: actions/setup-go@v3
with:
go-version: '1.19.4'
go-version: '1.19.5'
- name: Download go modules
- name: download Go modules
run: cd ./api && go get -t -v -d ./...
- name: Run Snyk to check for vulnerabilities
- name: scan vulnerabilities by Snyk
continue-on-error: true # To make sure that artifact upload gets called
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
@ -93,13 +97,13 @@ jobs:
yarn global add snyk
snyk test --file=./api/go.mod --json-file-output=snyk.json 2>/dev/null || :
- name: Upload go security scan result as artifact
- name: upload scan result as pull-request artifact
uses: actions/upload-artifact@v3
with:
name: go-security-scan-feature-result
path: snyk.json
- name: Download artifacts from develop branch
- name: download artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@ -111,24 +115,24 @@ jobs:
echo "null" > ./go-snyk-develop.json
fi
- name: Export scan result to html file
- name: pr vs develop scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=table --export --export-filename="/data/go-result")
- name: Upload go result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-go-result-compare-to-develop-${{github.run_id}}
path: go-result.html
- name: Analyse the go diff result
- name: analyse different vulnerabilities against develop branch
id: set-diff-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
echo "::set-output name=go_diff_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=snyk --path="/data/go-snyk-feature.json" --compare-to="/data/go-snyk-develop.json" --output-type=matrix)
echo "go_diff_result=${result}" >> $GITHUB_OUTPUT
image-vulnerability:
name: Build docker image and Image vulnerability check
name: Image Vulnerability Check
runs-on: ubuntu-latest
if: >-
github.event.pull_request &&
@ -136,16 +140,16 @@ jobs:
outputs:
imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
steps:
- name: Checkout code
- name: checkout code
uses: actions/checkout@master
- name: Use golang 1.19.4
- name: install Go 1.19.5
uses: actions/setup-go@v3
with:
go-version: '1.19.4'
go-version: '1.19.5'
- name: Use Node.js 18.x
uses: actions/setup-node@v1
- name: install Node.js 18.x
uses: actions/setup-node@v3
with:
node-version: 18.x
@ -153,36 +157,36 @@ jobs:
run: yarn --frozen-lockfile
- name: build
run: make build
run: make build-all
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: set up docker buildx
uses: docker/setup-buildx-action@v2
- name: Build and push
uses: docker/build-push-action@v2
- name: build and compress image
uses: docker/build-push-action@v4
with:
context: .
file: build/linux/Dockerfile
tags: trivy-portainer:${{ github.sha }}
outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
- name: Load docker image
- name: load docker image
run: |
docker load --input /tmp/trivy-portainer-image.tar
- name: Run Trivy vulnerability scanner
- name: scan vulnerabilities by Trivy
uses: docker://docker.io/aquasec/trivy:latest
continue-on-error: true
with:
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}
- name: Upload image security scan result as artifact
- name: upload image security scan result as artifact
uses: actions/upload-artifact@v3
with:
name: image-security-scan-feature-result
path: image-trivy.json
- name: Download artifacts from develop branch
- name: download artifacts from develop branch built by nightly scan
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
@ -194,24 +198,24 @@ jobs:
echo "null" > ./image-trivy-develop.json
fi
- name: Export scan result to html file
- name: pr vs develop scan report comparison export to html
run: |
$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=table --export --export-filename="/data/image-result")
- name: Upload image result html file
- name: upload html file as artifact
uses: actions/upload-artifact@v3
with:
name: html-image-result-compare-to-develop-${{github.run_id}}
path: image-result.html
- name: Analyse the image diff result
- name: analyse different vulnerabilities against develop branch
id: set-diff-matrix
run: |
result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
echo "::set-output name=image_diff_result::${result}"
result=$(docker run --rm -v ${{ github.workspace }}:/data portainerci/code-security-report:latest diff --report-type=trivy --path="/data/image-trivy-feature.json" --compare-to="/data/image-trivy-develop.json" --output-type=matrix)
echo "image_diff_result=${result}" >> $GITHUB_OUTPUT
result-analysis:
name: Analyse scan result compared to develop
name: Analyse Scan Result Against develop Branch
needs: [client-dependencies, server-dependencies, image-vulnerability]
runs-on: ubuntu-latest
if: >-
@ -223,16 +227,16 @@ jobs:
godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
steps:
- name: Check job status of diff result
- name: check job status of diff result
if: >-
matrix.jsdiff.status == 'failure' ||
matrix.godiff.status == 'failure' ||
matrix.imagediff.status == 'failure'
run: |
echo ${{ matrix.jsdiff.status }}
echo ${{ matrix.godiff.status }}
echo ${{ matrix.imagediff.status }}
echo ${{ matrix.jsdiff.summary }}
echo ${{ matrix.godiff.summary }}
echo ${{ matrix.imagediff.summary }}
echo "${{ matrix.jsdiff.status }}"
echo "${{ matrix.godiff.status }}"
echo "${{ matrix.imagediff.status }}"
echo "${{ matrix.jsdiff.summary }}"
echo "${{ matrix.godiff.summary }}"
echo "${{ matrix.imagediff.summary }}"
exit 1

View file

@ -91,7 +91,15 @@ Then build and run the project in a Docker container:
$ yarn start
```
Portainer can now be accessed at <https://localhost:9443>.
Portainer server can now be accessed at <https://localhost:9443>. and UI dev server runs on <http://localhost:8999>.
if you want to build the project you can run:
```sh
make build-all
```
For additional make commands, run `make help`.
Find more detailed steps at <https://docs.portainer.io/contribute/build>.

View file

@ -21,29 +21,29 @@ GOTESTSUM=go run gotest.tools/gotestsum@latest
init-dist:
@mkdir -p dist
build: build-server build-client ## Build the server and client
build-all: deps build-server build-client ## Build the client, server and download external dependancies (doesn't build an image)
build-client: init-dist client-deps ## Build the client
build-client: init-dist ## Build the client
export NODE_ENV=$(ENV) && yarn build --config $(WEBPACK_CONFIG)
build-server: init-dist ## Build the server binary
./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
build-image: build ## Build the Portainer image locally
build-image: build-all ## Build the Portainer image locally
docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
devops: clean init-dist server-deps build-client ## Build the server binary for CI
build-storybook: ## Build and serve the storybook files
yarn storybook:build
devops: clean deps build-client ## Build the everything target specifically for CI
echo "Building the devops binary..."
@./build/build_binary_azuredevops.sh "$(PLATFORM)" "$(ARCH)"
build-storybook:
yarn storybook:build
##@ Build dependencies
.PHONY: deps server-deps client-deps tidy
deps-all: server-deps client-deps ## Download all client and server build dependancies
deps: server-deps client-deps ## Download all client and server build dependancies
server-deps: ## Download dependant server binaries
server-deps: init-dist ## Download dependant server binaries
@./build/download_binaries.sh $(PLATFORM) $(ARCH)
client-deps: ## Install client dependencies
@ -79,7 +79,7 @@ dev: ## Run both the client and server in development mode
dev-client: ## Run the client in development mode
yarn dev
dev-server: ## Run the server in development mode
dev-server: build-server ## Run the server in development mode
@./dev/run_container.sh
@ -113,18 +113,12 @@ dev-extension: build-server build-client ## Run the extension in development mod
##@ Docs
.PHONY: docs-build docs-validate docs-clean docs-validate-clean
docs-build: ## Build docs
cd api && $(SWAG) init -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
docs-build: init-dist ## Build docs
cd api && $(SWAG) init -o "../dist/docs" -ot "yaml" -g ./http/handler/handler.go --parseDependency --parseInternal --parseDepth 2 --markdownFiles ./
docs-validate: docs-build ## Validate docs
yarn swagger2openapi --warnOnly api/docs/swagger.yaml -o api/docs/openapi.yaml
yarn swagger-cli validate api/docs/openapi.yaml
docs-clean: ## Clean docs
rm -rf api/docs
docs-validate-clean: docs-validate docs-clean ## Validate and clean docs
yarn swagger2openapi --warnOnly dist/docs/swagger.yaml -o dist/docs/openapi.yaml
yarn swagger-cli validate dist/docs/openapi.yaml
##@ Helpers
.PHONY: help

View file

@ -19,5 +19,6 @@ docker run -d \
-v /var/run/docker.sock:/var/run/alternative.sock:z \
-v /tmp:/tmp \
--name portainer \
--rm \
portainer/base \
/app/portainer "${@:PORTAINER_FLAGS}"