diff --git a/api/git/azure.go b/api/git/azure.go index 72d35802c..03a531dc2 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -15,8 +15,6 @@ import ( "github.com/portainer/portainer/api/crypto" gittypes "github.com/portainer/portainer/api/git/types" - "github.com/go-git/go-git/v5/plumbing/transport/client" - githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/pkg/errors" ) @@ -51,21 +49,22 @@ type azureItem struct { } type azureClient struct { - client *http.Client baseUrl string } func NewAzureClient() *azureClient { - httpsCli := newHttpClientForAzure() return &azureClient{ - client: httpsCli, baseUrl: "https://dev.azure.com", } } -func newHttpClientForAzure() *http.Client { +func newHttpClientForAzure(insecureSkipVerify bool) *http.Client { tlsConfig := crypto.CreateTLSConfiguration() + if insecureSkipVerify { + tlsConfig.InsecureSkipVerify = true + } + httpsCli := &http.Client{ Transport: &http.Transport{ TLSClientConfig: tlsConfig, @@ -74,8 +73,6 @@ func newHttpClientForAzure() *http.Client { Timeout: 300 * time.Second, } - client.InstallProtocol("https", githttp.NewClient(httpsCli)) - return httpsCli } @@ -109,6 +106,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO if err != nil { return "", errors.WithMessage(err, "failed to create temp file") } + defer zipFile.Close() req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil) @@ -122,10 +120,14 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO return "", errors.WithMessage(err, "failed to create a new HTTP request") } - res, err := a.client.Do(req) + client := newHttpClientForAzure(opt.tlsSkipVerify) + defer client.CloseIdleConnections() + + res, err := client.Do(req) if err != nil { return "", errors.WithMessage(err, "failed to make an HTTP request") } + defer res.Body.Close() if res.StatusCode != http.StatusOK { @@ -171,7 +173,10 @@ func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureI return nil, errors.WithMessage(err, "failed to create a new HTTP request") } - resp, err := a.client.Do(req) + client := newHttpClientForAzure(opt.tlsSkipVerify) + defer client.CloseIdleConnections() + + resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } @@ -410,7 +415,10 @@ func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, e return nil, errors.WithMessage(err, "failed to create a new HTTP request") } - resp, err := a.client.Do(req) + client := newHttpClientForAzure(opt.tlsSkipVerify) + defer client.CloseIdleConnections() + + resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } @@ -467,7 +475,10 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, return nil, errors.WithMessage(err, "failed to create a new HTTP request") } - resp, err := a.client.Do(req) + client := newHttpClientForAzure(opt.tlsSkipVerify) + defer client.CloseIdleConnections() + + resp, err := client.Do(req) if err != nil { return nil, errors.WithMessage(err, "failed to make an HTTP request") } diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index f5aba3218..214acc31d 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -58,7 +58,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) { t.Run(tt.name, func(t *testing.T) { dst := t.TempDir() repositoryUrl := fmt.Sprintf(tt.args.repositoryURLFormat, tt.args.password) - err := service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "") + err := service.CloneRepository(dst, repositoryUrl, tt.args.referenceName, "", "", false) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) }) @@ -73,7 +73,7 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { dst := t.TempDir() - err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat) + err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat, false) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } @@ -84,7 +84,7 @@ func TestService_LatestCommitID_Azure(t *testing.T) { pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") service := NewService(context.TODO()) - id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat) + id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat, false) assert.NoError(t, err) assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") } @@ -96,7 +96,7 @@ func TestService_ListRefs_Azure(t *testing.T) { username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") service := NewService(context.TODO()) - refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false) + refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(refs), 1) } @@ -108,8 +108,8 @@ func TestService_ListRefs_Azure_Concurrently(t *testing.T) { username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) - go service.ListRefs(privateAzureRepoURL, username, accessToken, false) - service.ListRefs(privateAzureRepoURL, username, accessToken, false) + go service.ListRefs(privateAzureRepoURL, username, accessToken, false, false) + service.ListRefs(privateAzureRepoURL, username, accessToken, false, false) time.Sleep(2 * time.Second) } @@ -247,7 +247,7 @@ func TestService_ListFiles_Azure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions) + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false) if tt.expect.shouldFail { assert.Error(t, err) if tt.expect.err != nil { @@ -270,8 +270,8 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) { username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) - go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}) - service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}) + go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false) + service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{}, false) time.Sleep(2 * time.Second) } diff --git a/api/git/azure_test.go b/api/git/azure_test.go index bc80d1de4..2cb073c59 100644 --- a/api/git/azure_test.go +++ b/api/git/azure_test.go @@ -292,7 +292,6 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { defer server.Close() a := &azureClient{ - client: server.Client(), baseUrl: server.URL, } @@ -329,7 +328,6 @@ func Test_azureDownloader_latestCommitID(t *testing.T) { defer server.Close() a := &azureClient{ - client: server.Client(), baseUrl: server.URL, } @@ -442,6 +440,7 @@ func Test_listRefs_azure(t *testing.T) { accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT") username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") + tests := []struct { name string args baseOption diff --git a/api/git/backup.go b/api/git/backup.go index 90fef3fb8..df1946c0d 100644 --- a/api/git/backup.go +++ b/api/git/backup.go @@ -20,6 +20,8 @@ type CloneOptions struct { ReferenceName string Username string Password string + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } func CloneWithBackup(gitService portainer.GitService, fileService portainer.FileService, options CloneOptions) (clean func(), err error) { @@ -43,7 +45,7 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File cleanUp = true - err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password) + err = gitService.CloneRepository(options.ProjectPath, options.URL, options.ReferenceName, options.Username, options.Password, options.TLSSkipVerify) if err != nil { cleanUp = false restoreError := filesystem.MoveDirectory(backupProjectPath, options.ProjectPath) diff --git a/api/git/git.go b/api/git/git.go index 436e3d2fe..7024deaf8 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -28,9 +28,10 @@ func NewGitClient(preserveGitDir bool) *gitClient { func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error { gitOptions := git.CloneOptions{ - URL: opt.repositoryUrl, - Depth: opt.depth, - Auth: getAuth(opt.username, opt.password), + URL: opt.repositoryUrl, + Depth: opt.depth, + InsecureSkipTLS: opt.tlsSkipVerify, + Auth: getAuth(opt.username, opt.password), } if opt.referenceName != "" { @@ -60,7 +61,8 @@ func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string }) listOptions := &git.ListOptions{ - Auth: getAuth(opt.username, opt.password), + Auth: getAuth(opt.username, opt.password), + InsecureSkipTLS: opt.tlsSkipVerify, } refs, err := remote.List(listOptions) @@ -110,7 +112,8 @@ func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, err }) listOptions := &git.ListOptions{ - Auth: getAuth(opt.username, opt.password), + Auth: getAuth(opt.username, opt.password), + InsecureSkipTLS: opt.tlsSkipVerify, } refs, err := rem.List(listOptions) @@ -132,12 +135,13 @@ func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, err // listFiles list all filenames under the specific repository func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { cloneOption := &git.CloneOptions{ - URL: opt.repositoryUrl, - NoCheckout: true, - Depth: 1, - SingleBranch: true, - ReferenceName: plumbing.ReferenceName(opt.referenceName), - Auth: getAuth(opt.username, opt.password), + URL: opt.repositoryUrl, + NoCheckout: true, + Depth: 1, + SingleBranch: true, + ReferenceName: plumbing.ReferenceName(opt.referenceName), + Auth: getAuth(opt.username, opt.password), + InsecureSkipTLS: opt.tlsSkipVerify, } repo, err := git.Clone(memory.NewStorage(), nil, cloneOption) diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index 6c72532dc..7b5bda468 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -24,7 +24,7 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) { dst := t.TempDir() repositoryUrl := privateGitRepoURL - err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken) + err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken, false) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } @@ -37,7 +37,7 @@ func TestService_LatestCommitID_GitHub(t *testing.T) { service := newService(context.TODO(), 0, 0) repositoryUrl := privateGitRepoURL - id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken) + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken, false) assert.NoError(t, err) assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") } @@ -50,7 +50,7 @@ func TestService_ListRefs_GitHub(t *testing.T) { service := newService(context.TODO(), 0, 0) repositoryUrl := privateGitRepoURL - refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(refs), 1) } @@ -63,8 +63,8 @@ func TestService_ListRefs_Github_Concurrently(t *testing.T) { service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) repositoryUrl := privateGitRepoURL - go service.ListRefs(repositoryUrl, username, accessToken, false) - service.ListRefs(repositoryUrl, username, accessToken, false) + go service.ListRefs(repositoryUrl, username, accessToken, false, false) + service.ListRefs(repositoryUrl, username, accessToken, false, false) time.Sleep(2 * time.Second) } @@ -202,7 +202,7 @@ func TestService_ListFiles_GitHub(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions) + paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions, false) if tt.expect.shouldFail { assert.Error(t, err) if tt.expect.err != nil { @@ -226,8 +226,8 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) { username := getRequiredValue(t, "GITHUB_USERNAME") service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) - go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) time.Sleep(2 * time.Second) } @@ -240,8 +240,8 @@ func TestService_purgeCache_Github(t *testing.T) { username := getRequiredValue(t, "GITHUB_USERNAME") service := NewService(context.TODO()) - service.ListRefs(repositoryUrl, username, accessToken, false) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + service.ListRefs(repositoryUrl, username, accessToken, false, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoFileCache.Len()) @@ -261,8 +261,8 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) { // 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result service := newService(context.TODO(), 2, 40*timeout) - service.ListRefs(repositoryUrl, username, accessToken, false) - service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + service.ListRefs(repositoryUrl, username, accessToken, false, false) + service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoFileCache.Len()) @@ -293,12 +293,12 @@ func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) { service := newService(context.TODO(), 2, 0) repositoryUrl := privateGitRepoURL - refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(refs), 1) assert.Equal(t, 1, service.repoRefCache.Len()) - refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false) + _, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false) assert.Error(t, err) assert.Equal(t, 1, service.repoRefCache.Len()) } @@ -311,26 +311,26 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) { service := newService(context.TODO(), 2, 0) repositoryUrl := privateGitRepoURL - refs, err := service.ListRefs(repositoryUrl, username, accessToken, false) + refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(refs), 1) assert.Equal(t, 1, service.repoRefCache.Len()) - files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 1, service.repoFileCache.Len()) - files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}) + files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 2, service.repoFileCache.Len()) - refs, err = service.ListRefs(repositoryUrl, username, "fake-token", false) + _, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false) assert.Error(t, err) assert.Equal(t, 1, service.repoRefCache.Len()) - refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true) + _, err = service.ListRefs(repositoryUrl, username, "fake-token", true, false) assert.Error(t, err) assert.Equal(t, 1, service.repoRefCache.Len()) // The relevant file caches should be removed too @@ -344,12 +344,12 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) { accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") repositoryUrl := privateGitRepoURL - files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}) + files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{}, false) assert.NoError(t, err) assert.GreaterOrEqual(t, len(files), 1) assert.Equal(t, 1, service.repoFileCache.Len()) - files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}) + _, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{}, false) assert.Error(t, err) assert.Equal(t, 0, service.repoFileCache.Len()) } diff --git a/api/git/git_test.go b/api/git/git_test.go index 0e2d6caeb..fe461ad2f 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -38,7 +38,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) { dir := t.TempDir() t.Logf("Cloning into %s", dir) - err := service.CloneRepository(dir, repositoryURL, referenceName, "", "") + err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false) assert.NoError(t, err) assert.Equal(t, 1, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") } @@ -50,7 +50,7 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) { dir := t.TempDir() t.Logf("Cloning into %s", dir) - err := service.CloneRepository(dir, repositoryURL, referenceName, "", "") + err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false) assert.NoError(t, err) assert.NoDirExists(t, filepath.Join(dir, ".git")) } @@ -84,7 +84,7 @@ func Test_latestCommitID(t *testing.T) { repositoryURL := setup(t) referenceName := "refs/heads/main" - id, err := service.LatestCommitID(repositoryURL, referenceName, "", "") + id, err := service.LatestCommitID(repositoryURL, referenceName, "", "", false) assert.NoError(t, err) assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) diff --git a/api/git/service.go b/api/git/service.go index 7f062cd7a..6b08e2d19 100644 --- a/api/git/service.go +++ b/api/git/service.go @@ -2,6 +2,7 @@ package git import ( "context" + "strconv" "strings" "sync" "time" @@ -20,6 +21,7 @@ type baseOption struct { repositoryUrl string username string password string + tlsSkipVerify bool } // fetchOption allows to specify the reference name of the target repository @@ -119,13 +121,14 @@ func (service *Service) timerHasStopped() bool { // CloneRepository clones a git repository using the specified URL in the specified // destination folder. -func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { +func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error { options := cloneOption{ fetchOption: fetchOption{ baseOption: baseOption{ repositoryUrl: repositoryURL, username: username, password: password, + tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, }, @@ -144,12 +147,13 @@ func (service *Service) cloneRepository(destination string, options cloneOption) } // LatestCommitID returns SHA1 of the latest commit of the specified reference -func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { +func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) { options := fetchOption{ baseOption: baseOption{ repositoryUrl: repositoryURL, username: username, password: password, + tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, } @@ -162,8 +166,8 @@ func (service *Service) LatestCommitID(repositoryURL, referenceName, username, p } // ListRefs will list target repository's references without cloning the repository -func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { - refCacheKey := generateCacheKey(repositoryURL, password) +func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) { + refCacheKey := generateCacheKey(repositoryURL, username, password, strconv.FormatBool(tlsSkipVerify)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result service.repoRefCache.Remove(refCacheKey) @@ -193,6 +197,7 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR repositoryUrl: repositoryURL, username: username, password: password, + tlsSkipVerify: tlsSkipVerify, } var ( @@ -219,8 +224,8 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR // ListFiles will list all the files of the target repository with specific extensions. // If extension is not provided, it will list all the files under the target repository -func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { - repoKey := generateCacheKey(repositoryURL, referenceName) +func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { + repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify)) if service.cacheEnabled && hardRefresh { // Should remove the cache explicitly, so that the following normal list can show the correct result @@ -246,6 +251,7 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo repositoryUrl: repositoryURL, username: username, password: password, + tlsSkipVerify: tlsSkipVerify, }, referenceName: referenceName, } diff --git a/api/git/types/types.go b/api/git/types/types.go index 21f55a699..12d95e093 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -3,8 +3,8 @@ package gittypes import "errors" var ( - ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.") - ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.") + ErrIncorrectRepositoryURL = errors.New("git repository could not be found, please ensure that the URL is correct") + ErrAuthenticationFailure = errors.New("authentication failed, please ensure that the git credentials are correct") ) // RepoConfig represents a configuration for a repo @@ -19,6 +19,8 @@ type RepoConfig struct { Authentication *GitAuthentication // Repository hash ConfigHash string `example:"bc4c183d756879ea4d173315338110b31004b8e0"` + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } type GitAuthentication struct { diff --git a/api/git/update/update.go b/api/git/update/update.go index e0c23eefd..3b81fae4a 100644 --- a/api/git/update/update.go +++ b/api/git/update/update.go @@ -6,14 +6,13 @@ import ( "github.com/pkg/errors" portainer "github.com/portainer/portainer/api" - "github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/git" gittypes "github.com/portainer/portainer/api/git/types" "github.com/rs/zerolog/log" ) // UpdateGitObject updates a git object based on its config -func UpdateGitObject(gitService portainer.GitService, dataStore dataservices.DataStore, objId string, gitConfig *gittypes.RepoConfig, autoUpdateConfig *portainer.AutoUpdateSettings, projectPath string) (bool, string, error) { +func UpdateGitObject(gitService portainer.GitService, objId string, gitConfig *gittypes.RepoConfig, forceUpdate bool, projectPath string) (bool, string, error) { if gitConfig == nil { return false, "", nil } @@ -29,13 +28,13 @@ func UpdateGitObject(gitService portainer.GitService, dataStore dataservices.Dat return false, "", errors.WithMessagef(err, "failed to get credentials for %v", objId) } - newHash, err := gitService.LatestCommitID(gitConfig.URL, gitConfig.ReferenceName, username, password) + newHash, err := gitService.LatestCommitID(gitConfig.URL, gitConfig.ReferenceName, username, password, gitConfig.TLSSkipVerify) if err != nil { return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId) } hashChanged := !strings.EqualFold(newHash, gitConfig.ConfigHash) - forceUpdate := autoUpdateConfig != nil && autoUpdateConfig.ForceUpdate + if !hashChanged && !forceUpdate { log.Debug(). Str("hash", newHash). @@ -48,9 +47,10 @@ func UpdateGitObject(gitService portainer.GitService, dataStore dataservices.Dat } cloneParams := &cloneRepositoryParameters{ - url: gitConfig.URL, - ref: gitConfig.ReferenceName, - toDir: projectPath, + url: gitConfig.URL, + ref: gitConfig.ReferenceName, + toDir: projectPath, + tlsSkipVerify: gitConfig.TLSSkipVerify, } if gitConfig.Authentication != nil { cloneParams.auth = &gitAuth{ @@ -78,6 +78,8 @@ type cloneRepositoryParameters struct { ref string toDir string auth *gitAuth + // tlsSkipVerify skips SSL verification when cloning the Git repository + tlsSkipVerify bool `example:"false"` } type gitAuth struct { @@ -87,8 +89,8 @@ type gitAuth struct { func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error { if cloneParams.auth != nil { - return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password) + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password, cloneParams.tlsSkipVerify) } - return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "") + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "", cloneParams.tlsSkipVerify) } diff --git a/api/http/handler/customtemplates/customtemplate_create.go b/api/http/handler/customtemplates/customtemplate_create.go index d8fdb3865..e0d273efc 100644 --- a/api/http/handler/customtemplates/customtemplate_create.go +++ b/api/http/handler/customtemplates/customtemplate_create.go @@ -213,6 +213,8 @@ type customTemplateFromGitRepositoryPayload struct { ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` // Definitions of variables in the stack file Variables []portainer.CustomTemplateVariableDefinition + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -279,7 +281,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( repositoryPassword = "" } - err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword, payload.TLSSkipVerify) if err != nil { if err == gittypes.ErrAuthenticationFailure { return nil, fmt.Errorf("invalid git credential") diff --git a/api/http/handler/edgestacks/edgestack_create.go b/api/http/handler/edgestacks/edgestack_create.go index bc5beca5a..aa707384e 100644 --- a/api/http/handler/edgestacks/edgestack_create.go +++ b/api/http/handler/edgestacks/edgestack_create.go @@ -201,6 +201,8 @@ type swarmStackFromGitRepositoryPayload struct { Registries []portainer.RegistryID // Uses the manifest's namespaces instead of the default one UseManifestNamespaces bool + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -247,6 +249,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request, dryru URL: payload.RepositoryURL, ReferenceName: payload.RepositoryReferenceName, ConfigFilePath: payload.FilePathInRepository, + TLSSkipVerify: payload.TLSSkipVerify, } if payload.RepositoryAuthentication { @@ -345,7 +348,7 @@ func (handler *Handler) storeManifestFromGitRepository(stackFolder string, relat repositoryPassword = repositoryConfig.Authentication.Password } - err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword) + err = handler.GitService.CloneRepository(projectPath, repositoryConfig.URL, repositoryConfig.ReferenceName, repositoryUsername, repositoryPassword, repositoryConfig.TLSSkipVerify) if err != nil { return "", "", "", err } diff --git a/api/http/handler/edgestacks/edgestack_test.go b/api/http/handler/edgestacks/edgestack_test.go index 3b01ece5e..2036ccc70 100644 --- a/api/http/handler/edgestacks/edgestack_test.go +++ b/api/http/handler/edgestacks/edgestack_test.go @@ -18,32 +18,12 @@ import ( "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/edge/edgestacks" + "github.com/portainer/portainer/api/internal/testhelpers" "github.com/portainer/portainer/api/jwt" "github.com/pkg/errors" ) -type gitService struct { - cloneErr error - id string -} - -func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { - return g.cloneErr -} - -func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - return g.id, nil -} - -func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { - return nil, nil -} - -func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { - return nil, nil -} - // Helpers func setupHandler(t *testing.T) (*Handler, string, func()) { t.Helper() @@ -98,7 +78,7 @@ func setupHandler(t *testing.T) (*Handler, string, func()) { t.Fatal(err) } - handler.GitService = &gitService{errors.New("Clone error"), "git-service-id"} + handler.GitService = testhelpers.NewGitService(errors.New("Clone error"), "git-service-id") return handler, rawAPIKey, storeTeardown } diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index 9cf7e304e..912da7761 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -162,9 +162,11 @@ type composeStackFromGitRepositoryPayload struct { Env []portainer.Pair // Whether the stack is from a app template FromAppTemplate bool `example:"false"` + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } -func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload { +func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool, repoSkipSSLVerify bool) stackbuilders.StackPayload { return stackbuilders.StackPayload{ Name: name, RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{ @@ -173,6 +175,7 @@ func createStackPayloadFromComposeGitPayload(name, repoUrl, repoReference, repoU Authentication: repoAuthentication, Username: repoUsername, Password: repoPassword, + TLSSkipVerify: repoSkipSSLVerify, }, ComposeFile: composeFile, AdditionalFiles: additionalFiles, @@ -258,7 +261,9 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite payload.AdditionalFiles, payload.AutoUpdate, payload.Env, - payload.FromAppTemplate) + payload.FromAppTemplate, + payload.TLSSkipVerify, + ) composeStackBuilder := stackbuilders.CreateComposeStackGitBuilder(securityContext, handler.DataStore, diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index c3db2829c..da20789db 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -46,9 +46,11 @@ type kubernetesGitDeploymentPayload struct { ManifestFile string AdditionalFiles []string AutoUpdate *portainer.AutoUpdateSettings + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } -func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings) stackbuilders.StackPayload { +func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication, composeFormat bool, namespace, manifest string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, repoSkipSSLVerify bool) stackbuilders.StackPayload { return stackbuilders.StackPayload{ StackName: name, RepositoryConfigPayload: stackbuilders.RepositoryConfigPayload{ @@ -57,6 +59,7 @@ func createStackPayloadFromK8sGitPayload(name, repoUrl, repoReference, repoUsern Authentication: repoAuthentication, Username: repoUsername, Password: repoPassword, + TLSSkipVerify: repoSkipSSLVerify, }, Namespace: namespace, ComposeFormat: composeFormat, @@ -203,7 +206,9 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr payload.Namespace, payload.ManifestFile, payload.AdditionalFiles, - payload.AutoUpdate) + payload.AutoUpdate, + payload.TLSSkipVerify, + ) k8sStackBuilder := stackbuilders.CreateKubernetesStackGitBuilder(handler.DataStore, handler.FileService, diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index bcc71e14e..031c518c9 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -117,6 +117,8 @@ type swarmStackFromGitRepositoryPayload struct { AdditionalFiles []string `example:"[nz.compose.yml, uat.compose.yml]"` // Optional auto update configuration AutoUpdate *portainer.AutoUpdateSettings + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -138,7 +140,7 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err return nil } -func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool) stackbuilders.StackPayload { +func createStackPayloadFromSwarmGitPayload(name, swarmID, repoUrl, repoReference, repoUsername, repoPassword string, repoAuthentication bool, composeFile string, additionalFiles []string, autoUpdate *portainer.AutoUpdateSettings, env []portainer.Pair, fromAppTemplate bool, repoSkipSSLVerify bool) stackbuilders.StackPayload { return stackbuilders.StackPayload{ Name: name, SwarmID: swarmID, @@ -201,7 +203,9 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, payload.AdditionalFiles, payload.AutoUpdate, payload.Env, - payload.FromAppTemplate) + payload.FromAppTemplate, + payload.TLSSkipVerify, + ) swarmStackBuilder := stackbuilders.CreateSwarmStackGitBuilder(securityContext, handler.DataStore, diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index e35dd9df0..12bae9bc0 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -158,7 +158,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * Username: payload.RepositoryUsername, Password: password, } - _, err = handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password) + _, err = handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password, stack.GitConfig.TLSSkipVerify) if err != nil { return httperror.InternalServerError("Unable to fetch git repository", err) } diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go index 13e0bcb3f..9c33401b3 100644 --- a/api/http/handler/stacks/stack_update_git_redeploy.go +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -145,7 +145,16 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) repositoryUsername = payload.RepositoryUsername } - clean, err := git.CloneWithBackup(handler.GitService, handler.FileService, git.CloneOptions{ProjectPath: stack.ProjectPath, URL: stack.GitConfig.URL, ReferenceName: stack.GitConfig.ReferenceName, Username: repositoryUsername, Password: repositoryPassword}) + cloneOptions := git.CloneOptions{ + ProjectPath: stack.ProjectPath, + URL: stack.GitConfig.URL, + ReferenceName: stack.GitConfig.ReferenceName, + Username: repositoryUsername, + Password: repositoryPassword, + TLSSkipVerify: stack.GitConfig.TLSSkipVerify, + } + + clean, err := git.CloneWithBackup(handler.GitService, handler.FileService, cloneOptions) if err != nil { return httperror.InternalServerError("Unable to clone git repository directory", err) } @@ -157,7 +166,7 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) return httpErr } - newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword) + newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword, stack.GitConfig.TLSSkipVerify) if err != nil { return httperror.InternalServerError("Unable get latest commit id", errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID)) } diff --git a/api/http/handler/stacks/update_kubernetes_stack.go b/api/http/handler/stacks/update_kubernetes_stack.go index a541f666c..e172a0c33 100644 --- a/api/http/handler/stacks/update_kubernetes_stack.go +++ b/api/http/handler/stacks/update_kubernetes_stack.go @@ -73,7 +73,7 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer. Username: payload.RepositoryUsername, Password: password, } - _, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password) + _, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password, stack.GitConfig.TLSSkipVerify) if err != nil { return httperror.InternalServerError("Unable to fetch git repository", err) } diff --git a/api/http/handler/templates/template_file.go b/api/http/handler/templates/template_file.go index e7e5a23a0..95254dbb6 100644 --- a/api/http/handler/templates/template_file.go +++ b/api/http/handler/templates/template_file.go @@ -100,7 +100,7 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht defer handler.cleanUp(projectPath) - err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "") + err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, "", "", "", false) if err != nil { return httperror.InternalServerError("Unable to clone git repository", err) } diff --git a/api/http/proxy/factory/docker/transport.go b/api/http/proxy/factory/docker/transport.go index 86a252dae..d63883960 100644 --- a/api/http/proxy/factory/docker/transport.go +++ b/api/http/proxy/factory/docker/transport.go @@ -395,7 +395,7 @@ func (transport *Transport) updateDefaultGitBranch(request *http.Request) error remote := request.URL.Query().Get("remote") if strings.HasSuffix(remote, ".git") { repositoryURL := remote[:len(remote)-4] - latestCommitID, err := transport.gitService.LatestCommitID(repositoryURL, "", "", "") + latestCommitID, err := transport.gitService.LatestCommitID(repositoryURL, "", "", "", false) if err != nil { return err } diff --git a/api/http/proxy/factory/docker/transport_test.go b/api/http/proxy/factory/docker/transport_test.go index cff9dd7fe..bc42d5e5a 100644 --- a/api/http/proxy/factory/docker/transport_test.go +++ b/api/http/proxy/factory/docker/transport_test.go @@ -1,29 +1,16 @@ package docker import ( + "fmt" "net/http" "net/http/httptest" "testing" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/testhelpers" "github.com/stretchr/testify/assert" ) -type noopGitService struct{} - -func (s *noopGitService) CloneRepository(destination string, repositoryURL, referenceName, username, password string) error { - return nil -} -func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - return "my-latest-commit-id", nil -} -func (g *noopGitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { - return nil, nil -} -func (g *noopGitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { - return nil, nil -} - func TestTransport_updateDefaultGitBranch(t *testing.T) { type fields struct { gitService portainer.GitService @@ -33,8 +20,10 @@ func TestTransport_updateDefaultGitBranch(t *testing.T) { request *http.Request } + commitId := "my-latest-commit-id" + defaultFields := fields{ - gitService: &noopGitService{}, + gitService: testhelpers.NewGitService(nil, commitId), } tests := []struct { @@ -51,7 +40,7 @@ func TestTransport_updateDefaultGitBranch(t *testing.T) { request: httptest.NewRequest(http.MethodPost, "http://unixsocket/build?dockerfile=Dockerfile&remote=https://my-host.com/my-user/my-repo.git&t=my-image", nil), }, wantErr: false, - expectedQuery: "dockerfile=Dockerfile&remote=https%3A%2F%2Fmy-host.com%2Fmy-user%2Fmy-repo.git%23my-latest-commit-id&t=my-image", + expectedQuery: fmt.Sprintf("dockerfile=Dockerfile&remote=https%%3A%%2F%%2Fmy-host.com%%2Fmy-user%%2Fmy-repo.git%%23%s&t=my-image", commitId), }, { name: "not append commit ID", diff --git a/api/internal/testhelpers/git_service.go b/api/internal/testhelpers/git_service.go index 8dca32cd1..fae9caec6 100644 --- a/api/internal/testhelpers/git_service.go +++ b/api/internal/testhelpers/git_service.go @@ -1,12 +1,32 @@ package testhelpers -type gitService struct{} +import portainer "github.com/portainer/portainer/api" + +type gitService struct { + cloneErr error + id string +} // NewGitService creates new mock for portainer.GitService. -func NewGitService() *gitService { - return &gitService{} +func NewGitService(cloneErr error, id string) portainer.GitService { + return &gitService{ + cloneErr: cloneErr, + id: id, + } } -func (service *gitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string) error { - return nil +func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error { + return g.cloneErr +} + +func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) { + return g.id, nil +} + +func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) { + return nil, nil +} + +func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) { + return nil, nil } diff --git a/api/portainer.go b/api/portainer.go index f816deef1..194d4b7b3 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1395,10 +1395,10 @@ type ( // GitService represents a service for managing Git GitService interface { - CloneRepository(destination string, repositoryURL, referenceName, username, password string) error - LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) - ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) - ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string) ([]string, error) + CloneRepository(destination string, repositoryURL, referenceName, username, password string, tlsSkipVerify bool) error + LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) + ListRefs(repositoryURL, username, password string, hardRefresh bool, tlsSkipVerify bool) ([]string, error) + ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string, tlsSkipVerify bool) ([]string, error) } // OpenAMTService represents a service for managing OpenAMT diff --git a/api/stacks/deployments/deploy.go b/api/stacks/deployments/deploy.go index 292bdf086..32598c1ef 100644 --- a/api/stacks/deployments/deploy.go +++ b/api/stacks/deployments/deploy.go @@ -53,14 +53,14 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data Str("author", author). Str("stack", stack.Name). Int("endpoint_id", int(stack.EndpointID)). - Msg("cannot autoupdate a stack, stack author user is missing") + Msg("cannot auto update a stack, stack author user is missing") return &StackAuthorMissingErr{int(stack.ID), author} } var gitCommitChangedOrForceUpdate bool if !stack.FromAppTemplate { - updated, newHash, err := update.UpdateGitObject(gitService, datastore, fmt.Sprintf("stack:%d", stackID), stack.GitConfig, stack.AutoUpdate, stack.ProjectPath) + updated, newHash, err := update.UpdateGitObject(gitService, fmt.Sprintf("stack:%d", stackID), stack.GitConfig, false, stack.ProjectPath) if err != nil { return err } @@ -99,7 +99,7 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data err := deployer.DeployKubernetesStack(stack, endpoint, user) if err != nil { - return errors.WithMessagef(err, "failed to deploy a kubternetes app stack %v", stackID) + return errors.WithMessagef(err, "failed to deploy a kubernetes app stack %v", stackID) } default: return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) diff --git a/api/stacks/deployments/deploy_test.go b/api/stacks/deployments/deploy_test.go index e644590f7..b012f65b3 100644 --- a/api/stacks/deployments/deploy_test.go +++ b/api/stacks/deployments/deploy_test.go @@ -6,33 +6,13 @@ import ( "testing" "github.com/portainer/portainer/api/datastore" + "github.com/portainer/portainer/api/internal/testhelpers" portainer "github.com/portainer/portainer/api" gittypes "github.com/portainer/portainer/api/git/types" "github.com/stretchr/testify/assert" ) -type gitService struct { - cloneErr error - id string -} - -func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { - return g.cloneErr -} - -func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { - return g.id, nil -} - -func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) { - return nil, nil -} - -func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) { - return nil, nil -} - type noopDeployer struct{} func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error { @@ -67,7 +47,7 @@ func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { err = store.Stack().Create(&portainer.Stack{ID: 1, CreatedBy: "admin"}) assert.NoError(t, err, "failed to create a test stack") - err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) + err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(nil, "")) assert.NoError(t, err) } @@ -97,7 +77,7 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { }}) assert.NoError(t, err, "failed to create a test stack") - err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"}) + err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(nil, "oldHash")) assert.NoError(t, err) } @@ -125,7 +105,7 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { }}) assert.NoError(t, err, "failed to create a test stack") - err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"}) + err = RedeployWhenChanged(1, nil, store, testhelpers.NewGitService(cloneErr, "newHash")) assert.Error(t, err) assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup") } @@ -162,7 +142,7 @@ func Test_redeployWhenChanged(t *testing.T) { stack.Type = portainer.DockerComposeStack store.Stack().UpdateStack(stack.ID, &stack) - err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) assert.NoError(t, err) }) @@ -170,7 +150,7 @@ func Test_redeployWhenChanged(t *testing.T) { stack.Type = portainer.DockerSwarmStack store.Stack().UpdateStack(stack.ID, &stack) - err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) assert.NoError(t, err) }) @@ -178,7 +158,7 @@ func Test_redeployWhenChanged(t *testing.T) { stack.Type = portainer.KubernetesStack store.Stack().UpdateStack(stack.ID, &stack) - err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + err = RedeployWhenChanged(1, &noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash")) assert.NoError(t, err) }) } diff --git a/api/stacks/stackbuilders/stack_git_builder.go b/api/stacks/stackbuilders/stack_git_builder.go index 3b47595b6..099b4c873 100644 --- a/api/stacks/stackbuilders/stack_git_builder.go +++ b/api/stacks/stackbuilders/stack_git_builder.go @@ -67,6 +67,8 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho repoConfig.URL = payload.URL repoConfig.ReferenceName = payload.ReferenceName + repoConfig.TLSSkipVerify = payload.TLSSkipVerify + repoConfig.ConfigFilePath = payload.ComposeFile if payload.ComposeFile == "" { repoConfig.ConfigFilePath = filesystem.ComposeFileDefaultName diff --git a/api/stacks/stackbuilders/stack_payload.go b/api/stacks/stackbuilders/stack_payload.go index df5835190..86a90456d 100644 --- a/api/stacks/stackbuilders/stack_payload.go +++ b/api/stacks/stackbuilders/stack_payload.go @@ -52,4 +52,6 @@ type RepositoryConfigPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true // and RepositoryGitCredentialID is 0 Password string `example:"myGitPassword"` + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` } diff --git a/api/stacks/stackutils/gitops.go b/api/stacks/stackutils/gitops.go index 325e13637..dcb601763 100644 --- a/api/stacks/stackutils/gitops.go +++ b/api/stacks/stackutils/gitops.go @@ -27,7 +27,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig stackFolder := fmt.Sprintf("%d", stackID) projectPath := fileService.GetStackProjectPath(stackFolder) - err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password) + err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password, config.TLSSkipVerify) if err != nil { if err == gittypes.ErrAuthenticationFailure { newErr := ErrInvalidGitCredential @@ -38,7 +38,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig return "", newErr } - commitID, err := gitService.LatestCommitID(config.URL, config.ReferenceName, username, password) + commitID, err := gitService.LatestCommitID(config.URL, config.ReferenceName, username, password, config.TLSSkipVerify) if err != nil { newErr := fmt.Errorf("unable to fetch git repository id: %w", err) return "", newErr diff --git a/app/edge/services/edge-stack.js b/app/edge/services/edge-stack.js index 2a6147e45..b325fe960 100644 --- a/app/edge/services/edge-stack.js +++ b/app/edge/services/edge-stack.js @@ -56,6 +56,7 @@ angular.module('portainer.edge').factory('EdgeStackService', function EdgeStackS RepositoryAuthentication: repositoryOptions.RepositoryAuthentication, RepositoryUsername: repositoryOptions.RepositoryUsername, RepositoryPassword: repositoryOptions.RepositoryPassword, + TLSSkipVerify: repositoryOptions.TLSSkipVerify, } ).$promise; } catch (err) { diff --git a/app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js b/app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js index c30ece1a8..2d4a78fa3 100644 --- a/app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js +++ b/app/edge/views/edge-stacks/createEdgeStackView/create-edge-stack-view.controller.js @@ -23,6 +23,7 @@ export default class CreateEdgeStackViewController { Groups: [], DeploymentType: 0, UseManifestNamespaces: false, + TLSSkipVerify: false, }; this.EditorType = EditorType; @@ -215,6 +216,7 @@ export default class CreateEdgeStackViewController { RepositoryAuthentication: this.formValues.RepositoryAuthentication, RepositoryUsername: this.formValues.RepositoryUsername, RepositoryPassword: this.formValues.RepositoryPassword, + TLSSkipVerify: this.formValues.TLSSkipVerify, }; return this.EdgeStackService.createStackFromGitRepository( { diff --git a/app/kubernetes/views/deploy/deployController.js b/app/kubernetes/views/deploy/deployController.js index d10734791..38eb34b22 100644 --- a/app/kubernetes/views/deploy/deployController.js +++ b/app/kubernetes/views/deploy/deployController.js @@ -59,6 +59,7 @@ class KubernetesDeployController { ComposeFilePathInRepository: '', Variables: {}, AutoUpdate: parseAutoUpdateResponse(), + TLSSkipVerify: false, }; this.ManifestDeployTypes = KubernetesDeployManifestTypes; @@ -248,6 +249,7 @@ class KubernetesDeployController { }; if (method === KubernetesDeployRequestMethods.REPOSITORY) { + payload.TLSSkipVerify = this.formValues.TLSSkipVerify; payload.RepositoryURL = this.formValues.RepositoryURL; payload.RepositoryReferenceName = this.formValues.RepositoryReferenceName; payload.RepositoryAuthentication = this.formValues.RepositoryAuthentication ? true : false; diff --git a/app/portainer/services/api/stackService.js b/app/portainer/services/api/stackService.js index 2cc85260e..2e581bc33 100644 --- a/app/portainer/services/api/stackService.js +++ b/app/portainer/services/api/stackService.js @@ -355,6 +355,7 @@ angular.module('portainer.app').factory('StackService', [ RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, FromAppTemplate: repositoryOptions.FromAppTemplate, + TLSSkipVerify: repositoryOptions.TLSSkipVerify, }; if (repositoryOptions.AutoUpdate) { @@ -382,6 +383,7 @@ angular.module('portainer.app').factory('StackService', [ RepositoryPassword: repositoryOptions.RepositoryPassword, Env: env, FromAppTemplate: repositoryOptions.FromAppTemplate, + TLSSkipVerify: repositoryOptions.TLSSkipVerify, }; if (repositoryOptions.AutoUpdate) { diff --git a/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js b/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js index 82b8b9324..8a39edbe7 100644 --- a/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js +++ b/app/portainer/views/custom-templates/create-custom-template-view/createCustomTemplateViewController.js @@ -44,6 +44,7 @@ class CreateCustomTemplateViewController { Type: 1, AccessControlData: new AccessControlFormData(), Variables: [], + TLSSkipVerify: false, }; this.state = { diff --git a/app/portainer/views/stacks/create/createStackController.js b/app/portainer/views/stacks/create/createStackController.js index 4a08b7205..f7acd9de7 100644 --- a/app/portainer/views/stacks/create/createStackController.js +++ b/app/portainer/views/stacks/create/createStackController.js @@ -57,6 +57,7 @@ angular EnableWebhook: false, Variables: {}, AutoUpdate: parseAutoUpdateResponse(), + TLSSkipVerify: false, }; $scope.state = { @@ -175,6 +176,7 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, AutoUpdate: transformAutoUpdateViewModel($scope.formValues.AutoUpdate, $scope.state.webhookId), + TLSSkipVerify: $scope.formValues.TLSSkipVerify, }; return StackService.createSwarmStackFromGitRepository(name, repositoryOptions, env, endpointId); @@ -201,6 +203,7 @@ angular RepositoryUsername: $scope.formValues.RepositoryUsername, RepositoryPassword: $scope.formValues.RepositoryPassword, AutoUpdate: transformAutoUpdateViewModel($scope.formValues.AutoUpdate, $scope.state.webhookId), + TLSSkipVerify: $scope.formValues.TLSSkipVerify, }; return StackService.createComposeStackFromGitRepository(name, repositoryOptions, env, endpointId); diff --git a/app/react/portainer/gitops/ComposePathField/PathSelector.tsx b/app/react/portainer/gitops/ComposePathField/PathSelector.tsx index 9f4e41144..fb97ceea0 100644 --- a/app/react/portainer/gitops/ComposePathField/PathSelector.tsx +++ b/app/react/portainer/gitops/ComposePathField/PathSelector.tsx @@ -35,6 +35,7 @@ export function PathSelector({ repository: model.RepositoryURL, keyword: searchTerm, reference: model.RepositoryReferenceName, + tlsSkipVerify: model.TLSSkipVerify, ...creds, }; const enabled = Boolean( diff --git a/app/react/portainer/gitops/GitForm.stories.tsx b/app/react/portainer/gitops/GitForm.stories.tsx index d2a0aa39e..dfb60768e 100644 --- a/app/react/portainer/gitops/GitForm.stories.tsx +++ b/app/react/portainer/gitops/GitForm.stories.tsx @@ -70,6 +70,7 @@ export function Primary({ ComposeFilePathInRepository: '', NewCredentialName: '', SaveCredential: false, + TLSSkipVerify: false, }; return ( diff --git a/app/react/portainer/gitops/GitForm.tsx b/app/react/portainer/gitops/GitForm.tsx index 569ceef80..bb290752a 100644 --- a/app/react/portainer/gitops/GitForm.tsx +++ b/app/react/portainer/gitops/GitForm.tsx @@ -9,6 +9,7 @@ import { TimeWindowDisplay } from '@/react/portainer/gitops/TimeWindowDisplay'; import { FormSection } from '@@/form-components/FormSection'; import { validateForm } from '@@/form-components/validate-form'; +import { SwitchField } from '@@/form-components/SwitchField'; import { GitCredential } from '../account/git-credentials/types'; @@ -104,6 +105,19 @@ export function GitForm({ )} + +
+
+ handleChange({ TLSSkipVerify: value })} + name="TLSSkipVerify" + tooltip="Enabling this will allow skipping TLS validation for any self-signed certificate." + labelClass="col-sm-3 col-lg-2" + /> +
+
); @@ -127,7 +141,18 @@ export function buildGitValidationSchema( ): SchemaOf { return object({ RepositoryURL: string() - .url('Invalid Url') + .test('valid URL', 'The URL must be a valid URL', (value) => { + if (!value) { + return true; + } + + try { + const url = new URL(value); + return !!url.hostname; + } catch { + return false; + } + }) .required('Repository URL is required'), RepositoryReferenceName: refFieldValidation(), ComposeFilePathInRepository: string().required( @@ -136,5 +161,6 @@ export function buildGitValidationSchema( AdditionalFiles: array(string().required('Path is required')).default([]), RepositoryURLValid: boolean().default(false), AutoUpdate: autoUpdateValidation().nullable(), + TLSSkipVerify: boolean().default(false), }).concat(gitAuthValidation(gitCredentials)); } diff --git a/app/react/portainer/gitops/GitFormUrlField.tsx b/app/react/portainer/gitops/GitFormUrlField.tsx index 9d322789e..15dabc1cf 100644 --- a/app/react/portainer/gitops/GitFormUrlField.tsx +++ b/app/react/portainer/gitops/GitFormUrlField.tsx @@ -40,14 +40,18 @@ export function GitFormUrlField({ const creds = getAuthentication(model); const [force, setForce] = useState(false); - const repoStatusQuery = useCheckRepo(value, creds, force, { - onSettled(isValid) { - onChangeRepositoryValid(!!isValid); - setForce(false); - }, - // disabled check on CE since it's not supported - enabled: isBE, - }); + const repoStatusQuery = useCheckRepo( + value, + { creds, force, tlsSkipVerify: model.TLSSkipVerify }, + { + onSettled(isValid) { + onChangeRepositoryValid(!!isValid); + setForce(false); + }, + // disabled check on CE since it's not supported + enabled: isBE, + } + ); const [debouncedValue, debouncedOnChange] = useDebounce(value, onChange); @@ -115,7 +119,7 @@ export function useUrlValidation(force: boolean) { const model = context.parent as GitFormModel; const creds = getAuthentication(model); - return checkRepo(url, creds, force); + return checkRepo(url, { creds, force }); } ); diff --git a/app/react/portainer/gitops/RefField/RefSelector.tsx b/app/react/portainer/gitops/RefField/RefSelector.tsx index 85964517a..39012e429 100644 --- a/app/react/portainer/gitops/RefField/RefSelector.tsx +++ b/app/react/portainer/gitops/RefField/RefSelector.tsx @@ -24,6 +24,7 @@ export function RefSelector({ const payload = { repository: model.RepositoryURL, stackId, + tlsSkipVerify: model.TLSSkipVerify, ...creds, }; diff --git a/app/react/portainer/gitops/RefField/types.ts b/app/react/portainer/gitops/RefField/types.ts index c0b7913b2..0bcd99afd 100644 --- a/app/react/portainer/gitops/RefField/types.ts +++ b/app/react/portainer/gitops/RefField/types.ts @@ -2,4 +2,5 @@ import { GitCredentialsModel } from '../types'; export interface RefFieldModel extends GitCredentialsModel { RepositoryURL: string; + TLSSkipVerify?: boolean; } diff --git a/app/react/portainer/gitops/queries/useCheckRepo.ts b/app/react/portainer/gitops/queries/useCheckRepo.ts index bad000653..9a64bb1ae 100644 --- a/app/react/portainer/gitops/queries/useCheckRepo.ts +++ b/app/react/portainer/gitops/queries/useCheckRepo.ts @@ -8,19 +8,23 @@ interface Creds { password?: string; gitCredentialId?: number; } +interface CheckRepoOptions { + creds?: Creds; + force?: boolean; + tlsSkipVerify?: boolean; +} export function useCheckRepo( url: string, - creds: Creds, - force: boolean, + options: CheckRepoOptions, { enabled, onSettled, }: { enabled?: boolean; onSettled?(isValid?: boolean): void } = {} ) { return useQuery( - ['git_repo_valid', url, creds, force], - () => checkRepo(url, creds, force), + ['git_repo_valid', url, options], + () => checkRepo(url, options), { enabled: !!url && enabled, onSettled, @@ -31,13 +35,12 @@ export function useCheckRepo( export async function checkRepo( repository: string, - creds: Creds, - force: boolean + { force, ...options }: CheckRepoOptions ): Promise { try { await axios.post( '/gitops/repo/refs', - { repository, ...creds }, + { repository, tlsSkipVerify: options.tlsSkipVerify, ...options.creds }, force ? { params: { force } } : {} ); return true; @@ -45,11 +48,12 @@ export async function checkRepo( throw parseAxiosError(error as Error, '', (axiosError: AxiosError) => { let details = axiosError.response?.data.details; + const { creds = {} } = options; // If no credentials were provided alter error from git to indicate repository is not found or is private if ( !(creds.username && creds.password) && details === - 'Authentication failed, please ensure that the git credentials are correct.' + 'authentication failed, please ensure that the git credentials are correct' ) { details = 'Git repository could not be found or is private, please ensure that the URL is correct or credentials are provided.'; diff --git a/app/react/portainer/gitops/queries/useGitRefs.ts b/app/react/portainer/gitops/queries/useGitRefs.ts index d1457799b..a8658f22d 100644 --- a/app/react/portainer/gitops/queries/useGitRefs.ts +++ b/app/react/portainer/gitops/queries/useGitRefs.ts @@ -6,6 +6,7 @@ interface RefsPayload { repository: string; username?: string; password?: string; + tlsSkipVerify?: boolean; } export function useGitRefs( diff --git a/app/react/portainer/gitops/types.ts b/app/react/portainer/gitops/types.ts index ec20a93da..4c0014ba4 100644 --- a/app/react/portainer/gitops/types.ts +++ b/app/react/portainer/gitops/types.ts @@ -60,6 +60,7 @@ export interface GitFormModel extends GitAuthModel { SaveCredential?: boolean; NewCredentialName?: string; + TLSSkipVerify: boolean; /** * Auto update