feat(gitops): support to list git repository refs and file tree [EE-2673] (#7100)

api/cmd/portainer/main.go

@@ -210,8 +210,8 @@ func initOAuthService() portainer.OAuthService {
 	return oauth.NewService()
 }
 
-func initGitService() portainer.GitService {
+func initGitService(ctx context.Context) portainer.GitService {
-	return git.NewService()
+	return git.NewService(ctx)
 }
 
 func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
@@ -580,8 +580,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	ldapService := initLDAPService()
 
 	oauthService := initOAuthService()
-
+	gitService := initGitService(shutdownCtx)
-	gitService := initGitService()
 
 	openAMTService := openamt.NewService()
 

api/git/azure.go

@@ -2,6 +2,7 @@ package git
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +11,10 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 )
@@ -32,20 +36,47 @@ type azureOptions struct {
 	username, password string
 }
 
-type azureDownloader struct {
+// azureRef abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#refs
+type azureRef struct {
+	Name     string `json:"name"`
+	ObjectID string `json:"objectId"`
+}
+
+// azureItem abstracts from the response of https://docs.microsoft.com/en-us/rest/api/azure/devops/git/items/get?view=azure-devops-rest-6.0#download
+type azureItem struct {
+	ObjectID string `json:"objectId"`
+	CommitId string `json:"commitId"`
+	Path     string `json:"path"`
+}
+
+type azureClient struct {
 	client  *http.Client
 	baseUrl string
 }
 
-func NewAzureDownloader(client *http.Client) *azureDownloader {
+func NewAzureClient() *azureClient {
-	return &azureDownloader{
+	httpsCli := newHttpClientForAzure()
-		client:  client,
+	return &azureClient{
+		client:  httpsCli,
 		baseUrl: "https://dev.azure.com",
 	}
 }
 
-func (a *azureDownloader) download(ctx context.Context, destination string, options cloneOptions) error {
+func newHttpClientForAzure() *http.Client {
-	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, options)
+	httpsCli := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+			Proxy:           http.ProxyFromEnvironment,
+		},
+		Timeout: 300 * time.Second,
+	}
+
+	client.InstallProtocol("https", githttp.NewClient(httpsCli))
+	return httpsCli
+}
+
+func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error {
+	zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt)
 	if err != nil {
 		return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
 	}
@@ -59,12 +90,12 @@ func (a *azureDownloader) download(ctx context.Context, destination string, opti
 	return nil
 }
 
-func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, options cloneOptions) (string, error) {
+func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) {
-	config, err := parseUrl(options.repositoryUrl)
+	config, err := parseUrl(opt.repositoryUrl)
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to parse url")
 	}
-	downloadUrl, err := a.buildDownloadUrl(config, options.referenceName)
+	downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName)
 	if err != nil {
 		return "", errors.WithMessage(err, "failed to build download url")
 	}
@@ -75,8 +106,8 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	defer zipFile.Close()
 
 	req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
-	if options.username != "" || options.password != "" {
+	if opt.username != "" || opt.password != "" {
-		req.SetBasicAuth(options.username, options.password)
+		req.SetBasicAuth(opt.username, opt.password)
 	} else if config.username != "" || config.password != "" {
 		req.SetBasicAuth(config.username, config.password)
 	}
@@ -102,53 +133,58 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option
 	return zipFile.Name(), nil
 }
 
-func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) {
+func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
-	config, err := parseUrl(options.repositoryUrl)
+	rootItem, err := a.getRootItem(ctx, opt)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to parse url")
+		return "", err
+	}
+	return rootItem.CommitId, nil
+}
+
+func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) {
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
 	}
 
-	rootItemUrl, err := a.buildRootItemUrl(config, options.referenceName)
+	rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to build azure root item url")
+		return nil, errors.WithMessage(err, "failed to build azure root item url")
 	}
 
 	req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
-	if options.username != "" || options.password != "" {
+	if opt.username != "" || opt.password != "" {
-		req.SetBasicAuth(options.username, options.password)
+		req.SetBasicAuth(opt.username, opt.password)
 	} else if config.username != "" || config.password != "" {
 		req.SetBasicAuth(config.username, config.password)
 	}
 
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to create a new HTTP request")
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
 	}
 
 	resp, err := a.client.Do(req)
 	if err != nil {
-		return "", errors.WithMessage(err, "failed to make an HTTP request")
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status)
+		return nil, checkAzureStatusCode(fmt.Errorf("failed to get repository root item with a status \"%v\"", resp.Status), resp.StatusCode)
 	}
 
 	var items struct {
-		Value []struct {
+		Value []azureItem
-			CommitId string `json:"commitId"`
-		}
 	}
 
 	if err := json.NewDecoder(resp.Body).Decode(&items); err != nil {
-		return "", errors.Wrap(err, "could not parse Azure items response")
+		return nil, errors.Wrap(err, "could not parse Azure items response")
 	}
 
 	if len(items.Value) == 0 || items.Value[0].CommitId == "" {
-		return "", errors.Errorf("failed to get latest commitID in the repository")
+		return nil, errors.Errorf("failed to get latest commitID in the repository")
 	}
-
+	return &items.Value[0], nil
-	return items.Value[0].CommitId, nil
 }
 
 func parseUrl(rawUrl string) (*azureOptions, error) {
@@ -219,7 +255,7 @@ func parseHttpUrl(rawUrl string) (*azureOptions, error) {
 	return &opt, nil
 }
 
-func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
+func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName string) (string, error) {
 	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
 		a.baseUrl,
 		url.PathEscape(config.organisation),
@@ -246,7 +282,7 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
-func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
+func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName string) (string, error) {
 	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/items",
 		a.baseUrl,
 		url.PathEscape(config.organisation),
@@ -270,6 +306,49 @@ func (a *azureDownloader) buildRootItemUrl(config *azureOptions, referenceName s
 	return u.String(), nil
 }
 
+func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) {
+	// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/refs/list?view=azure-devops-rest-6.0#gitref
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository))
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl)
+	}
+
+	q := u.Query()
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
+func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string) (string, error) {
+	// ref@https://docs.microsoft.com/en-us/rest/api/azure/devops/git/trees/get?view=azure-devops-rest-6.0
+	rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/trees/%s",
+		a.baseUrl,
+		url.PathEscape(config.organisation),
+		url.PathEscape(config.project),
+		url.PathEscape(config.repository),
+		url.PathEscape(rootObjectHash),
+	)
+	u, err := url.Parse(rawUrl)
+
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
+	}
+	q := u.Query()
+	// projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0
+	q.Set("recursive", "true")
+	q.Set("api-version", "6.0")
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
 const (
 	branchPrefix = "refs/heads/"
 	tagPrefix    = "refs/tags/"
@@ -294,3 +373,119 @@ func getVersionType(name string) string {
 	}
 	return "commit"
 }
+
+func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
+	}
+
+	listRefsUrl, err := a.buildRefsUrl(config)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to build list refs url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, checkAzureStatusCode(fmt.Errorf("failed to list refs with a status \"%v\"", resp.Status), resp.StatusCode)
+	}
+
+	var refs struct {
+		Value []azureRef
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil {
+		return nil, errors.Wrap(err, "could not parse Azure refs response")
+	}
+
+	var ret []string
+	for _, value := range refs.Value {
+		if value.Name == "HEAD" {
+			continue
+		}
+		ret = append(ret, value.Name)
+	}
+
+	return ret, nil
+}
+
+// listFiles list all filenames under the specific repository
+func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
+	rootItem, err := a.getRootItem(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := parseUrl(opt.repositoryUrl)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to parse url")
+	}
+
+	listTreeUrl, err := a.buildTreeUrl(config, rootItem.ObjectID)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to build list tree url")
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil)
+	if opt.username != "" || opt.password != "" {
+		req.SetBasicAuth(opt.username, opt.password)
+	} else if config.username != "" || config.password != "" {
+		req.SetBasicAuth(config.username, config.password)
+	}
+
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to create a new HTTP request")
+	}
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return nil, errors.WithMessage(err, "failed to make an HTTP request")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to list tree url with a status \"%v\"", resp.Status)
+	}
+
+	var tree struct {
+		TreeEntries []struct {
+			RelativePath string `json:"relativePath"`
+		} `json:"treeEntries"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&tree); err != nil {
+		return nil, errors.Wrap(err, "could not parse Azure tree response")
+	}
+
+	var allPaths []string
+	for _, treeEntry := range tree.TreeEntries {
+		allPaths = append(allPaths, treeEntry.RelativePath)
+	}
+
+	return allPaths, nil
+}
+
+func checkAzureStatusCode(err error, code int) error {
+	if code == http.StatusNotFound {
+		return ErrIncorrectRepositoryURL
+	} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
+		return ErrAuthenticationFailure
+	}
+	return err
+}

api/git/azure_integration_test.go

@@ -1,20 +1,26 @@
 package git
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	_ "github.com/joho/godotenv/autoload"
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
+)
+
 func TestService_ClonePublicRepository_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
 	type args struct {
 		repositoryURLFormat string
@@ -30,7 +36,7 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 		{
 			name: "Clone Azure DevOps repo branch",
 			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
 				referenceName:       "refs/heads/main",
 				username:            "",
 				password:            pat,
@@ -40,8 +46,8 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
 		{
 			name: "Clone Azure DevOps repo tag",
 			args: args{
-				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/Playground/_git/dev_integration",
+				repositoryURLFormat: "https://:%s@portainer.visualstudio.com/gitops-test/_git/gitops-test",
-				referenceName:       "refs/tags/v1.1",
+				referenceName:       "refs/heads/tags/v1.1",
 				username:            "",
 				password:            pat,
 			},
@@ -63,12 +69,11 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
 	dst := t.TempDir()
 
-	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	err := service.CloneRepository(dst, privateAzureRepoURL, "refs/heads/main", "", pat)
-	err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", "", pat)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
 }
@@ -77,14 +82,200 @@ func TestService_LatestCommitID_Azure(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	pat := getRequiredValue(t, "AZURE_DEVOPS_PAT")
-	service := NewService()
+	service := NewService(context.TODO())
 
-	repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration"
+	id, err := service.LatestCommitID(privateAzureRepoURL, "refs/heads/main", "", pat)
-	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
 
+func TestService_ListRefs_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := NewService(context.TODO())
+
+	refs, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+}
+
+func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+	service.ListRefs(privateAzureRepoURL, username, accessToken, false)
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_ListFiles_Azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+	service := newService(context.TODO(), 0, 0)
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+
+	tests := []struct {
+		name       string
+		args       fetchOption
+		extensions []string
+		expect     expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 19,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"yml"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and non-existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"hcl"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}
+
+func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
+
+	time.Sleep(2 * time.Second)
+}
+
 func getRequiredValue(t *testing.T, name string) string {
 	value, ok := os.LookupEnv(name)
 	if !ok {

api/git/azure_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func Test_buildDownloadUrl(t *testing.T) {
-	a := NewAzureDownloader(nil)
+	a := NewAzureClient()
 	u, err := a.buildDownloadUrl(&azureOptions{
 		organisation: "organisation",
 		project:      "project",
@@ -29,7 +29,7 @@ func Test_buildDownloadUrl(t *testing.T) {
 }
 
 func Test_buildRootItemUrl(t *testing.T) {
-	a := NewAzureDownloader(nil)
+	a := NewAzureClient()
 	u, err := a.buildRootItemUrl(&azureOptions{
 		organisation: "organisation",
 		project:      "project",
@@ -45,6 +45,40 @@ func Test_buildRootItemUrl(t *testing.T) {
 	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
 }
 
+func Test_buildRefsUrl(t *testing.T) {
+	a := NewAzureClient()
+	u, err := a.buildRefsUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	})
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?api-version=6.0")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
+func Test_buildTreeUrl(t *testing.T) {
+	a := NewAzureClient()
+	u, err := a.buildTreeUrl(&azureOptions{
+		organisation: "organisation",
+		project:      "project",
+		repository:   "repository",
+	}, "sha1")
+
+	expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/trees/sha1?api-version=6.0&recursive=true")
+	actualUrl, _ := url.Parse(u)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedUrl.Host, actualUrl.Host)
+	assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme)
+	assert.Equal(t, expectedUrl.Path, actualUrl.Path)
+	assert.Equal(t, expectedUrl.Query(), actualUrl.Query())
+}
+
 func Test_parseAzureUrl(t *testing.T) {
 	type args struct {
 		url string
@@ -200,7 +234,7 @@ func Test_isAzureUrl(t *testing.T) {
 
 func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 	type args struct {
-		options cloneOptions
+		options baseOption
 	}
 	type basicAuth struct {
 		username, password string
@@ -213,7 +247,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "username, password embedded",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
 				},
 			},
@@ -225,7 +259,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "username, password embedded, clone options take precedence",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
 					username:      "u",
 					password:      "p",
@@ -239,7 +273,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 		{
 			name: "no credentials",
 			args: args{
-				options: cloneOptions{
+				options: baseOption{
 					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
 				},
 			},
@@ -256,11 +290,17 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
 			}))
 			defer server.Close()
 
-			a := &azureDownloader{
+			a := &azureClient{
 				client:  server.Client(),
 				baseUrl: server.URL,
 			}
-			_, err := a.downloadZipFromAzureDevOps(context.Background(), tt.args.options)
+
+			option := cloneOption{
+				fetchOption: fetchOption{
+					baseOption: tt.args.options,
+				},
+			}
+			_, err := a.downloadZipFromAzureDevOps(context.Background(), option)
 			assert.Error(t, err)
 			assert.Equal(t, tt.want, zipRequestAuth)
 		})
@@ -287,22 +327,25 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
 	}))
 	defer server.Close()
 
-	a := &azureDownloader{
+	a := &azureClient{
 		client:  server.Client(),
 		baseUrl: server.URL,
 	}
 
 	tests := []struct {
 		name    string
-		args    fetchOptions
+		args    fetchOption
 		want    string
 		wantErr bool
 	}{
 		{
 			name: "should be able to parse response",
-			args: fetchOptions{
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
+				},
 				referenceName: "",
-				repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"},
+			},
 			want:    "27104ad7549d9e66685e115a497533f18024be9c",
 			wantErr: false,
 		},
@@ -319,3 +362,262 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
 		})
 	}
 }
+
+type testRepoManager struct {
+	called bool
+}
+
+func (t *testRepoManager) download(_ context.Context, _ string, _ cloneOption) error {
+	t.called = true
+	return nil
+}
+
+func (t *testRepoManager) latestCommitID(_ context.Context, _ fetchOption) (string, error) {
+	return "", nil
+}
+
+func (t *testRepoManager) listRefs(_ context.Context, _ baseOption) ([]string, error) {
+	return nil, nil
+}
+
+func (t *testRepoManager) listFiles(_ context.Context, _ fetchOption) ([]string, error) {
+	return nil, nil
+}
+func Test_cloneRepository_azure(t *testing.T) {
+	tests := []struct {
+		name   string
+		url    string
+		called bool
+	}{
+		{
+			name:   "Azure HTTP URL",
+			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			called: true,
+		},
+		{
+			name:   "Azure SSH URL",
+			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			called: true,
+		},
+		{
+			name:   "Something else",
+			url:    "https://example.com",
+			called: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azure := &testRepoManager{}
+			git := &testRepoManager{}
+
+			s := &Service{azure: azure, git: git}
+			s.cloneRepository("", cloneOption{
+				fetchOption: fetchOption{
+					baseOption: baseOption{
+
+						repositoryUrl: tt.url,
+					},
+				},
+				depth: 1,
+			})
+
+			// if azure API is called, git isn't and vice versa
+			assert.Equal(t, tt.called, azure.called)
+			assert.Equal(t, tt.called, !git.called)
+		})
+	}
+}
+
+func Test_listRefs_azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewAzureClient()
+
+	type expectResult struct {
+		err       error
+		refsCount int
+	}
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	tests := []struct {
+		name   string
+		args   baseOption
+		expect expectResult
+	}{
+		{
+			name: "list refs of a real repository",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err:       nil,
+				refsCount: 2,
+			},
+		},
+		{
+			name: "list refs of a real repository with incorrect credential",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      "test-username",
+				password:      "test-token",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a real repository without providing credential",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL,
+				username:      "",
+				password:      "",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a fake repository",
+			args: baseOption{
+				repositoryUrl: privateAzureRepoURL + "fake",
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err: ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			refs, err := client.listRefs(context.TODO(), tt.args)
+			if tt.expect.err == nil {
+				assert.NoError(t, err)
+				if tt.expect.refsCount > 0 {
+					assert.Greater(t, len(refs), 0)
+				}
+			} else {
+				assert.Error(t, err)
+				assert.Equal(t, tt.expect.err, err)
+			}
+		})
+	}
+
+}
+
+func Test_listFiles_azure(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewAzureClient()
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+
+	accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
+	username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
+	tests := []struct {
+		name   string
+		args   fetchOption
+		expect expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 19,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateAzureRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := client.listFiles(context.TODO(), tt.args)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}

api/git/git.go

@@ -2,52 +2,31 @@ package git
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
 )
 
-var (
-	ErrAuthenticationFailure = errors.New("Authentication failed, please ensure that the git credentials are correct.")
-)
-
-type fetchOptions struct {
-	repositoryUrl string
-	username      string
-	password      string
-	referenceName string
-}
-
-type cloneOptions struct {
-	repositoryUrl string
-	username      string
-	password      string
-	referenceName string
-	depth         int
-}
-
-type downloader interface {
-	download(ctx context.Context, dst string, opt cloneOptions) error
-	latestCommitID(ctx context.Context, opt fetchOptions) (string, error)
-}
-
 type gitClient struct {
 	preserveGitDirectory bool
 }
 
-func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) error {
+func NewGitClient(preserveGitDir bool) *gitClient {
+	return &gitClient{
+		preserveGitDirectory: preserveGitDir,
+	}
+}
+
+func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error {
 	gitOptions := git.CloneOptions{
 		URL:   opt.repositoryUrl,
 		Depth: opt.depth,
@@ -74,7 +53,7 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e
 	return nil
 }
 
-func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) {
+func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) {
 	remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
 		Name: "origin",
 		URLs: []string{opt.repositoryUrl},
@@ -124,66 +103,78 @@ func getAuth(username, password string) *githttp.BasicAuth {
 	return nil
 }
 
-// Service represents a service for managing Git.
+func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
-type Service struct {
+	rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
-	httpsCli *http.Client
+		Name: "origin",
-	azure    downloader
+		URLs: []string{opt.repositoryUrl},
-	git      downloader
+	})
+
+	listOptions := &git.ListOptions{
+		Auth: getAuth(opt.username, opt.password),
+	}
+
+	refs, err := rem.List(listOptions)
+	if err != nil {
+		return nil, checkGitError(err)
+	}
+
+	var ret []string
+	for _, ref := range refs {
+		if ref.Name().String() == "HEAD" {
+			continue
+		}
+		ret = append(ret, ref.Name().String())
+	}
+
+	return ret, nil
 }
 
-// NewService initializes a new service.
+// listFiles list all filenames under the specific repository
-func NewService() *Service {
+func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) {
-	httpsCli := &http.Client{
+	cloneOption := &git.CloneOptions{
-		Transport: &http.Transport{
+		URL:           opt.repositoryUrl,
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		NoCheckout:    true,
-			Proxy:           http.ProxyFromEnvironment,
+		Depth:         1,
-		},
+		SingleBranch:  true,
-		Timeout: 300 * time.Second,
+		ReferenceName: plumbing.ReferenceName(opt.referenceName),
+		Auth:          getAuth(opt.username, opt.password),
 	}
 
-	client.InstallProtocol("https", githttp.NewClient(httpsCli))
+	repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
-
+	if err != nil {
-	return &Service{
+		return nil, checkGitError(err)
-		httpsCli: httpsCli,
-		azure:    NewAzureDownloader(httpsCli),
-		git:      gitClient{},
 	}
+
+	head, err := repo.Head()
+	if err != nil {
+		return nil, err
+	}
+
+	commit, err := repo.CommitObject(head.Hash())
+	if err != nil {
+		return nil, err
+	}
+
+	tree, err := commit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	var allPaths []string
+	tree.Files().ForEach(func(f *object.File) error {
+		allPaths = append(allPaths, f.Name)
+		return nil
+	})
+
+	return allPaths, nil
 }
 
-// CloneRepository clones a git repository using the specified URL in the specified
+func checkGitError(err error) error {
-// destination folder.
+	errMsg := err.Error()
-func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	if errMsg == "repository not found" {
-	options := cloneOptions{
+		return ErrIncorrectRepositoryURL
-		repositoryUrl: repositoryURL,
+	} else if errMsg == "authentication required" {
-		username:      username,
+		return ErrAuthenticationFailure
-		password:      password,
-		referenceName: referenceName,
-		depth:         1,
 	}
-
+	return err
-	return service.cloneRepository(destination, options)
-}
-
-func (service *Service) cloneRepository(destination string, options cloneOptions) error {
-	if isAzureUrl(options.repositoryUrl) {
-		return service.azure.download(context.TODO(), destination, options)
-	}
-
-	return service.git.download(context.TODO(), destination, options)
-}
-
-// LatestCommitID returns SHA1 of the latest commit of the specified reference
-func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
-	options := fetchOptions{
-		repositoryUrl: repositoryURL,
-		username:      username,
-		password:      password,
-		referenceName: referenceName,
-	}
-
-	if isAzureUrl(options.repositoryUrl) {
-		return service.azure.latestCommitID(context.TODO(), options)
-	}
-
-	return service.git.latestCommitID(context.TODO(), options)
 }

api/git/git_integration_test.go

@@ -1,22 +1,28 @@
 package git
 
 import (
+	"context"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
 
+const (
+	privateGitRepoURL string = "https://github.com/portainer/private-test-repository.git"
+)
+
 func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
 	ensureIntegrationTest(t)
 
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
-	service := NewService()
+	service := newService(context.TODO(), 0, 0)
 
 	dst := t.TempDir()
 
-	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	repositoryUrl := privateGitRepoURL
 	err := service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.FileExists(t, filepath.Join(dst, "README.md"))
@@ -27,10 +33,318 @@ func TestService_LatestCommitID_GitHub(t *testing.T) {
 
 	accessToken := getRequiredValue(t, "GITHUB_PAT")
 	username := getRequiredValue(t, "GITHUB_USERNAME")
-	service := NewService()
+	service := newService(context.TODO(), 0, 0)
 
-	repositoryUrl := "https://github.com/portainer/private-test-repository.git"
+	repositoryUrl := privateGitRepoURL
 	id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken)
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty")
 }
+
+func TestService_ListRefs_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 0, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+}
+
+func TestService_ListRefs_Github_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	repositoryUrl := privateGitRepoURL
+	go service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_ListFiles_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+	service := newService(context.TODO(), 0, 0)
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+
+	tests := []struct {
+		name       string
+		args       fetchOption
+		extensions []string
+		expect     expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 15,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"yml"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref and non-existing file extension",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			extensions: []string{"hcl"},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 2,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			extensions: []string{},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := service.ListFiles(tt.args.repositoryUrl, tt.args.referenceName, tt.args.username, tt.args.password, false, tt.extensions)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
+		})
+	}
+}
+
+func TestService_ListFiles_Github_Concurrently(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
+
+	go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+
+	time.Sleep(2 * time.Second)
+}
+
+func TestService_purgeCache_Github(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := NewService(context.TODO())
+
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+
+	assert.Equal(t, 1, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	service.purgeCache()
+	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_purgeCacheByTTL_Github(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	timeout := 100 * time.Millisecond
+	repositoryUrl := privateGitRepoURL
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
+	service := newService(context.TODO(), 2, 40*timeout)
+
+	service.ListRefs(repositoryUrl, username, accessToken, false)
+	service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.Equal(t, 1, service.repoRefCache.Len())
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	// 40*timeout is designed for giving enough time for TTL being activated
+	time.Sleep(40 * timeout)
+	assert.Equal(t, 0, service.repoRefCache.Len())
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_canStopCacheCleanTimer_whenContextDone(t *testing.T) {
+	timeout := 10 * time.Millisecond
+	deadlineCtx, _ := context.WithDeadline(context.TODO(), time.Now().Add(10*timeout))
+
+	service := NewService(deadlineCtx)
+	assert.False(t, service.timerHasStopped(), "timer should not be stopped")
+
+	<-time.After(20 * timeout)
+
+	assert.True(t, service.timerHasStopped(), "timer should be stopped")
+}
+
+func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 2, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	assert.Error(t, err)
+	assert.Equal(t, 0, service.repoRefCache.Len())
+}
+
+func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	service := newService(context.TODO(), 2, 0)
+
+	repositoryUrl := privateGitRepoURL
+	refs, err := service.ListRefs(repositoryUrl, username, accessToken, false)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(refs), 1)
+	assert.Equal(t, 1, service.repoRefCache.Len())
+
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/test", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 2, service.repoFileCache.Len())
+
+	refs, err = service.ListRefs(repositoryUrl, username, "fake-token", true)
+	assert.Error(t, err)
+	assert.Equal(t, 0, service.repoRefCache.Len())
+
+	// The relevant file caches should be removed too
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}
+
+func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	service := newService(context.TODO(), 2, 0)
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+	repositoryUrl := privateGitRepoURL
+	files, err := service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, len(files), 1)
+	assert.Equal(t, 1, service.repoFileCache.Len())
+
+	files, err = service.ListFiles(repositoryUrl, "refs/heads/main", username, "fake-token", true, []string{})
+	assert.Error(t, err)
+	assert.Equal(t, 0, service.repoFileCache.Len())
+}

api/git/git_test.go

@@ -32,7 +32,7 @@ func setup(t *testing.T) string {
 }
 
 func Test_ClonePublicRepository_Shallow(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
 	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
 
@@ -44,7 +44,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) {
 }
 
 func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: false}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(false)} // no need for http client since the test access the repo via file system.
 	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
 
@@ -56,7 +56,7 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
 }
 
 func Test_cloneRepository(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
 
 	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
@@ -64,10 +64,14 @@ func Test_cloneRepository(t *testing.T) {
 	dir := t.TempDir()
 	t.Logf("Cloning into %s", dir)
 
-	err := service.cloneRepository(dir, cloneOptions{
+	err := service.cloneRepository(dir, cloneOption{
-		repositoryUrl: repositoryURL,
+		fetchOption: fetchOption{
-		referenceName: referenceName,
+			baseOption: baseOption{
-		depth:         10,
+				repositoryUrl: repositoryURL,
+			},
+			referenceName: referenceName,
+		},
+		depth: 10,
 	})
 
 	assert.NoError(t, err)
@@ -75,7 +79,7 @@ func Test_cloneRepository(t *testing.T) {
 }
 
 func Test_latestCommitID(t *testing.T) {
-	service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system.
+	service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
 
 	repositoryURL := setup(t)
 	referenceName := "refs/heads/main"
@@ -106,53 +110,196 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int {
 	return count
 }
 
-type testDownloader struct {
+func Test_listRefsPrivateRepository(t *testing.T) {
-	called bool
+	ensureIntegrationTest(t)
-}
 
-func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) error {
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
-	t.called = true
+	username := getRequiredValue(t, "GITHUB_USERNAME")
-	return nil
-}
 
-func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) {
+	client := NewGitClient(false)
-	return "", nil
+
-}
+	type expectResult struct {
+		err       error
+		refsCount int
+	}
 
-func Test_cloneRepository_azure(t *testing.T) {
 	tests := []struct {
 		name   string
-		url    string
+		args   baseOption
-		called bool
+		expect expectResult
 	}{
 		{
-			name:   "Azure HTTP URL",
+			name: "list refs of a real private repository",
-			url:    "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository",
+			args: baseOption{
-			called: true,
+				repositoryUrl: privateGitRepoURL,
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err:       nil,
+				refsCount: 2,
+			},
 		},
 		{
-			name:   "Azure SSH URL",
+			name: "list refs of a real private repository with incorrect credential",
-			url:    "git@ssh.dev.azure.com:v3/Organisation/Project/Repository",
+			args: baseOption{
-			called: true,
+				repositoryUrl: privateGitRepoURL,
+				username:      "test-username",
+				password:      "test-token",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
 		},
 		{
-			name:   "Something else",
+			name: "list refs of a fake repository without providing credential",
-			url:    "https://example.com",
+			args: baseOption{
-			called: false,
+				repositoryUrl: privateGitRepoURL + "fake",
+				username:      "",
+				password:      "",
+			},
+			expect: expectResult{
+				err: ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list refs of a fake repository",
+			args: baseOption{
+				repositoryUrl: privateGitRepoURL + "fake",
+				username:      username,
+				password:      accessToken,
+			},
+			expect: expectResult{
+				err: ErrIncorrectRepositoryURL,
+			},
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			azure := &testDownloader{}
+			refs, err := client.listRefs(context.TODO(), tt.args)
-			git := &testDownloader{}
+			if tt.expect.err == nil {
-
+				assert.NoError(t, err)
-			s := &Service{azure: azure, git: git}
+				if tt.expect.refsCount > 0 {
-			s.cloneRepository("", cloneOptions{repositoryUrl: tt.url, depth: 1})
+					assert.Greater(t, len(refs), 0)
-
+				}
-			// if azure API is called, git isn't and vice versa
+			} else {
-			assert.Equal(t, tt.called, azure.called)
+				assert.Error(t, err)
-			assert.Equal(t, tt.called, !git.called)
+				assert.Equal(t, tt.expect.err, err)
+			}
+		})
+	}
+}
+
+func Test_listFilesPrivateRepository(t *testing.T) {
+	ensureIntegrationTest(t)
+
+	client := NewGitClient(false)
+
+	type expectResult struct {
+		shouldFail   bool
+		err          error
+		matchedCount int
+	}
+
+	accessToken := getRequiredValue(t, "GITHUB_PAT")
+	username := getRequiredValue(t, "GITHUB_USERNAME")
+
+	tests := []struct {
+		name   string
+		args   fetchOption
+		expect expectResult
+	}{
+		{
+			name: "list tree with real repository and head ref but incorrect credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      "test-username",
+					password:      "test-token",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref but no credential",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      "",
+					password:      "",
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrAuthenticationFailure,
+			},
+		},
+		{
+			name: "list tree with real repository and head ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/heads/main",
+			},
+			expect: expectResult{
+				err:          nil,
+				matchedCount: 15,
+			},
+		},
+		{
+			name: "list tree with real repository but non-existing ref",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL,
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+			},
+		},
+		{
+			name: "list tree with fake repository ",
+			args: fetchOption{
+				baseOption: baseOption{
+					repositoryUrl: privateGitRepoURL + "fake",
+					username:      username,
+					password:      accessToken,
+				},
+				referenceName: "refs/fake/feature",
+			},
+			expect: expectResult{
+				shouldFail: true,
+				err:        ErrIncorrectRepositoryURL,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			paths, err := client.listFiles(context.TODO(), tt.args)
+			if tt.expect.shouldFail {
+				assert.Error(t, err)
+				if tt.expect.err != nil {
+					assert.Equal(t, tt.expect.err, err)
+				}
+			} else {
+				assert.NoError(t, err)
+				if tt.expect.matchedCount > 0 {
+					assert.Greater(t, len(paths), 0)
+				}
+			}
 		})
 	}
 }

api/git/service.go

@@ -0,0 +1,320 @@
+package git
+
+import (
+	"context"
+	"errors"
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
+)
+
+var (
+	ErrIncorrectRepositoryURL = errors.New("Git repository could not be found, please ensure that the URL is correct.")
+	ErrAuthenticationFailure  = errors.New("Authentication failed, please ensure that the git credentials are correct.")
+
+	REPOSITORY_CACHE_SIZE = 4
+	REPOSITORY_CACHE_TTL  = 5 * time.Minute
+)
+
+// baseOption provides a minimum group of information to operate a git repository, like git-remote
+type baseOption struct {
+	repositoryUrl string
+	username      string
+	password      string
+}
+
+// fetchOption allows to specify the reference name of the target repository
+type fetchOption struct {
+	baseOption
+	referenceName string
+}
+
+// cloneOption allows to add a history truncated to the specified number of commits
+type cloneOption struct {
+	fetchOption
+	depth int
+}
+
+type repoManager interface {
+	download(ctx context.Context, dst string, opt cloneOption) error
+	latestCommitID(ctx context.Context, opt fetchOption) (string, error)
+	listRefs(ctx context.Context, opt baseOption) ([]string, error)
+	listFiles(ctx context.Context, opt fetchOption) ([]string, error)
+}
+
+// Service represents a service for managing Git.
+type Service struct {
+	shutdownCtx  context.Context
+	azure        repoManager
+	git          repoManager
+	timerStopped bool
+	mut          sync.Mutex
+
+	cacheEnabled bool
+	// Cache the result of repository refs, key is repository URL
+	repoRefCache *lru.Cache
+	// Cache the result of repository file tree, key is the concatenated string of repository URL and ref value
+	repoFileCache *lru.Cache
+}
+
+// NewService initializes a new service.
+func NewService(ctx context.Context) *Service {
+	return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL)
+}
+
+func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {
+	service := &Service{
+		shutdownCtx:  ctx,
+		azure:        NewAzureClient(),
+		git:          NewGitClient(false),
+		timerStopped: false,
+		cacheEnabled: cacheSize > 0,
+	}
+
+	if service.cacheEnabled {
+		var err error
+		service.repoRefCache, err = lru.New(cacheSize)
+		if err != nil {
+			log.Printf("[DEBUG] [git] [message: failed to create ref cache: %v\n", err)
+		}
+
+		service.repoFileCache, err = lru.New(cacheSize)
+		if err != nil {
+			log.Printf("[DEBUG] [git] [message: failed to create file cache: %v\n", err)
+		}
+
+		if cacheTTL > 0 {
+			go service.startCacheCleanTimer(cacheTTL)
+		}
+	}
+
+	return service
+}
+
+// startCacheCleanTimer starts a timer to purge caches periodically
+func (service *Service) startCacheCleanTimer(d time.Duration) {
+	ticker := time.NewTicker(d)
+
+	for {
+		select {
+		case <-ticker.C:
+			service.purgeCache()
+
+		case <-service.shutdownCtx.Done():
+			ticker.Stop()
+			service.mut.Lock()
+			service.timerStopped = true
+			service.mut.Unlock()
+			return
+		}
+	}
+}
+
+// timerHasStopped shows the CacheClean timer state with thread-safe way
+func (service *Service) timerHasStopped() bool {
+	service.mut.Lock()
+	defer service.mut.Unlock()
+	ret := service.timerStopped
+	return ret
+}
+
+// CloneRepository clones a git repository using the specified URL in the specified
+// destination folder.
+func (service *Service) CloneRepository(destination, repositoryURL, referenceName, username, password string) error {
+	options := cloneOption{
+		fetchOption: fetchOption{
+			baseOption: baseOption{
+				repositoryUrl: repositoryURL,
+				username:      username,
+				password:      password,
+			},
+			referenceName: referenceName,
+		},
+		depth: 1,
+	}
+
+	return service.cloneRepository(destination, options)
+}
+
+func (service *Service) cloneRepository(destination string, options cloneOption) error {
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.download(context.TODO(), destination, options)
+	}
+
+	return service.git.download(context.TODO(), destination, options)
+}
+
+// LatestCommitID returns SHA1 of the latest commit of the specified reference
+func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
+	options := fetchOption{
+		baseOption: baseOption{
+			repositoryUrl: repositoryURL,
+			username:      username,
+			password:      password,
+		},
+		referenceName: referenceName,
+	}
+
+	if isAzureUrl(options.repositoryUrl) {
+		return service.azure.latestCommitID(context.TODO(), options)
+	}
+
+	return service.git.latestCommitID(context.TODO(), options)
+}
+
+// ListRefs will list target repository's references without cloning the repository
+func (service *Service) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	if service.cacheEnabled && hardRefresh {
+		// Should remove the cache explicitly, so that the following normal list can show the correct result
+		service.repoRefCache.Remove(repositoryURL)
+		// Remove file caches pointed to the same repository
+		for _, fileCacheKey := range service.repoFileCache.Keys() {
+			key, ok := fileCacheKey.(string)
+			if ok {
+				if strings.HasPrefix(key, repositoryURL) {
+					service.repoFileCache.Remove(key)
+				}
+			}
+		}
+	}
+
+	if service.repoRefCache != nil {
+		// Lookup the refs cache first
+		cache, ok := service.repoRefCache.Get(repositoryURL)
+		if ok {
+			refs, success := cache.([]string)
+			if success {
+				return refs, nil
+			}
+		}
+	}
+
+	options := baseOption{
+		repositoryUrl: repositoryURL,
+		username:      username,
+		password:      password,
+	}
+
+	var (
+		refs []string
+		err  error
+	)
+	if isAzureUrl(options.repositoryUrl) {
+		refs, err = service.azure.listRefs(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		refs, err = service.git.listRefs(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if service.cacheEnabled && service.repoRefCache != nil {
+		service.repoRefCache.Add(options.repositoryUrl, refs)
+	}
+	return refs, nil
+}
+
+// ListFiles will list all the files of the target repository with specific extensions.
+// If extension is not provided, it will list all the files under the target repository
+func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
+	repoKey := generateCacheKey(repositoryURL, referenceName)
+
+	if service.cacheEnabled && hardRefresh {
+		// Should remove the cache explicitly, so that the following normal list can show the correct result
+		service.repoFileCache.Remove(repoKey)
+	}
+
+	if service.repoFileCache != nil {
+		// lookup the files cache first
+		cache, ok := service.repoFileCache.Get(repoKey)
+		if ok {
+			files, success := cache.([]string)
+			if success {
+				// For the case while searching files in a repository without include extensions for the first time,
+				// but with include extensions for the second time
+				includedFiles := filterFiles(files, includedExts)
+				return includedFiles, nil
+			}
+		}
+	}
+
+	options := fetchOption{
+		baseOption: baseOption{
+			repositoryUrl: repositoryURL,
+			username:      username,
+			password:      password,
+		},
+		referenceName: referenceName,
+	}
+
+	var (
+		files []string
+		err   error
+	)
+	if isAzureUrl(options.repositoryUrl) {
+		files, err = service.azure.listFiles(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		files, err = service.git.listFiles(context.TODO(), options)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	includedFiles := filterFiles(files, includedExts)
+	if service.cacheEnabled && service.repoFileCache != nil {
+		service.repoFileCache.Add(repoKey, includedFiles)
+		return includedFiles, nil
+	}
+	return includedFiles, nil
+}
+
+func (service *Service) purgeCache() {
+	if service.repoRefCache != nil {
+		service.repoRefCache.Purge()
+	}
+
+	if service.repoFileCache != nil {
+		service.repoFileCache.Purge()
+	}
+}
+
+func generateCacheKey(names ...string) string {
+	return strings.Join(names, "-")
+}
+
+func matchExtensions(target string, exts []string) bool {
+	if len(exts) == 0 {
+		return true
+	}
+
+	for _, ext := range exts {
+		if strings.HasSuffix(target, ext) {
+			return true
+		}
+	}
+	return false
+}
+
+func filterFiles(paths []string, includedExts []string) []string {
+	if len(includedExts) == 0 {
+		return paths
+	}
+
+	var includedFiles []string
+	for _, filename := range paths {
+		// filter out the filenames with non-included extension
+		if matchExtensions(filename, includedExts) {
+			includedFiles = append(includedFiles, filename)
+		}
+	}
+	return includedFiles
+}

api/http/handler/edgestacks/edgestack_test.go

@@ -34,6 +34,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass
 	return g.id, nil
 }
 
+func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	return nil, nil
+}
+
+func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
+	return nil, nil
+}
+
 // Helpers
 func setupHandler(t *testing.T) (*Handler, string, func()) {
 	t.Helper()

api/http/proxy/factory/docker/transport_test.go

@@ -1,11 +1,12 @@
 package docker
 
 import (
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
 )
 
 type noopGitService struct{}
@@ -16,6 +17,12 @@ func (s *noopGitService) CloneRepository(destination string, repositoryURL, refe
 func (s *noopGitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) {
 	return "my-latest-commit-id", nil
 }
+func (g *noopGitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	return nil, nil
+}
+func (g *noopGitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
+	return nil, nil
+}
 
 func TestTransport_updateDefaultGitBranch(t *testing.T) {
 	type fields struct {

api/portainer.go

@@ -1323,6 +1323,8 @@ type (
 	GitService interface {
 		CloneRepository(destination string, repositoryURL, referenceName, username, password string) error
 		LatestCommitID(repositoryURL, referenceName, username, password string) (string, error)
+		ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error)
+		ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includeExts []string) ([]string, error)
 	}
 
 	// OpenAMTService represents a service for managing OpenAMT

api/stacks/deploy_test.go

@@ -25,6 +25,14 @@ func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, pass
 	return g.id, nil
 }
 
+func (g *gitService) ListRefs(repositoryURL, username, password string, hardRefresh bool) ([]string, error) {
+	return nil, nil
+}
+
+func (g *gitService) ListFiles(repositoryURL, referenceName, username, password string, hardRefresh bool, includedExts []string) ([]string, error) {
+	return nil, nil
+}
+
 type noopDeployer struct{}
 
 func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool, pullImage bool) error {

app/portainer/components/forms/git-form/git-form.html

@@ -1,7 +1,11 @@
 <ng-form name="$ctrl.gitForm">
   <div class="col-sm-12 form-section-title"> Git repository </div>
+  <git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
+
   <git-form-url-field value="$ctrl.model.RepositoryURL" on-change="($ctrl.onChangeURL)"></git-form-url-field>
+
   <git-form-ref-field value="$ctrl.model.RepositoryReferenceName" on-change="($ctrl.onChangeRefName)"></git-form-ref-field>
+
   <git-form-compose-path-field
     value="$ctrl.model.ComposeFilePathInRepository"
     on-change="($ctrl.onChangeComposePath)"
@@ -10,8 +14,6 @@
 
   <git-form-additional-files-panel ng-if="$ctrl.additionalFile" model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-additional-files-panel>
 
-  <git-form-auth-fieldset model="$ctrl.model" on-change="($ctrl.onChange)"></git-form-auth-fieldset>
-
   <git-form-auto-update-fieldset
     ng-if="$ctrl.autoUpdate"
     model="$ctrl.model"