feat(oci): oci helm support [r8s-361] (#787)

pkg/liboras/generic_listrepo_client.go

@@ -0,0 +1,47 @@
+package liboras
+
+import (
+	"context"
+	"errors"
+
+	portainer "github.com/portainer/portainer/api"
+	"oras.land/oras-go/v2/registry/remote"
+)
+
+// GenericListRepoClient implements RepositoryListClient for standard OCI registries
+// This client handles repository listing for registries that follow the standard OCI distribution spec
+type GenericListRepoClient struct {
+	registry       *portainer.Registry
+	registryClient *remote.Registry
+}
+
+// NewGenericListRepoClient creates a new generic repository listing client
+func NewGenericListRepoClient(registry *portainer.Registry) *GenericListRepoClient {
+	return &GenericListRepoClient{
+		registry: registry,
+		// registryClient will be set when needed
+	}
+}
+
+// SetRegistryClient sets the ORAS registry client for repository listing operations
+func (c *GenericListRepoClient) SetRegistryClient(registryClient *remote.Registry) {
+	c.registryClient = registryClient
+}
+
+// ListRepositories fetches repositories from a standard OCI registry using ORAS
+func (c *GenericListRepoClient) ListRepositories(ctx context.Context) ([]string, error) {
+	if c.registryClient == nil {
+		return nil, errors.New("registry client not initialized for repository listing")
+	}
+
+	var repositories []string
+	err := c.registryClient.Repositories(ctx, "", func(repos []string) error {
+		repositories = append(repositories, repos...)
+		return nil
+	})
+	if err != nil {
+		return nil, errors.New("failed to list repositories")
+	}
+
+	return repositories, nil
+}

pkg/liboras/github_listrepo_client.go

@@ -0,0 +1,57 @@
+package liboras
+
+import (
+	"context"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy/factory/github"
+	"github.com/rs/zerolog/log"
+)
+
+// GithubListRepoClient implements RepositoryListClient specifically for GitHub registries
+// This client handles the GitHub Packages API's unique repository listing implementation
+type GithubListRepoClient struct {
+	registry *portainer.Registry
+	client   *github.Client
+}
+
+// NewGithubListRepoClient creates a new GitHub repository listing client
+func NewGithubListRepoClient(registry *portainer.Registry) *GithubListRepoClient {
+	// Prefer the management configuration credentials when available
+	token := registry.Password
+	if registry.ManagementConfiguration != nil && registry.ManagementConfiguration.Password != "" {
+		token = registry.ManagementConfiguration.Password
+	}
+
+	client := github.NewClient(token)
+
+	return &GithubListRepoClient{
+		registry: registry,
+		client:   client,
+	}
+}
+
+// ListRepositories fetches repositories from a GitHub registry using the GitHub Packages API
+func (c *GithubListRepoClient) ListRepositories(ctx context.Context) ([]string, error) {
+	repositories, err := c.client.GetContainerPackages(
+		ctx,
+		c.registry.Github.UseOrganisation,
+		c.registry.Github.OrganisationName,
+	)
+	if err != nil {
+		log.Error().
+			Str("registry_name", c.registry.Name).
+			Err(err).
+			Msg("Failed to list GitHub repositories")
+		return nil, fmt.Errorf("failed to list GitHub repositories: %w", err)
+	}
+
+	log.Debug().
+		Bool("use_organisation", c.registry.Github.UseOrganisation).
+		Str("organisation_name", c.registry.Github.OrganisationName).
+		Int("repository_count", len(repositories)).
+		Msg("Successfully listed GitHub repositories")
+
+	return repositories, nil
+}

pkg/liboras/gitlab_listrepo_client.go

@@ -0,0 +1,47 @@
+package liboras
+
+import (
+	"context"
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/http/proxy/factory/gitlab"
+	"github.com/rs/zerolog/log"
+)
+
+// GitlabListRepoClient implements RepositoryListClient specifically for GitLab registries
+// This client handles the GitLab Container Registry API's unique repository listing implementation
+type GitlabListRepoClient struct {
+	registry *portainer.Registry
+	client   *gitlab.Client
+}
+
+// NewGitlabListRepoClient creates a new GitLab repository listing client
+func NewGitlabListRepoClient(registry *portainer.Registry) *GitlabListRepoClient {
+	client := gitlab.NewClient(registry.Gitlab.InstanceURL, registry.Password)
+
+	return &GitlabListRepoClient{
+		registry: registry,
+		client:   client,
+	}
+}
+
+// ListRepositories fetches repositories from a GitLab registry using the GitLab API
+func (c *GitlabListRepoClient) ListRepositories(ctx context.Context) ([]string, error) {
+	repositories, err := c.client.GetRegistryRepositoryNames(ctx, c.registry.Gitlab.ProjectID)
+	if err != nil {
+		log.Error().
+			Str("registry_name", c.registry.Name).
+			Err(err).
+			Msg("Failed to list GitLab repositories")
+		return nil, fmt.Errorf("failed to list GitLab repositories: %w", err)
+	}
+
+	log.Debug().
+		Str("gitlab_url", c.registry.Gitlab.InstanceURL).
+		Int("project_id", c.registry.Gitlab.ProjectID).
+		Int("repository_count", len(repositories)).
+		Msg("Successfully listed GitLab repositories")
+
+	return repositories, nil
+}

pkg/liboras/listrepo_client.go

@@ -0,0 +1,39 @@
+package liboras
+
+import (
+	"context"
+
+	portainer "github.com/portainer/portainer/api"
+	"oras.land/oras-go/v2/registry/remote"
+)
+
+// RepositoryListClient provides an interface specifically for listing repositories
+// This exists because listing repositories isn't a standard OCI operation, and we need to handle
+// different registry types differently.
+type RepositoryListClient interface {
+	// ListRepositories returns a list of repository names from the registry
+	ListRepositories(ctx context.Context) ([]string, error)
+}
+
+// RepositoryListClientFactory creates repository listing clients based on registry type
+type RepositoryListClientFactory struct{}
+
+// NewRepositoryListClientFactory creates a new factory instance
+func NewRepositoryListClientFactory() *RepositoryListClientFactory {
+	return &RepositoryListClientFactory{}
+}
+
+// CreateListClientWithRegistry creates a repository listing client based on the registry type
+// and automatically configures it with the provided ORAS registry client for generic registries
+func (f *RepositoryListClientFactory) CreateListClientWithRegistry(registry *portainer.Registry, registryClient *remote.Registry) (RepositoryListClient, error) {
+	switch registry.Type {
+	case portainer.GitlabRegistry:
+		return NewGitlabListRepoClient(registry), nil
+	case portainer.GithubRegistry:
+		return NewGithubListRepoClient(registry), nil
+	default:
+		genericClient := NewGenericListRepoClient(registry)
+		genericClient.SetRegistryClient(registryClient)
+		return genericClient, nil
+	}
+}

pkg/liboras/registry.go

@@ -0,0 +1,79 @@
+package liboras
+
+import (
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
+)
+
+func CreateClient(registry portainer.Registry) (*remote.Registry, error) {
+	registryClient, err := remote.NewRegistry(registry.URL)
+	if err != nil {
+		log.Error().Err(err).Str("registryUrl", registry.URL).Msg("Failed to create registry client")
+		return nil, err
+	}
+	// By default, oras sends multiple requests to get the full list of repos/tags/referrers.
+	// set a high page size limit for fewer round trips.
+	// e.g. https://github.com/oras-project/oras-go/blob/v2.6.0/registry/remote/registry.go#L129-L142
+	registryClient.RepositoryListPageSize = 1000
+	registryClient.TagListPageSize = 1000
+	registryClient.ReferrerListPageSize = 1000
+
+	// Only apply authentication if explicitly enabled AND credentials are provided
+	if registry.Authentication &&
+		strings.TrimSpace(registry.Username) != "" &&
+		strings.TrimSpace(registry.Password) != "" {
+
+		registryClient.Client = &auth.Client{
+			Client: retry.DefaultClient,
+			Cache:  auth.NewCache(),
+			Credential: auth.StaticCredential(registry.URL, auth.Credential{
+				Username: registry.Username,
+				Password: registry.Password,
+			}),
+		}
+
+		log.Debug().
+			Str("registryURL", registry.URL).
+			Str("registryType", getRegistryTypeName(registry.Type)).
+			Bool("authentication", true).
+			Msg("Created ORAS registry client with authentication")
+	} else {
+		// Use default client for anonymous access
+		registryClient.Client = retry.DefaultClient
+
+		log.Debug().
+			Str("registryURL", registry.URL).
+			Str("registryType", getRegistryTypeName(registry.Type)).
+			Bool("authentication", false).
+			Msg("Created ORAS registry client for anonymous access")
+	}
+
+	return registryClient, nil
+}
+
+// getRegistryTypeName returns a human-readable name for the registry type
+func getRegistryTypeName(registryType portainer.RegistryType) string {
+	switch registryType {
+	case portainer.QuayRegistry:
+		return "Quay"
+	case portainer.AzureRegistry:
+		return "Azure"
+	case portainer.CustomRegistry:
+		return "Custom"
+	case portainer.GitlabRegistry:
+		return "GitLab"
+	case portainer.ProGetRegistry:
+		return "ProGet"
+	case portainer.DockerHubRegistry:
+		return "DockerHub"
+	case portainer.EcrRegistry:
+		return "ECR"
+	default:
+		return "Unknown"
+	}
+}

pkg/liboras/registry_test.go

@@ -0,0 +1,252 @@
+package liboras
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/stretchr/testify/assert"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
+)
+
+func TestCreateClient_AuthenticationScenarios(t *testing.T) {
+	tests := []struct {
+		name                string
+		registry            portainer.Registry
+		expectAuthenticated bool
+		description         string
+	}{
+		{
+			name: "authentication disabled should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: false,
+				Username:       "testuser",
+				Password:       "testpass",
+			},
+			expectAuthenticated: false,
+			description:         "Even with credentials present, authentication=false should result in anonymous access",
+		},
+		{
+			name: "authentication enabled with valid credentials should create authenticated client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "testuser",
+				Password:       "testpass",
+			},
+			expectAuthenticated: true,
+			description:         "Valid credentials with authentication=true should result in authenticated access",
+		},
+		{
+			name: "authentication enabled with empty username should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "",
+				Password:       "testpass",
+			},
+			expectAuthenticated: false,
+			description:         "Empty username should fallback to anonymous access",
+		},
+		{
+			name: "authentication enabled with whitespace-only username should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "   ",
+				Password:       "testpass",
+			},
+			expectAuthenticated: false,
+			description:         "Whitespace-only username should fallback to anonymous access",
+		},
+		{
+			name: "authentication enabled with empty password should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "testuser",
+				Password:       "",
+			},
+			expectAuthenticated: false,
+			description:         "Empty password should fallback to anonymous access",
+		},
+		{
+			name: "authentication enabled with whitespace-only password should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "testuser",
+				Password:       "   ",
+			},
+			expectAuthenticated: false,
+			description:         "Whitespace-only password should fallback to anonymous access",
+		},
+		{
+			name: "authentication enabled with both credentials empty should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: true,
+				Username:       "",
+				Password:       "",
+			},
+			expectAuthenticated: false,
+			description:         "Both credentials empty should fallback to anonymous access",
+		},
+		{
+			name: "public registry with no authentication should create anonymous client",
+			registry: portainer.Registry{
+				URL:            "docker.io",
+				Authentication: false,
+				Username:       "",
+				Password:       "",
+			},
+			expectAuthenticated: false,
+			description:         "Public registries without authentication should use anonymous access",
+		},
+		{
+			name: "GitLab registry with valid credentials should create authenticated client",
+			registry: portainer.Registry{
+				Type:           portainer.GitlabRegistry,
+				URL:            "registry.gitlab.com",
+				Authentication: true,
+				Username:       "gitlab-ci-token",
+				Password:       "glpat-xxxxxxxxxxxxxxxxxxxx",
+				Gitlab: portainer.GitlabRegistryData{
+					ProjectID:   12345,
+					InstanceURL: "https://gitlab.com",
+					ProjectPath: "my-group/my-project",
+				},
+			},
+			expectAuthenticated: true,
+			description:         "GitLab registry with valid credentials should result in authenticated access",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client, err := CreateClient(tt.registry)
+
+			assert.NoError(t, err, "CreateClient should not return an error")
+			assert.NotNil(t, client, "Client should not be nil")
+
+			// Check if the client has authentication configured
+			if tt.expectAuthenticated {
+				// Should have auth.Client with credentials
+				authClient, ok := client.Client.(*auth.Client)
+				assert.True(t, ok, "Expected auth.Client for authenticated access")
+				assert.NotNil(t, authClient, "Auth client should not be nil")
+				assert.NotNil(t, authClient.Credential, "Credential function should be set")
+			} else {
+				// Should use retry.DefaultClient (no authentication)
+				assert.Equal(t, retry.DefaultClient, client.Client,
+					"Expected retry.DefaultClient for anonymous access")
+			}
+		})
+	}
+}
+
+func TestCreateClient_RegistryTypes(t *testing.T) {
+	registryTypes := []struct {
+		name         string
+		registryType portainer.RegistryType
+		expectedName string
+	}{
+		{"DockerHub", portainer.DockerHubRegistry, "DockerHub"},
+		{"Azure", portainer.AzureRegistry, "Azure"},
+		{"Custom", portainer.CustomRegistry, "Custom"},
+		{"GitLab", portainer.GitlabRegistry, "GitLab"},
+		{"Quay", portainer.QuayRegistry, "Quay"},
+		{"ProGet", portainer.ProGetRegistry, "ProGet"},
+		{"ECR", portainer.EcrRegistry, "ECR"},
+	}
+
+	for _, rt := range registryTypes {
+		t.Run(rt.name, func(t *testing.T) {
+			registry := portainer.Registry{
+				URL:            "registry.example.com",
+				Type:           rt.registryType,
+				Authentication: false,
+			}
+
+			client, err := CreateClient(registry)
+
+			assert.NoError(t, err, "CreateClient should not return an error")
+			assert.NotNil(t, client, "Client should not be nil")
+
+			// Verify that getRegistryTypeName returns the expected name
+			typeName := getRegistryTypeName(rt.registryType)
+			assert.Equal(t, rt.expectedName, typeName, "Registry type name mismatch")
+		})
+	}
+}
+
+func TestGetRegistryTypeName(t *testing.T) {
+	tests := []struct {
+		registryType portainer.RegistryType
+		expectedName string
+	}{
+		{portainer.QuayRegistry, "Quay"},
+		{portainer.AzureRegistry, "Azure"},
+		{portainer.CustomRegistry, "Custom"},
+		{portainer.GitlabRegistry, "GitLab"},
+		{portainer.ProGetRegistry, "ProGet"},
+		{portainer.DockerHubRegistry, "DockerHub"},
+		{portainer.EcrRegistry, "ECR"},
+		{portainer.RegistryType(999), "Unknown"}, // Unknown type
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.expectedName, func(t *testing.T) {
+			result := getRegistryTypeName(tt.registryType)
+			assert.Equal(t, tt.expectedName, result, "Registry type name mismatch")
+		})
+	}
+}
+
+func TestCreateClient_ErrorHandling(t *testing.T) {
+	tests := []struct {
+		name        string
+		registry    portainer.Registry
+		expectError bool
+	}{
+		{
+			name: "valid registry URL should not error",
+			registry: portainer.Registry{
+				URL:            "registry.example.com",
+				Authentication: false,
+			},
+			expectError: false,
+		},
+		{
+			name: "empty registry URL should error",
+			registry: portainer.Registry{
+				URL:            "",
+				Authentication: false,
+			},
+			expectError: true,
+		},
+		{
+			name: "invalid registry URL should error",
+			registry: portainer.Registry{
+				URL:            "://invalid-url",
+				Authentication: false,
+			},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			client, err := CreateClient(tt.registry)
+
+			if tt.expectError {
+				assert.Error(t, err, "Expected an error but got none")
+				assert.Nil(t, client, "Client should be nil when error occurs")
+			} else {
+				assert.NoError(t, err, "Expected no error but got: %v", err)
+				assert.NotNil(t, client, "Client should not be nil")
+			}
+		})
+	}
+}

pkg/liboras/repository.go

@@ -0,0 +1,126 @@
+package liboras
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/concurrent"
+	"github.com/segmentio/encoding/json"
+	"golang.org/x/mod/semver"
+	"oras.land/oras-go/v2/registry"
+	"oras.land/oras-go/v2/registry/remote"
+)
+
+// ListRepositories retrieves all repositories from a registry using specialized repository listing clients
+// Each registry type has different repository listing implementations that require specific API calls
+func ListRepositories(ctx context.Context, registry *portainer.Registry, registryClient *remote.Registry) ([]string, error) {
+	factory := NewRepositoryListClientFactory()
+	listClient, err := factory.CreateListClientWithRegistry(registry, registryClient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create repository list client: %w", err)
+	}
+
+	return listClient.ListRepositories(ctx)
+}
+
+// FilterRepositoriesByMediaType filters repositories to only include those with the expected media type
+func FilterRepositoriesByMediaType(ctx context.Context, repositoryNames []string, registryClient *remote.Registry, expectedMediaType string) ([]string, error) {
+	// Run concurrently as this can take 10s+ to complete in serial
+	var tasks []concurrent.Func
+	for _, repoName := range repositoryNames {
+		name := repoName
+		task := func(ctx context.Context) (any, error) {
+			repository, err := registryClient.Repository(ctx, name)
+			if err != nil {
+				return nil, err
+			}
+
+			if HasMediaType(ctx, repository, expectedMediaType) {
+				return name, nil
+			}
+			return nil, nil // not a repository with the expected media type
+		}
+		tasks = append(tasks, task)
+	}
+
+	// 10 is a reasonable max concurrency limit
+	results, err := concurrent.Run(ctx, 10, tasks...)
+	if err != nil {
+		return nil, err
+	}
+
+	// Collect repository names
+	var repositories []string
+	for _, result := range results {
+		if result.Result != nil {
+			if repoName, ok := result.Result.(string); ok {
+				repositories = append(repositories, repoName)
+			}
+		}
+	}
+
+	return repositories, nil
+}
+
+// HasMediaType checks if a repository has artifacts with the specified media type
+func HasMediaType(ctx context.Context, repository registry.Repository, expectedMediaType string) bool {
+	// Check the first available tag
+	// Reasonable limitation - it won't work for repos where the latest tag is missing the expected media type but other tags have it
+	// This tradeoff is worth it for the performance benefits
+	var latestTag string
+	err := repository.Tags(ctx, "", func(tagList []string) error {
+		if len(tagList) > 0 {
+			// Order the taglist by latest semver, then get the latest tag
+			// e.g. ["1.0", "1.1"] -> ["1.1", "1.0"] -> "1.1"
+			sort.Slice(tagList, func(i, j int) bool {
+				return semver.Compare(tagList[i], tagList[j]) > 0
+			})
+			latestTag = tagList[0]
+		}
+		return nil
+	})
+
+	if err != nil {
+		return false
+	}
+
+	if latestTag == "" {
+		return false
+	}
+
+	descriptor, err := repository.Resolve(ctx, latestTag)
+	if err != nil {
+		return false
+	}
+
+	return descriptorHasMediaType(ctx, repository, descriptor, expectedMediaType)
+}
+
+// descriptorHasMediaType checks if a descriptor or its manifest contains the expected media type
+func descriptorHasMediaType(ctx context.Context, repository registry.Repository, descriptor ocispec.Descriptor, expectedMediaType string) bool {
+	// Check if the descriptor indicates the expected media type
+	if descriptor.MediaType == expectedMediaType {
+		return true
+	}
+
+	// Otherwise, look for the expected media type in the entire manifest content
+	manifestReader, err := repository.Manifests().Fetch(ctx, descriptor)
+	if err != nil {
+		return false
+	}
+	defer manifestReader.Close()
+
+	content, err := io.ReadAll(manifestReader)
+	if err != nil {
+		return false
+	}
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(content, &manifest); err != nil {
+		return false
+	}
+	return manifest.Config.MediaType == expectedMediaType
+}