fix(edgegroups): convert the related endpoint IDs to roaring bitmaps to increase performance BE-12053 (#903)

api/http/handler/edgegroups/associated_endpoints.go

@@ -4,6 +4,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/roar"
 )
 
 type endpointSetType map[portainer.EndpointID]bool
@@ -49,22 +50,29 @@ func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, p
 	return results, nil
 }
 
-func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs []portainer.EndpointID) ([]portainer.EndpointID, error) {
+func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs roar.Roar[portainer.EndpointID]) ([]portainer.EndpointID, error) {
+	var innerErr error
+
 	results := []portainer.EndpointID{}
-	for _, endpointID := range endpointIDs {
+
+	endpointIDs.Iterate(func(endpointID portainer.EndpointID) bool {
 		endpoint, err := tx.Endpoint().Endpoint(endpointID)
 		if err != nil {
-			return nil, err
+			innerErr = err
+
+			return false
 		}
 
 		if !endpoint.UserTrusted {
-			continue
+			return true
 		}
 
 		results = append(results, endpoint.ID)
-	}
 
-	return results, nil
+		return true
+	})
+
+	return results, innerErr
 }
 
 func mapEndpointGroupToEndpoints(endpoints []portainer.Endpoint) map[portainer.EndpointGroupID]endpointSetType {

api/http/handler/edgegroups/edgegroup_create.go

@@ -7,6 +7,7 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/internal/endpointutils"
+	"github.com/portainer/portainer/api/roar"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 )
@@ -52,6 +53,7 @@ func calculateEndpointsOrTags(tx dataservices.DataStoreTx, edgeGroup *portainer.
 	}
 
 	edgeGroup.Endpoints = endpointIDs
+	edgeGroup.EndpointIDs = roar.FromSlice(endpointIDs)
 
 	return nil
 }
@@ -94,6 +96,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 			Dynamic:      payload.Dynamic,
 			TagIDs:       []portainer.TagID{},
 			Endpoints:    []portainer.EndpointID{},
+			EndpointIDs:  roar.Roar[portainer.EndpointID]{},
 			PartialMatch: payload.PartialMatch,
 		}
 
@@ -108,5 +111,5 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 		return nil
 	})
 
-	return txResponse(w, edgeGroup, err)
+	return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
 }

api/http/handler/edgegroups/edgegroup_create_test.go

@@ -0,0 +1,62 @@
+package edgegroups
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEdgeGroupCreateHandler(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
+	handler.DataStore = store
+
+	err := store.EndpointGroup().Create(&portainer.EndpointGroup{
+		ID:   1,
+		Name: "Test Group",
+	})
+	require.NoError(t, err)
+
+	for i := range 3 {
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:      portainer.EndpointID(i + 1),
+			Name:    "Test Endpoint " + strconv.Itoa(i+1),
+			Type:    portainer.EdgeAgentOnDockerEnvironment,
+			GroupID: 1,
+		})
+		require.NoError(t, err)
+
+		err = store.EndpointRelation().Create(&portainer.EndpointRelation{
+			EndpointID: portainer.EndpointID(i + 1),
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		})
+		require.NoError(t, err)
+	}
+
+	rr := httptest.NewRecorder()
+
+	req := httptest.NewRequest(
+		http.MethodPost,
+		"/edge_groups",
+		strings.NewReader(`{"Name": "New Edge Group", "Endpoints": [1, 2, 3]}`),
+	)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Result().StatusCode)
+
+	var responseGroup portainer.EdgeGroup
+	err = json.NewDecoder(rr.Body).Decode(&responseGroup)
+	require.NoError(t, err)
+
+	require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
+}

api/http/handler/edgegroups/edgegroup_inspect.go

@@ -5,6 +5,7 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/roar"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 	"github.com/portainer/portainer/pkg/libhttp/request"
 )
@@ -33,7 +34,9 @@ func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request)
 		return err
 	})
 
-	return txResponse(w, edgeGroup, err)
+	edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
+
+	return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
 }
 
 func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
@@ -50,7 +53,7 @@ func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*porta
 			return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
 		}
 
-		edgeGroup.Endpoints = endpoints
+		edgeGroup.EndpointIDs = roar.FromSlice(endpoints)
 	}
 
 	return edgeGroup, err

api/http/handler/edgegroups/edgegroup_inspect_test.go

@@ -0,0 +1,137 @@
+package edgegroups
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/roar"
+
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEdgeGroupInspectHandler(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
+	handler.DataStore = store
+
+	err := store.EndpointGroup().Create(&portainer.EndpointGroup{
+		ID:   1,
+		Name: "Test Group",
+	})
+	require.NoError(t, err)
+
+	for i := range 3 {
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:      portainer.EndpointID(i + 1),
+			Name:    "Test Endpoint " + strconv.Itoa(i+1),
+			Type:    portainer.EdgeAgentOnDockerEnvironment,
+			GroupID: 1,
+		})
+		require.NoError(t, err)
+
+		err = store.EndpointRelation().Create(&portainer.EndpointRelation{
+			EndpointID: portainer.EndpointID(i + 1),
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		})
+		require.NoError(t, err)
+	}
+
+	err = store.EdgeGroup().Create(&portainer.EdgeGroup{
+		ID:          1,
+		Name:        "Test Edge Group",
+		EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
+	})
+	require.NoError(t, err)
+
+	rr := httptest.NewRecorder()
+
+	req := httptest.NewRequest(
+		http.MethodGet,
+		"/edge_groups/1",
+		nil,
+	)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Result().StatusCode)
+
+	var responseGroup portainer.EdgeGroup
+	err = json.NewDecoder(rr.Body).Decode(&responseGroup)
+	require.NoError(t, err)
+
+	assert.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
+}
+
+func TestDynamicEdgeGroupInspectHandler(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
+	handler.DataStore = store
+
+	err := store.EndpointGroup().Create(&portainer.EndpointGroup{
+		ID:   1,
+		Name: "Test Group",
+	})
+	require.NoError(t, err)
+
+	err = store.Tag().Create(&portainer.Tag{
+		ID:   1,
+		Name: "Test Tag",
+		Endpoints: map[portainer.EndpointID]bool{
+			1: true,
+			2: true,
+			3: true,
+		},
+	})
+	require.NoError(t, err)
+
+	for i := range 3 {
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:          portainer.EndpointID(i + 1),
+			Name:        "Test Endpoint " + strconv.Itoa(i+1),
+			Type:        portainer.EdgeAgentOnDockerEnvironment,
+			GroupID:     1,
+			TagIDs:      []portainer.TagID{1},
+			UserTrusted: true,
+		})
+		require.NoError(t, err)
+
+		err = store.EndpointRelation().Create(&portainer.EndpointRelation{
+			EndpointID: portainer.EndpointID(i + 1),
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		})
+		require.NoError(t, err)
+	}
+
+	err = store.EdgeGroup().Create(&portainer.EdgeGroup{
+		ID:      1,
+		Name:    "Test Edge Group",
+		Dynamic: true,
+		TagIDs:  []portainer.TagID{1},
+	})
+	require.NoError(t, err)
+
+	rr := httptest.NewRecorder()
+
+	req := httptest.NewRequest(
+		http.MethodGet,
+		"/edge_groups/1",
+		nil,
+	)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Result().StatusCode)
+
+	var responseGroup portainer.EdgeGroup
+	err = json.NewDecoder(rr.Body).Decode(&responseGroup)
+	require.NoError(t, err)
+
+	require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
+}

api/http/handler/edgegroups/edgegroup_list.go

@@ -7,11 +7,17 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/roar"
 	httperror "github.com/portainer/portainer/pkg/libhttp/error"
 )
 
-type decoratedEdgeGroup struct {
+type shadowedEdgeGroup struct {
 	portainer.EdgeGroup
+	EndpointIds int `json:"EndpointIds,omitempty"` // Shadow to avoid exposing in the API
+}
+
+type decoratedEdgeGroup struct {
+	shadowedEdgeGroup
 	HasEdgeStack     bool `json:"HasEdgeStack"`
 	HasEdgeJob       bool `json:"HasEdgeJob"`
 	EndpointTypes    []portainer.EndpointType
@@ -76,8 +82,8 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
 		}
 
 		edgeGroup := decoratedEdgeGroup{
-			EdgeGroup:     orgEdgeGroup,
-			EndpointTypes: []portainer.EndpointType{},
+			shadowedEdgeGroup: shadowedEdgeGroup{EdgeGroup: orgEdgeGroup},
+			EndpointTypes:     []portainer.EndpointType{},
 		}
 		if edgeGroup.Dynamic {
 			endpointIDs, err := GetEndpointsByTags(tx, edgeGroup.TagIDs, edgeGroup.PartialMatch)
@@ -88,15 +94,16 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
 			edgeGroup.Endpoints = endpointIDs
 			edgeGroup.TrustedEndpoints = endpointIDs
 		} else {
-			trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.Endpoints)
+			trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.EndpointIDs)
 			if err != nil {
 				return nil, httperror.InternalServerError("Unable to retrieve environments for Edge group", err)
 			}
 
+			edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
 			edgeGroup.TrustedEndpoints = trustedEndpoints
 		}
 
-		endpointTypes, err := getEndpointTypes(tx, edgeGroup.Endpoints)
+		endpointTypes, err := getEndpointTypes(tx, edgeGroup.EndpointIDs)
 		if err != nil {
 			return nil, httperror.InternalServerError("Unable to retrieve environment types for Edge group", err)
 		}
@@ -111,15 +118,26 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
 	return decoratedEdgeGroups, nil
 }
 
-func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds []portainer.EndpointID) ([]portainer.EndpointType, error) {
+func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds roar.Roar[portainer.EndpointID]) ([]portainer.EndpointType, error) {
+	var innerErr error
+
 	typeSet := map[portainer.EndpointType]bool{}
-	for _, endpointID := range endpointIds {
+
+	endpointIds.Iterate(func(endpointID portainer.EndpointID) bool {
 		endpoint, err := tx.Endpoint().Endpoint(endpointID)
 		if err != nil {
-			return nil, fmt.Errorf("failed fetching environment: %w", err)
+			innerErr = fmt.Errorf("failed fetching environment: %w", err)
+
+			return false
 		}
 
 		typeSet[endpoint.Type] = true
+
+		return true
+	})
+
+	if innerErr != nil {
+		return nil, innerErr
 	}
 
 	endpointTypes := make([]portainer.EndpointType, 0, len(typeSet))

api/http/handler/edgegroups/edgegroup_list_test.go

@@ -1,11 +1,19 @@
 package edgegroups
 
 import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/roar"
+
+	"github.com/segmentio/encoding/json"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_getEndpointTypes(t *testing.T) {
@@ -38,7 +46,7 @@ func Test_getEndpointTypes(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		ans, err := getEndpointTypes(datastore, test.endpointIds)
+		ans, err := getEndpointTypes(datastore, roar.FromSlice(test.endpointIds))
 		assert.NoError(t, err, "getEndpointTypes shouldn't fail")
 
 		assert.ElementsMatch(t, test.expected, ans, "getEndpointTypes expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
@@ -48,6 +56,61 @@ func Test_getEndpointTypes(t *testing.T) {
 func Test_getEndpointTypes_failWhenEndpointDontExist(t *testing.T) {
 	datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
 
-	_, err := getEndpointTypes(datastore, []portainer.EndpointID{1})
+	_, err := getEndpointTypes(datastore, roar.FromSlice([]portainer.EndpointID{1}))
 	assert.Error(t, err, "getEndpointTypes should fail")
 }
+
+func TestEdgeGroupListHandler(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
+	handler.DataStore = store
+
+	err := store.EndpointGroup().Create(&portainer.EndpointGroup{
+		ID:   1,
+		Name: "Test Group",
+	})
+	require.NoError(t, err)
+
+	for i := range 3 {
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:      portainer.EndpointID(i + 1),
+			Name:    "Test Endpoint " + strconv.Itoa(i+1),
+			Type:    portainer.EdgeAgentOnDockerEnvironment,
+			GroupID: 1,
+		})
+		require.NoError(t, err)
+
+		err = store.EndpointRelation().Create(&portainer.EndpointRelation{
+			EndpointID: portainer.EndpointID(i + 1),
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		})
+		require.NoError(t, err)
+	}
+
+	err = store.EdgeGroup().Create(&portainer.EdgeGroup{
+		ID:          1,
+		Name:        "Test Edge Group",
+		EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
+	})
+	require.NoError(t, err)
+
+	rr := httptest.NewRecorder()
+
+	req := httptest.NewRequest(
+		http.MethodGet,
+		"/edge_groups",
+		nil,
+	)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Result().StatusCode)
+
+	var responseGroups []decoratedEdgeGroup
+	err = json.NewDecoder(rr.Body).Decode(&responseGroups)
+	require.NoError(t, err)
+
+	require.Len(t, responseGroups, 1)
+	require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroups[0].Endpoints)
+	require.Len(t, responseGroups[0].TrustedEndpoints, 0)
+}

api/http/handler/edgegroups/edgegroup_update.go

@@ -158,7 +158,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 		return nil
 	})
 
-	return txResponse(w, edgeGroup, err)
+	return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
 }
 
 func (handler *Handler) updateEndpointStacks(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) error {

api/http/handler/edgegroups/edgegroup_update_test.go

@@ -0,0 +1,70 @@
+package edgegroups
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/api/roar"
+
+	"github.com/segmentio/encoding/json"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEdgeGroupUpdateHandler(t *testing.T) {
+	_, store := datastore.MustNewTestStore(t, true, true)
+
+	handler := NewHandler(testhelpers.NewTestRequestBouncer())
+	handler.DataStore = store
+
+	err := store.EndpointGroup().Create(&portainer.EndpointGroup{
+		ID:   1,
+		Name: "Test Group",
+	})
+	require.NoError(t, err)
+
+	for i := range 3 {
+		err = store.Endpoint().Create(&portainer.Endpoint{
+			ID:      portainer.EndpointID(i + 1),
+			Name:    "Test Endpoint " + strconv.Itoa(i+1),
+			Type:    portainer.EdgeAgentOnDockerEnvironment,
+			GroupID: 1,
+		})
+		require.NoError(t, err)
+
+		err = store.EndpointRelation().Create(&portainer.EndpointRelation{
+			EndpointID: portainer.EndpointID(i + 1),
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		})
+		require.NoError(t, err)
+	}
+
+	err = store.EdgeGroup().Create(&portainer.EdgeGroup{
+		ID:          1,
+		Name:        "Test Edge Group",
+		EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
+	})
+	require.NoError(t, err)
+
+	rr := httptest.NewRecorder()
+
+	req := httptest.NewRequest(
+		http.MethodPut,
+		"/edge_groups/1",
+		strings.NewReader(`{"Endpoints": [1, 2, 3]}`),
+	)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Result().StatusCode)
+
+	var responseGroup portainer.EdgeGroup
+	err = json.NewDecoder(rr.Body).Decode(&responseGroup)
+	require.NoError(t, err)
+
+	require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
+}