Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-30 10:49:40 +02:00
Code Issues Releases Activity

fix(edgegroups): convert the related endpoint IDs to roaring bitmaps to increase performance BE-12053 (#903)

Browse source
This commit is contained in:
andres-portainer 2025-07-21 21:31:13 -03:00 • committed by GitHub
parent caf382b64c
commit 937456596a
32 changed files with 1041 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

20
api/dataservices/edgegroup/tx.go
View file

@ -17,11 +17,29 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
}
func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
return service.Tx.CreateObject(
es := group.Endpoints
group.Endpoints = nil // Clear deprecated field
err := service.Tx.CreateObject(
BucketName,
func(id uint64) (int, any) {
group.ID = portainer.EdgeGroupID(id)
return int(group.ID), group
},
)
group.Endpoints = es // Restore endpoints after create
return err
}
func (service ServiceTx) Update(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
es := group.Endpoints
group.Endpoints = nil // Clear deprecated field
err := service.BaseDataServiceTx.Update(ID, group)
group.Endpoints = es // Restore endpoints after update
return err
}

1
api/datastore/migrate_data.go
View file

@ -85,6 +85,7 @@ func (store *Store) newMigratorParameters(version *models.Version, flags *portai
EdgeStackService: store.EdgeStackService,
EdgeStackStatusService: store.EdgeStackStatusService,
EdgeJobService: store.EdgeJobService,
EdgeGroupService: store.EdgeGroupService,
TunnelServerService: store.TunnelServerService,
PendingActionsService: store.PendingActionsService,
}

23
api/datastore/migrator/migrate_2_33_0.go Normal file
View file

@ -0,0 +1,23 @@
package migrator
import (
"github.com/portainer/portainer/api/roar"
)
func (m *Migrator) migrateEdgeGroupEndpointsToRoars_2_33_0() error {
egs, err := m.edgeGroupService.ReadAll()
if err != nil {
return err
}
for _, eg := range egs {
eg.EndpointIDs = roar.FromSlice(eg.Endpoints)
eg.Endpoints = nil
if err := m.edgeGroupService.Update(eg.ID, &eg); err != nil {
return err
}
}
return nil
}

7
api/datastore/migrator/migrator.go
View file

@ -6,6 +6,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/models"
"github.com/portainer/portainer/api/dataservices/dockerhub"
"github.com/portainer/portainer/api/dataservices/edgegroup"
"github.com/portainer/portainer/api/dataservices/edgejob"
"github.com/portainer/portainer/api/dataservices/edgestack"
"github.com/portainer/portainer/api/dataservices/edgestackstatus"
@ -60,6 +61,7 @@ type (
edgeStackService *edgestack.Service
edgeStackStatusService *edgestackstatus.Service
edgeJobService *edgejob.Service
edgeGroupService *edgegroup.Service
TunnelServerService *tunnelserver.Service
pendingActionsService *pendingactions.Service
}
@ -89,6 +91,7 @@ type (
EdgeStackService *edgestack.Service
EdgeStackStatusService *edgestackstatus.Service
EdgeJobService *edgejob.Service
EdgeGroupService *edgegroup.Service
TunnelServerService *tunnelserver.Service
PendingActionsService *pendingactions.Service
}
@ -120,11 +123,13 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
edgeStackService: parameters.EdgeStackService,
edgeStackStatusService: parameters.EdgeStackStatusService,
edgeJobService: parameters.EdgeJobService,
edgeGroupService: parameters.EdgeGroupService,
TunnelServerService: parameters.TunnelServerService,
pendingActionsService: parameters.PendingActionsService,
}
migrator.initMigrations()
return migrator
}
@ -251,6 +256,8 @@ func (m *Migrator) initMigrations() {
m.addMigrations("2.32.0", m.addEndpointRelationForEdgeAgents_2_32_0)
m.addMigrations("2.33.0", m.migrateEdgeGroupEndpointsToRoars_2_33_0)
// Add new migrations above...
// One function per migration, each versions migration funcs in the same file.
}

20
api/http/handler/edgegroups/associated_endpoints.go
View file

@ -4,6 +4,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
)
type endpointSetType map[portainer.EndpointID]bool
@ -49,22 +50,29 @@ func GetEndpointsByTags(tx dataservices.DataStoreTx, tagIDs []portainer.TagID, p
return results, nil
}
func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs []portainer.EndpointID) ([]portainer.EndpointID, error) {
func getTrustedEndpoints(tx dataservices.DataStoreTx, endpointIDs roar.Roar[portainer.EndpointID]) ([]portainer.EndpointID, error) {
var innerErr error
results := []portainer.EndpointID{}
for _, endpointID := range endpointIDs {
endpointIDs.Iterate(func(endpointID portainer.EndpointID) bool {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return nil, err
innerErr = err
return false
}
if !endpoint.UserTrusted {
continue
return true
}
results = append(results, endpoint.ID)
}
return results, nil
return true
})
return results, innerErr
}
func mapEndpointGroupToEndpoints(endpoints []portainer.Endpoint) map[portainer.EndpointGroupID]endpointSetType {

5
api/http/handler/edgegroups/edgegroup_create.go
View file

@ -7,6 +7,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
)
@ -52,6 +53,7 @@ func calculateEndpointsOrTags(tx dataservices.DataStoreTx, edgeGroup *portainer.
}
edgeGroup.Endpoints = endpointIDs
edgeGroup.EndpointIDs = roar.FromSlice(endpointIDs)
return nil
}
@ -94,6 +96,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
Dynamic: payload.Dynamic,
TagIDs: []portainer.TagID{},
Endpoints: []portainer.EndpointID{},
EndpointIDs: roar.Roar[portainer.EndpointID]{},
PartialMatch: payload.PartialMatch,
}
@ -108,5 +111,5 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
return nil
})
return txResponse(w, edgeGroup, err)
return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
}

62
api/http/handler/edgegroups/edgegroup_create_test.go Normal file
View file

@ -0,0 +1,62 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestEdgeGroupCreateHandler(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPost,
"/edge_groups",
strings.NewReader(`{"Name": "New Edge Group", "Endpoints": [1, 2, 3]}`),
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}

7
api/http/handler/edgegroups/edgegroup_inspect.go
View file

@ -5,6 +5,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
)
@ -33,7 +34,9 @@ func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request)
return err
})
return txResponse(w, edgeGroup, err)
edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
}
func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
@ -50,7 +53,7 @@ func getEdgeGroup(tx dataservices.DataStoreTx, ID portainer.EdgeGroupID) (*porta
return nil, httperror.InternalServerError("Unable to retrieve environments and environment groups for Edge group", err)
}
edgeGroup.Endpoints = endpoints
edgeGroup.EndpointIDs = roar.FromSlice(endpoints)
}
return edgeGroup, err

137
api/http/handler/edgegroups/edgegroup_inspect_test.go Normal file
View file

@ -0,0 +1,137 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestEdgeGroupInspectHandler(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups/1",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
assert.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}
func TestDynamicEdgeGroupInspectHandler(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
err = store.Tag().Create(&portainer.Tag{
ID: 1,
Name: "Test Tag",
Endpoints: map[portainer.EndpointID]bool{
1: true,
2: true,
3: true,
},
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
TagIDs: []portainer.TagID{1},
UserTrusted: true,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
Dynamic: true,
TagIDs: []portainer.TagID{1},
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups/1",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}

32
api/http/handler/edgegroups/edgegroup_list.go
View file

@ -7,11 +7,17 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/roar"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
type decoratedEdgeGroup struct {
type shadowedEdgeGroup struct {
portainer.EdgeGroup
EndpointIds int `json:"EndpointIds,omitempty"` // Shadow to avoid exposing in the API
}
type decoratedEdgeGroup struct {
shadowedEdgeGroup
HasEdgeStack bool `json:"HasEdgeStack"`
HasEdgeJob bool `json:"HasEdgeJob"`
EndpointTypes []portainer.EndpointType
@ -76,7 +82,7 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
}
edgeGroup := decoratedEdgeGroup{
EdgeGroup: orgEdgeGroup,
shadowedEdgeGroup: shadowedEdgeGroup{EdgeGroup: orgEdgeGroup},
EndpointTypes: []portainer.EndpointType{},
}
if edgeGroup.Dynamic {
@ -88,15 +94,16 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
edgeGroup.Endpoints = endpointIDs
edgeGroup.TrustedEndpoints = endpointIDs
} else {
trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.Endpoints)
trustedEndpoints, err := getTrustedEndpoints(tx, edgeGroup.EndpointIDs)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environments for Edge group", err)
}
edgeGroup.Endpoints = edgeGroup.EndpointIDs.ToSlice()
edgeGroup.TrustedEndpoints = trustedEndpoints
}
endpointTypes, err := getEndpointTypes(tx, edgeGroup.Endpoints)
endpointTypes, err := getEndpointTypes(tx, edgeGroup.EndpointIDs)
if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve environment types for Edge group", err)
}
@ -111,15 +118,26 @@ func getEdgeGroupList(tx dataservices.DataStoreTx) ([]decoratedEdgeGroup, error)
return decoratedEdgeGroups, nil
}
func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds []portainer.EndpointID) ([]portainer.EndpointType, error) {
func getEndpointTypes(tx dataservices.DataStoreTx, endpointIds roar.Roar[portainer.EndpointID]) ([]portainer.EndpointType, error) {
var innerErr error
typeSet := map[portainer.EndpointType]bool{}
for _, endpointID := range endpointIds {
endpointIds.Iterate(func(endpointID portainer.EndpointID) bool {
endpoint, err := tx.Endpoint().Endpoint(endpointID)
if err != nil {
return nil, fmt.Errorf("failed fetching environment: %w", err)
innerErr = fmt.Errorf("failed fetching environment: %w", err)
return false
}
typeSet[endpoint.Type] = true
return true
})
if innerErr != nil {
return nil, innerErr
}
endpointTypes := make([]portainer.EndpointType, 0, len(typeSet))

67
api/http/handler/edgegroups/edgegroup_list_test.go
View file

@ -1,11 +1,19 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_getEndpointTypes(t *testing.T) {
@ -38,7 +46,7 @@ func Test_getEndpointTypes(t *testing.T) {
}
for _, test := range tests {
ans, err := getEndpointTypes(datastore, test.endpointIds)
ans, err := getEndpointTypes(datastore, roar.FromSlice(test.endpointIds))
assert.NoError(t, err, "getEndpointTypes shouldn't fail")
assert.ElementsMatch(t, test.expected, ans, "getEndpointTypes expected to return %b for %v, but returned %b", test.expected, test.endpointIds, ans)
@ -48,6 +56,61 @@ func Test_getEndpointTypes(t *testing.T) {
func Test_getEndpointTypes_failWhenEndpointDontExist(t *testing.T) {
datastore := testhelpers.NewDatastore(testhelpers.WithEndpoints([]portainer.Endpoint{}))
_, err := getEndpointTypes(datastore, []portainer.EndpointID{1})
_, err := getEndpointTypes(datastore, roar.FromSlice([]portainer.EndpointID{1}))
assert.Error(t, err, "getEndpointTypes should fail")
}
func TestEdgeGroupListHandler(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1, 2, 3}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodGet,
"/edge_groups",
nil,
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroups []decoratedEdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroups)
require.NoError(t, err)
require.Len(t, responseGroups, 1)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroups[0].Endpoints)
require.Len(t, responseGroups[0].TrustedEndpoints, 0)
}

2
api/http/handler/edgegroups/edgegroup_update.go
View file

@ -158,7 +158,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
return nil
})
return txResponse(w, edgeGroup, err)
return txResponse(w, shadowedEdgeGroup{EdgeGroup: *edgeGroup}, err)
}
func (handler *Handler) updateEndpointStacks(tx dataservices.DataStoreTx, endpoint *portainer.Endpoint, edgeGroups []portainer.EdgeGroup, edgeStacks []portainer.EdgeStack) error {

70
api/http/handler/edgegroups/edgegroup_update_test.go Normal file
View file

@ -0,0 +1,70 @@
package edgegroups
import (
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
func TestEdgeGroupUpdateHandler(t *testing.T) {
_, store := datastore.MustNewTestStore(t, true, true)
handler := NewHandler(testhelpers.NewTestRequestBouncer())
handler.DataStore = store
err := store.EndpointGroup().Create(&portainer.EndpointGroup{
ID: 1,
Name: "Test Group",
})
require.NoError(t, err)
for i := range 3 {
err = store.Endpoint().Create(&portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "Test Endpoint " + strconv.Itoa(i+1),
Type: portainer.EdgeAgentOnDockerEnvironment,
GroupID: 1,
})
require.NoError(t, err)
err = store.EndpointRelation().Create(&portainer.EndpointRelation{
EndpointID: portainer.EndpointID(i + 1),
EdgeStacks: map[portainer.EdgeStackID]bool{},
})
require.NoError(t, err)
}
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Test Edge Group",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
rr := httptest.NewRecorder()
req := httptest.NewRequest(
http.MethodPut,
"/edge_groups/1",
strings.NewReader(`{"Endpoints": [1, 2, 3]}`),
)
handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Result().StatusCode)
var responseGroup portainer.EdgeGroup
err = json.NewDecoder(rr.Body).Decode(&responseGroup)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.EndpointID{1, 2, 3}, responseGroup.Endpoints)
}

5
api/http/handler/edgestacks/edgestack_create_test.go
View file

@ -8,9 +8,10 @@ import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/require"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// Create
@ -24,7 +25,7 @@ func TestCreateAndInspect(t *testing.T) {
Name: "EdgeGroup 1",
Dynamic: false,
TagIDs: nil,
Endpoints: []portainer.EndpointID{endpoint.ID},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpoint.ID}),
PartialMatch: false,
}

3
api/http/handler/edgestacks/edgestack_test.go
View file

@ -15,6 +15,7 @@ import (
"github.com/portainer/portainer/api/internal/edge/edgestacks"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/roar"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
@ -103,7 +104,7 @@ func createEdgeStack(t *testing.T, store dataservices.DataStore, endpointID port
Name: "EdgeGroup 1",
Dynamic: false,
TagIDs: nil,
Endpoints: []portainer.EndpointID{endpointID},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpointID}),
PartialMatch: false,
}

7
api/http/handler/edgestacks/edgestack_update_test.go
View file

@ -9,9 +9,10 @@ import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/require"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/require"
)
// Update
@ -43,7 +44,7 @@ func TestUpdateAndInspect(t *testing.T) {
Name: "EdgeGroup 2",
Dynamic: false,
TagIDs: nil,
Endpoints: []portainer.EndpointID{newEndpoint.ID},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{newEndpoint.ID}),
PartialMatch: false,
}
@ -112,7 +113,7 @@ func TestUpdateWithInvalidEdgeGroups(t *testing.T) {
Name: "EdgeGroup 2",
Dynamic: false,
TagIDs: nil,
Endpoints: []portainer.EndpointID{8889},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{8889}),
PartialMatch: false,
}

3
api/http/handler/endpointedge/endpointedge_status_inspect_test.go
View file

@ -16,6 +16,7 @@ import (
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/jwt"
"github.com/portainer/portainer/api/roar"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
@ -367,7 +368,7 @@ func TestEdgeJobsResponse(t *testing.T) {
staticEdgeGroup := portainer.EdgeGroup{
ID: 1,
Endpoints: []portainer.EndpointID{endpointFromStaticEdgeGroup.ID},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpointFromStaticEdgeGroup.ID}),
}
err := handler.DataStore.EdgeGroup().Create(&staticEdgeGroup)
require.NoError(t, err)

5
api/http/handler/endpoints/endpoint_delete.go
View file

@ -3,7 +3,6 @@ package endpoints
import (
"errors"
"net/http"
"slices"
"strconv"
portainer "github.com/portainer/portainer/api"
@ -200,9 +199,7 @@ func (handler *Handler) deleteEndpoint(tx dataservices.DataStoreTx, endpointID p
}
for _, edgeGroup := range edgeGroups {
edgeGroup.Endpoints = slices.DeleteFunc(edgeGroup.Endpoints, func(e portainer.EndpointID) bool {
return e == endpoint.ID
})
edgeGroup.EndpointIDs.Remove(endpoint.ID)
if err := tx.EdgeGroup().Update(edgeGroup.ID, &edgeGroup); err != nil {
log.Warn().Err(err).Msg("Unable to update edge group")

5
api/http/handler/endpoints/endpoint_delete_test.go
View file

@ -11,6 +11,7 @@ import (
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/proxy"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
)
func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
@ -44,7 +45,7 @@ func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
if err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "edgegroup-1",
Endpoints: endpointIDs,
EndpointIDs: roar.FromSlice(endpointIDs),
}); err != nil {
t.Fatal("could not create edge group:", err)
}
@ -78,7 +79,7 @@ func TestEndpointDeleteEdgeGroupsConcurrently(t *testing.T) {
t.Fatal("could not retrieve the edge group:", err)
}
if len(edgeGroup.Endpoints) > 0 {
if edgeGroup.EndpointIDs.Len() > 0 {
t.Fatal("the edge group is not consistent")
}
}

67
api/http/handler/endpoints/filter.go
View file

@ -14,7 +14,7 @@ import (
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/edge"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/slicesx"
"github.com/portainer/portainer/api/roar"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/pkg/errors"
@ -146,7 +146,9 @@ func (handler *Handler) filterEndpointsByQuery(
totalAvailableEndpoints := len(filteredEndpoints)
if len(query.endpointIds) > 0 {
filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, query.endpointIds)
endpointIDs := roar.FromSlice(query.endpointIds)
filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, endpointIDs)
}
if len(query.excludeIds) > 0 {
@ -275,7 +277,7 @@ func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId port
return nil, errors.WithMessage(err, "Unable to retrieve edge stack from the database")
}
envIds := make([]portainer.EndpointID, 0)
envIds := roar.Roar[portainer.EndpointID]{}
for _, edgeGroupdId := range stack.EdgeGroups {
edgeGroup, err := datastore.EdgeGroup().Read(edgeGroupdId)
if err != nil {
@ -287,32 +289,37 @@ func filterEndpointsByEdgeStack(endpoints []portainer.Endpoint, edgeStackId port
if err != nil {
return nil, errors.WithMessage(err, "Unable to retrieve environments and environment groups for Edge group")
}
edgeGroup.Endpoints = endpointIDs
edgeGroup.EndpointIDs = roar.FromSlice(endpointIDs)
}
envIds = append(envIds, edgeGroup.Endpoints...)
envIds.Union(edgeGroup.EndpointIDs)
}
if statusFilter != nil {
n := 0
for _, envId := range envIds {
var innerErr error
envIds.Iterate(func(envId portainer.EndpointID) bool {
edgeStackStatus, err := datastore.EdgeStackStatus().Read(edgeStackId, envId)
if dataservices.IsErrObjectNotFound(err) {
continue
return true
} else if err != nil {
return nil, errors.WithMessagef(err, "Unable to retrieve edge stack status for environment %d", envId)
innerErr = errors.WithMessagef(err, "Unable to retrieve edge stack status for environment %d", envId)
return false
}
if endpointStatusInStackMatchesFilter(edgeStackStatus, envId, *statusFilter) {
envIds[n] = envId
n++
}
}
envIds = envIds[:n]
if !endpointStatusInStackMatchesFilter(edgeStackStatus, portainer.EndpointID(envId), *statusFilter) {
envIds.Remove(envId)
}
uniqueIds := slicesx.Unique(envIds)
filteredEndpoints := filteredEndpointsByIds(endpoints, uniqueIds)
return true
})
if innerErr != nil {
return nil, innerErr
}
}
filteredEndpoints := filteredEndpointsByIds(endpoints, envIds)
return filteredEndpoints, nil
}
@ -344,16 +351,14 @@ func filterEndpointsByEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGroups []
}
edgeGroups = edgeGroups[:n]
endpointIDSet := make(map[portainer.EndpointID]struct{})
endpointIDSet := roar.Roar[portainer.EndpointID]{}
for _, edgeGroup := range edgeGroups {
for _, endpointID := range edgeGroup.Endpoints {
endpointIDSet[endpointID] = struct{}{}
}
endpointIDSet.Union(edgeGroup.EndpointIDs)
}
n = 0
for _, endpoint := range endpoints {
if _, exists := endpointIDSet[endpoint.ID]; exists {
if endpointIDSet.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}
@ -369,12 +374,11 @@ func filterEndpointsByExcludeEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGr
}
n := 0
excludeEndpointIDSet := make(map[portainer.EndpointID]struct{})
excludeEndpointIDSet := roar.Roar[portainer.EndpointID]{}
for _, edgeGroup := range edgeGroups {
if _, ok := excludeEdgeGroupIDSet[edgeGroup.ID]; ok {
for _, endpointID := range edgeGroup.Endpoints {
excludeEndpointIDSet[endpointID] = struct{}{}
}
excludeEndpointIDSet.Union(edgeGroup.EndpointIDs)
} else {
edgeGroups[n] = edgeGroup
n++
@ -384,7 +388,7 @@ func filterEndpointsByExcludeEdgeGroupIDs(endpoints []portainer.Endpoint, edgeGr
n = 0
for _, endpoint := range endpoints {
if _, ok := excludeEndpointIDSet[endpoint.ID]; !ok {
if !excludeEndpointIDSet.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}
@ -609,15 +613,10 @@ func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.
return len(missingTags) == 0
}
func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.EndpointID) []portainer.Endpoint {
idsSet := make(map[portainer.EndpointID]bool, len(ids))
for _, id := range ids {
idsSet[id] = true
}
func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids roar.Roar[portainer.EndpointID]) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if idsSet[endpoint.ID] {
if ids.Contains(endpoint.ID) {
endpoints[n] = endpoint
n++
}

130
api/http/handler/endpoints/filter_test.go
View file

@ -8,9 +8,11 @@ import (
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/portainer/portainer/api/slicesx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type filterTest struct {
@ -175,7 +177,7 @@ func BenchmarkFilterEndpointsBySearchCriteria_PartialMatch(b *testing.B) {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
Endpoints: append([]portainer.EndpointID{}, endpointIDs...),
EndpointIDs: roar.FromSlice(endpointIDs),
Dynamic: true,
TagIDs: []portainer.TagID{1, 2, 3},
PartialMatch: true,
@ -224,7 +226,7 @@ func BenchmarkFilterEndpointsBySearchCriteria_FullMatch(b *testing.B) {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
Endpoints: append([]portainer.EndpointID{}, endpointIDs...),
EndpointIDs: roar.FromSlice(endpointIDs),
Dynamic: true,
TagIDs: []portainer.TagID{1},
})
@ -300,3 +302,127 @@ func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) *Handler {
return handler
}
func TestFilterEndpointsByEdgeStack(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1"},
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}
edgeStackId := portainer.EdgeStackID(1)
err := store.EdgeStack().Create(edgeStackId, &portainer.EdgeStack{
ID: edgeStackId,
Name: "Test Edge Stack",
EdgeGroups: []portainer.EdgeGroupID{1, 2},
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{2, 3}),
})
require.NoError(t, err)
es, err := filterEndpointsByEdgeStack(endpoints, edgeStackId, nil, store)
require.NoError(t, err)
require.Len(t, es, 3)
require.Contains(t, es, endpoints[0]) // Endpoint 1
require.Contains(t, es, endpoints[1]) // Endpoint 2
require.Contains(t, es, endpoints[2]) // Endpoint 3
require.NotContains(t, es, endpoints[3]) // Endpoint 4
}
func TestFilterEndpointsByEdgeGroup(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1"},
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{2, 3}),
})
require.NoError(t, err)
edgeGroups, err := store.EdgeGroup().ReadAll()
require.NoError(t, err)
es, egs := filterEndpointsByEdgeGroupIDs(endpoints, edgeGroups, []portainer.EdgeGroupID{1, 2})
require.NoError(t, err)
require.Len(t, es, 3)
require.Contains(t, es, endpoints[0]) // Endpoint 1
require.Contains(t, es, endpoints[1]) // Endpoint 2
require.Contains(t, es, endpoints[2]) // Endpoint 3
require.NotContains(t, es, endpoints[3]) // Endpoint 4
require.Len(t, egs, 2)
require.Equal(t, egs[0].ID, portainer.EdgeGroupID(1))
require.Equal(t, egs[1].ID, portainer.EdgeGroupID(2))
}
func TestFilterEndpointsByExcludeEdgeGroupIDs(t *testing.T) {
_, store := datastore.MustNewTestStore(t, false, false)
endpoints := []portainer.Endpoint{
{ID: 1, Name: "Endpoint 1"},
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
}
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 1,
Name: "Edge Group 1",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{1}),
})
require.NoError(t, err)
err = store.EdgeGroup().Create(&portainer.EdgeGroup{
ID: 2,
Name: "Edge Group 2",
EndpointIDs: roar.FromSlice([]portainer.EndpointID{2, 3}),
})
require.NoError(t, err)
edgeGroups, err := store.EdgeGroup().ReadAll()
require.NoError(t, err)
es, egs := filterEndpointsByExcludeEdgeGroupIDs(endpoints, edgeGroups, []portainer.EdgeGroupID{1})
require.NoError(t, err)
require.Len(t, es, 3)
require.Equal(t, es, []portainer.Endpoint{
{ID: 2, Name: "Endpoint 2"},
{ID: 3, Name: "Endpoint 3"},
{ID: 4, Name: "Endpoint 4"},
})
require.Len(t, egs, 1)
require.Equal(t, egs[0].ID, portainer.EdgeGroupID(2))
}

25
api/http/handler/endpoints/utils_update_edge_groups.go
View file

@ -1,12 +1,11 @@
package endpoints
import (
"slices"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/set"
"github.com/pkg/errors"
)
func updateEnvironmentEdgeGroups(tx dataservices.DataStoreTx, newEdgeGroups []portainer.EdgeGroupID, environmentID portainer.EndpointID) (bool, error) {
@ -19,12 +18,10 @@ func updateEnvironmentEdgeGroups(tx dataservices.DataStoreTx, newEdgeGroups []po
environmentEdgeGroupsSet := set.Set[portainer.EdgeGroupID]{}
for _, edgeGroup := range edgeGroups {
for _, eID := range edgeGroup.Endpoints {
if eID == environmentID {
if edgeGroup.EndpointIDs.Contains(environmentID) {
environmentEdgeGroupsSet[edgeGroup.ID] = true
}
}
}
union := set.Union(newEdgeGroupsSet, environmentEdgeGroupsSet)
intersection := set.Intersection(newEdgeGroupsSet, environmentEdgeGroupsSet)
@ -52,20 +49,16 @@ func updateEnvironmentEdgeGroups(tx dataservices.DataStoreTx, newEdgeGroups []po
}
removeEdgeGroups := environmentEdgeGroupsSet.Difference(newEdgeGroupsSet)
err = updateSet(removeEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.Endpoints = slices.DeleteFunc(edgeGroup.Endpoints, func(eID portainer.EndpointID) bool {
return eID == environmentID
})
})
if err != nil {
if err := updateSet(removeEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.EndpointIDs.Remove(environmentID)
}); err != nil {
return false, err
}
addToEdgeGroups := newEdgeGroupsSet.Difference(environmentEdgeGroupsSet)
err = updateSet(addToEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.Endpoints = append(edgeGroup.Endpoints, environmentID)
})
if err != nil {
if err := updateSet(addToEdgeGroups, func(edgeGroup *portainer.EdgeGroup) {
edgeGroup.EndpointIDs.Add(environmentID)
}); err != nil {
return false, err
}

13
api/http/handler/endpoints/utils_update_edge_groups_test.go
View file

@ -6,6 +6,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
"github.com/stretchr/testify/assert"
)
@ -17,7 +18,6 @@ func Test_updateEdgeGroups(t *testing.T) {
Name: name,
Dynamic: false,
TagIDs: make([]portainer.TagID, 0),
Endpoints: make([]portainer.EndpointID, 0),
}
if err := store.EdgeGroup().Create(group); err != nil {
@ -35,13 +35,8 @@ func Test_updateEdgeGroups(t *testing.T) {
group, err := store.EdgeGroup().Read(groupID)
is.NoError(err)
for _, endpoint := range group.Endpoints {
if endpoint == endpointID {
return
}
}
is.Fail("expected endpoint to be in group")
is.True(group.EndpointIDs.Contains(endpointID),
"expected endpoint to be in group")
}
}
@ -81,7 +76,7 @@ func Test_updateEdgeGroups(t *testing.T) {
endpointGroups := groupsByName(groups, testCase.endpointGroupNames)
for _, group := range endpointGroups {
group.Endpoints = append(group.Endpoints, testCase.endpoint.ID)
group.EndpointIDs.Add(testCase.endpoint.ID)
err = store.EdgeGroup().Update(group.ID, &group)
is.NoError(err)

1
api/http/handler/endpoints/utils_update_tags_test.go
View file

@ -10,7 +10,6 @@ import (
)
func Test_updateTags(t *testing.T) {
createTags := func(store *datastore.Store, tagNames []string) ([]portainer.Tag, error) {
tags := make([]portainer.Tag, len(tagNames))
for index, tagName := range tagNames {

19
api/http/handler/tags/tag_delete_test.go
View file

@ -1,7 +1,6 @@
package tags
import (
"github.com/portainer/portainer/api/dataservices"
"net/http"
"net/http/httptest"
"strconv"
@ -9,9 +8,11 @@ import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
portainerDsErrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/roar"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -129,7 +130,7 @@ func TestHandler_tagDelete(t *testing.T) {
staticEdgeGroup := &portainer.EdgeGroup{
ID: 2,
Name: "edgegroup-2",
Endpoints: []portainer.EndpointID{endpoint2.ID},
EndpointIDs: roar.FromSlice([]portainer.EndpointID{endpoint2.ID}),
}
require.NoError(t, store.EdgeGroup().Create(staticEdgeGroup))
@ -163,14 +164,14 @@ func TestHandler_tagDelete(t *testing.T) {
dynamicEdgeGroup, err = store.EdgeGroup().Read(dynamicEdgeGroup.ID)
require.NoError(t, err)
assert.Len(t, dynamicEdgeGroup.TagIDs, 0, "dynamic edge group should not have any tags")
assert.Len(t, dynamicEdgeGroup.Endpoints, 0, "dynamic edge group should not have any endpoints")
assert.Equal(t, 0, dynamicEdgeGroup.EndpointIDs.Len(), "dynamic edge group should not have any endpoints")
// Check that the static edge group is not updated
staticEdgeGroup, err = store.EdgeGroup().Read(staticEdgeGroup.ID)
require.NoError(t, err)
assert.Len(t, staticEdgeGroup.TagIDs, 0, "static edge group should not have any tags")
assert.Len(t, staticEdgeGroup.Endpoints, 1, "static edge group should have one endpoint")
assert.Equal(t, endpoint2.ID, staticEdgeGroup.Endpoints[0], "static edge group should have the endpoint-2")
assert.Equal(t, 1, staticEdgeGroup.EndpointIDs.Len(), "static edge group should have one endpoint")
assert.True(t, staticEdgeGroup.EndpointIDs.Contains(endpoint2.ID), "static edge group should have the endpoint-2")
})
// Test the tx.IsErrObjectNotFound logic when endpoint is not found during cleanup
@ -185,14 +186,10 @@ func TestHandler_tagDelete(t *testing.T) {
}
err := store.Tag().Create(tag)
if err != nil {
t.Fatal("could not create tag:", err)
}
require.NoError(t, err)
err = deleteTag(store, 1)
if err != nil {
t.Fatal("could not delete tag:", err)
}
require.NoError(t, err)
})
}

6
api/internal/edge/edgegroup.go
View file

@ -1,8 +1,6 @@
package edge
import (
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/endpointutils"
@ -12,7 +10,7 @@ import (
// EdgeGroupRelatedEndpoints returns a list of environments(endpoints) related to this Edge group
func EdgeGroupRelatedEndpoints(edgeGroup *portainer.EdgeGroup, endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup) []portainer.EndpointID {
if !edgeGroup.Dynamic {
return edgeGroup.Endpoints
return edgeGroup.EndpointIDs.ToSlice()
}
endpointGroupsMap := map[portainer.EndpointGroupID]*portainer.EndpointGroup{}
@ -72,7 +70,7 @@ func GetEndpointsFromEdgeGroups(edgeGroupIDs []portainer.EdgeGroupID, datastore
// edgeGroupRelatedToEndpoint returns true if edgeGroup is associated with environment(endpoint)
func edgeGroupRelatedToEndpoint(edgeGroup *portainer.EdgeGroup, endpoint *portainer.Endpoint, endpointGroup *portainer.EndpointGroup) bool {
if !edgeGroup.Dynamic {
return slices.Contains(edgeGroup.Endpoints, endpoint.ID)
return edgeGroup.EndpointIDs.Contains(endpoint.ID)
}
endpointTags := tag.Set(endpoint.TagIDs)

104
api/internal/edge/edgegroup_benchmark_test.go Normal file
View file

@ -0,0 +1,104 @@
package edge
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/roar"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
)
const n = 1_000_000
func BenchmarkWriteEdgeGroupOld(b *testing.B) {
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
_, store := datastore.MustNewTestStore(b, false, false)
var endpointIDs []portainer.EndpointID
for i := range n {
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
for b.Loop() {
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
Name: "Test Edge Group",
Endpoints: endpointIDs,
})
require.NoError(b, err)
}
}
func BenchmarkWriteEdgeGroupNew(b *testing.B) {
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
_, store := datastore.MustNewTestStore(b, false, false)
var ts []portainer.EndpointID
for i := range n {
ts = append(ts, portainer.EndpointID(i+1))
}
endpointIDs := roar.FromSlice(ts)
for b.Loop() {
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
Name: "Test Edge Group",
EndpointIDs: endpointIDs,
})
require.NoError(b, err)
}
}
func BenchmarkReadEdgeGroupOld(b *testing.B) {
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
_, store := datastore.MustNewTestStore(b, false, false)
var endpointIDs []portainer.EndpointID
for i := range n {
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
Name: "Test Edge Group",
Endpoints: endpointIDs,
})
require.NoError(b, err)
for b.Loop() {
_, err := store.EdgeGroup().ReadAll()
require.NoError(b, err)
}
}
func BenchmarkReadEdgeGroupNew(b *testing.B) {
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
_, store := datastore.MustNewTestStore(b, false, false)
var ts []portainer.EndpointID
for i := range n {
ts = append(ts, portainer.EndpointID(i+1))
}
endpointIDs := roar.FromSlice(ts)
err := store.EdgeGroup().Create(&portainer.EdgeGroup{
Name: "Test Edge Group",
EndpointIDs: endpointIDs,
})
require.NoError(b, err)
for b.Loop() {
_, err := store.EdgeGroup().ReadAll()
require.NoError(b, err)
}
}

16
api/portainer.go
View file

@ -7,17 +7,18 @@ import (
"net/http"
"time"
gittypes "github.com/portainer/portainer/api/git/types"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/api/roar"
"github.com/portainer/portainer/pkg/featureflags"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/system"
"github.com/docker/docker/api/types/volume"
gittypes "github.com/portainer/portainer/api/git/types"
models "github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/pkg/featureflags"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/segmentio/encoding/json"
"golang.org/x/oauth2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/version"
@ -269,8 +270,11 @@ type (
Name string `json:"Name"`
Dynamic bool `json:"Dynamic"`
TagIDs []TagID `json:"TagIds"`
Endpoints []EndpointID `json:"Endpoints"`
EndpointIDs roar.Roar[EndpointID] `json:"EndpointIds"`
PartialMatch bool `json:"PartialMatch"`
// Deprecated: only used for API responses
Endpoints []EndpointID `json:"Endpoints"`
}
// EdgeGroupID represents an Edge group identifier

145
api/roar/roar.go Normal file
View file

@ -0,0 +1,145 @@
package roar
import (
"fmt"
"github.com/RoaringBitmap/roaring/v2"
)
type Roar[T ~int] struct {
rb *roaring.Bitmap
}
// Iterate iterates over the bitmap, calling the given callback with each value in the bitmap. If the callback returns
// false, the iteration is halted.
// The iteration results are undefined if the bitmap is modified (e.g., with Add or Remove).
// There is no guarantee as to what order the values will be iterated.
func (r *Roar[T]) Iterate(f func(T) bool) {
if r.rb == nil {
return
}
r.rb.Iterate(func(e uint32) bool {
return f(T(e))
})
}
// Len returns the number of elements contained in the bitmap
func (r *Roar[T]) Len() int {
if r.rb == nil {
return 0
}
return int(r.rb.GetCardinality())
}
// Remove removes the given element from the bitmap
func (r *Roar[T]) Remove(e T) {
if r.rb == nil {
return
}
r.rb.Remove(uint32(e))
}
// Add adds the given element to the bitmap
func (r *Roar[T]) Add(e T) {
if r.rb == nil {
r.rb = roaring.New()
}
r.rb.AddInt(int(e))
}
// Contains returns whether the bitmap contains the given element or not
func (r *Roar[T]) Contains(e T) bool {
if r.rb == nil {
return false
}
return r.rb.ContainsInt(int(e))
}
// Union combines the elements of the given bitmap with this bitmap
func (r *Roar[T]) Union(other Roar[T]) {
if other.rb == nil {
return
} else if r.rb == nil {
r.rb = roaring.New()
}
r.rb.Or(other.rb)
}
// Intersection modifies this bitmap to only contain elements that are also in the other bitmap
func (r *Roar[T]) Intersection(other Roar[T]) {
if other.rb == nil {
if r.rb != nil {
r.rb.Clear()
}
return
}
if r.rb == nil {
r.rb = roaring.New()
}
r.rb.And(other.rb)
}
// ToSlice converts the bitmap to a slice of elements
func (r *Roar[T]) ToSlice() []T {
if r.rb == nil {
return nil
}
slice := make([]T, 0, r.rb.GetCardinality())
r.rb.Iterate(func(e uint32) bool {
slice = append(slice, T(e))
return true
})
return slice
}
func (r *Roar[T]) MarshalJSON() ([]byte, error) {
if r.rb == nil {
return []byte("null"), nil
}
r.rb.RunOptimize()
buf, err := r.rb.ToBase64()
if err != nil {
return nil, fmt.Errorf("failed to encode roaring bitmap: %w", err)
}
return fmt.Appendf(nil, `"%s"`, buf), nil
}
func (r *Roar[T]) UnmarshalJSON(data []byte) error {
if len(data) == 0 || string(data) == "null" {
return nil
}
r.rb = roaring.New()
_, err := r.rb.FromBase64(string(data[1 : len(data)-1]))
return err
}
// FromSlice creates a Roar by adding all elements from the provided slices
func FromSlice[T ~int](ess ...[]T) Roar[T] {
var r Roar[T]
for _, es := range ess {
for _, e := range es {
r.Add(e)
}
}
return r
}

123
api/roar/roar_test.go Normal file
View file

@ -0,0 +1,123 @@
package roar
import (
"slices"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestRoar(t *testing.T) {
r := Roar[int]{}
require.Equal(t, 0, r.Len())
r.Add(1)
require.Equal(t, 1, r.Len())
require.True(t, r.Contains(1))
require.False(t, r.Contains(2))
r.Add(2)
require.Equal(t, 2, r.Len())
require.True(t, r.Contains(2))
r.Remove(1)
require.Equal(t, 1, r.Len())
require.False(t, r.Contains(1))
s := FromSlice([]int{3, 4, 5})
require.Equal(t, 3, s.Len())
require.True(t, s.Contains(3))
require.True(t, s.Contains(4))
require.True(t, s.Contains(5))
r.Union(s)
require.Equal(t, 4, r.Len())
require.True(t, r.Contains(2))
require.True(t, r.Contains(3))
require.True(t, r.Contains(4))
require.True(t, r.Contains(5))
r.Iterate(func(id int) bool {
require.True(t, slices.Contains([]int{2, 3, 4, 5}, id))
return true
})
rSlice := r.ToSlice()
require.EqualValues(t, []int{2, 3, 4, 5}, rSlice)
r.Intersection(FromSlice([]int{4}))
require.Equal(t, 1, r.Len())
require.True(t, r.Contains(4))
require.False(t, r.Contains(2))
require.False(t, r.Contains(3))
require.False(t, r.Contains(5))
b, err := r.MarshalJSON()
require.NoError(t, err)
require.NotEqual(t, "null", string(b))
require.True(t, strings.HasPrefix(string(b), `"`))
require.True(t, strings.HasSuffix(string(b), `"`))
}
func TestNilSafety(t *testing.T) {
var r, s, u Roar[int]
r.Iterate(func(id int) bool {
require.Fail(t, "should not iterate over nil Roar")
return true
})
b, err := r.MarshalJSON()
require.NoError(t, err)
require.Equal(t, "null", string(b))
err = r.UnmarshalJSON([]byte("null"))
require.NoError(t, err)
require.Equal(t, 0, r.Len())
r.Contains(1)
r.Remove(1)
require.Equal(t, 0, r.Len())
require.Empty(t, r.ToSlice())
r.Add(1)
require.Equal(t, 1, r.Len())
require.False(t, r.Contains(2))
s.Union(r)
require.Equal(t, 1, s.Len())
require.True(t, s.Contains(1))
r.Union(u)
require.Equal(t, 1, r.Len())
require.True(t, r.Contains(1))
s.Intersection(u)
require.Equal(t, 0, s.Len())
u.Intersection(r)
require.Equal(t, 0, u.Len())
}
func TestJSON(t *testing.T) {
var r, u Roar[int]
r.Add(1)
r.Add(2)
r.Add(3)
b, err := r.MarshalJSON()
require.NoError(t, err)
require.NotEqual(t, "null", string(b))
err = u.UnmarshalJSON(b)
require.NoError(t, err)
require.Equal(t, 3, u.Len())
require.True(t, u.Contains(1))
require.True(t, u.Contains(2))
require.True(t, u.Contains(3))
}

3
go.mod
View file

@ -5,6 +5,7 @@ go 1.24.4
require (
github.com/Masterminds/semver v1.5.0
github.com/Microsoft/go-winio v0.6.2
github.com/RoaringBitmap/roaring/v2 v2.5.0
github.com/VictoriaMetrics/fastcache v1.12.0
github.com/aws/aws-sdk-go-v2 v1.30.3
github.com/aws/aws-sdk-go-v2/credentials v1.17.27
@ -102,6 +103,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/buger/goterm v1.0.4 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@ -225,6 +227,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/opencontainers/runtime-spec v1.2.1 // indirect

7
go.sum
View file

@ -38,6 +38,8 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n
github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
github.com/RoaringBitmap/roaring/v2 v2.5.0 h1:TJ45qCM7D7fIEBwKd9zhoR0/S1egfnSSIzLU1e1eYLY=
github.com/RoaringBitmap/roaring/v2 v2.5.0/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
@ -100,6 +102,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
@ -548,6 +552,8 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -945,6 +951,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=