1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-25 00:09:40 +02:00

fix(edge): filtering of edge devices [EE-3210] (#7077)

* fix(edge): filtering of edge devices [EE-3210]

fixes [EE-3210]

changes:
- replaces `edgeDeviceFilter` with two filters:
	- `edgeDevice`
	- `edgeDeviceUntrusted`

these filters will only apply to the edge endpoints in the query (so it's possible to get both regular endpoints and edge devices).

if `edgeDevice` is true, will filter out edge agents which are not an edge device.
			false, will filter out edge devices

`edgeDeviceUntrusted` applies only when `edgeDevice` is true. then false (default) will hide the untrusted edge devices, true will show only untrusted edge devices.

fix(edge/job-create): retrieve only trusted endpoints + fix endpoint selector pagination limits onChange

fix(endpoint-groups): remove listing of untrusted edge envs (aka in waiting room)

refactor(endpoints): move filter to another function

feat(endpoints): separate edge filters

refactor(environments): change getEnv api

refactor(endpoints): use single getEnv

feat(groups): show error when failed loading envs

style(endpoints): remove unused endpointsByGroup

* chore(deps): update go to 1.18

* fix(endpoint): filter out untrusted by default

* fix(edge): show correct endpoints

* style(endpoints): fix typo

* fix(endpoints): fix swagger

* fix(admin): use new getEnv function

Co-authored-by: LP B <xAt0mZ@users.noreply.github.com>
This commit is contained in:
Chaim Lev-Ari 2022-07-19 18:00:45 +02:00 committed by GitHub
parent 1a8fe82821
commit 05357ecce5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 868 additions and 601 deletions

View file

@ -4,24 +4,14 @@ import (
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/portainer/libhttp/request"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/internal/utils"
)
const (
EdgeDeviceFilterAll = "all"
EdgeDeviceFilterTrusted = "trusted"
EdgeDeviceFilterUntrusted = "untrusted"
EdgeDeviceFilterNone = "none"
)
const (
@ -42,14 +32,19 @@ var endpointGroupNames map[portainer.EndpointGroupID]string
// @security jwt
// @produce json
// @param start query int false "Start searching from"
// @param search query string false "Search query"
// @param groupId query int false "List environments(endpoints) of this group"
// @param limit query int false "Limit results to this value"
// @param sort query int false "Sort results by this value"
// @param order query int false "Order sorted results by desc/asc" Enum("asc", "desc")
// @param search query string false "Search query"
// @param groupIds query []int false "List environments(endpoints) of these groups"
// @param status query []int false "List environments(endpoints) by this status"
// @param types query []int false "List environments(endpoints) of this type"
// @param tagIds query []int false "search environments(endpoints) with these tags (depends on tagsPartialMatch)"
// @param tagsPartialMatch query bool false "If true, will return environment(endpoint) which has one of tagIds, if false (or missing) will return only environments(endpoints) that has all the tags"
// @param endpointIds query []int false "will return only these environments(endpoints)"
// @param edgeDeviceFilter query string false "will return only these edge environments, none will return only regular edge environments" Enum("all", "trusted", "untrusted", "none")
// @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned"
// @param edgeDevice query bool false "if exists true show only edge devices, false show only regular edge endpoints. if missing, will show both types (relevant only for edge endpoints)"
// @param edgeDeviceUntrusted query bool false "if true, show only untrusted endpoints, if false show only trusted (relevant only for edge devices, and if edgeDevice is true)"
// @param name query string false "will return only environments(endpoints) with this name"
// @success 200 {array} portainer.Endpoint "Endpoints"
// @failure 500 "Server error"
@ -60,103 +55,42 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
start--
}
search, _ := request.RetrieveQueryParameter(r, "search", true)
if search != "" {
search = strings.ToLower(search)
}
groupID, _ := request.RetrieveNumericQueryParameter(r, "groupId", true)
limit, _ := request.RetrieveNumericQueryParameter(r, "limit", true)
sortField, _ := request.RetrieveQueryParameter(r, "sort", true)
sortOrder, _ := request.RetrieveQueryParameter(r, "order", true)
var endpointTypes []int
request.RetrieveJSONQueryParameter(r, "types", &endpointTypes, true)
var tagIDs []portainer.TagID
request.RetrieveJSONQueryParameter(r, "tagIds", &tagIDs, true)
tagsPartialMatch, _ := request.RetrieveBooleanQueryParameter(r, "tagsPartialMatch", true)
var endpointIDs []portainer.EndpointID
request.RetrieveJSONQueryParameter(r, "endpointIds", &endpointIDs, true)
var statuses []int
request.RetrieveJSONQueryParameter(r, "status", &statuses, true)
var groupIDs []int
request.RetrieveJSONQueryParameter(r, "groupIds", &groupIDs, true)
endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environment groups from the database", err}
return httperror.InternalServerError("Unable to retrieve environment groups from the database", err)
}
endpoints, err := handler.DataStore.Endpoint().Endpoints()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve environments from the database", err}
return httperror.InternalServerError("Unable to retrieve environments from the database", err)
}
settings, err := handler.DataStore.Settings().Settings()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
return httperror.InternalServerError("Unable to retrieve settings from the database", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err}
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
query, err := parseQuery(r)
if err != nil {
return httperror.BadRequest("Invalid query parameters", err)
}
filteredEndpoints := security.FilterEndpoints(endpoints, endpointGroups, securityContext)
totalAvailableEndpoints := len(filteredEndpoints)
if groupID != 0 {
filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, []int{groupID})
filteredEndpoints, totalAvailableEndpoints, err := handler.filterEndpointsByQuery(filteredEndpoints, query, endpointGroups, settings)
if err != nil {
return httperror.InternalServerError("Unable to filter endpoints", err)
}
if endpointIDs != nil {
filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, endpointIDs)
}
if len(groupIDs) > 0 {
filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, groupIDs)
}
name, _ := request.RetrieveQueryParameter(r, "name", true)
if name != "" {
filteredEndpoints = filterEndpointsByName(filteredEndpoints, name)
}
edgeDeviceFilter, _ := request.RetrieveQueryParameter(r, "edgeDeviceFilter", false)
if edgeDeviceFilter != "" {
filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, edgeDeviceFilter)
}
if len(statuses) > 0 {
filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, statuses, settings)
}
if search != "" {
tags, err := handler.DataStore.Tag().Tags()
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve tags from the database", err}
}
tagsMap := make(map[portainer.TagID]string)
for _, tag := range tags {
tagsMap[tag.ID] = tag.Name
}
filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, endpointGroups, tagsMap, search)
}
if endpointTypes != nil {
filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, endpointTypes)
}
if tagIDs != nil {
filteredEndpoints = filteredEndpointsByTags(filteredEndpoints, tagIDs, endpointGroups, tagsPartialMatch)
}
// Sort endpoints by field
sortEndpointsByField(filteredEndpoints, endpointGroups, sortField, sortOrder == "desc")
filteredEndpointCount := len(filteredEndpoints)
@ -196,64 +130,6 @@ func paginateEndpoints(endpoints []portainer.Endpoint, start, limit int) []porta
return endpoints[start:end]
}
func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []int) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if utils.Contains(endpointGroupIDs, int(endpoint.GroupID)) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsBySearchCriteria(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
endpointTags := convertTagIDsToTags(tagsMap, endpoint.TagIDs)
if endpointMatchSearchCriteria(&endpoint, endpointTags, searchCriteria) {
filteredEndpoints = append(filteredEndpoints, endpoint)
continue
}
if endpointGroupMatchSearchCriteria(&endpoint, endpointGroups, tagsMap, searchCriteria) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []int, settings *portainer.Settings) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
status := endpoint.Status
if endpointutils.IsEdgeEndpoint(&endpoint) {
isCheckValid := false
edgeCheckinInterval := endpoint.EdgeCheckinInterval
if endpoint.EdgeCheckinInterval == 0 {
edgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 {
isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd)
}
status = portainer.EndpointStatusDown // Offline
if isCheckValid {
status = portainer.EndpointStatusUp // Online
}
}
if utils.Contains(statuses, int(status)) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func sortEndpointsByField(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, sortField string, isSortDesc bool) {
switch sortField {
@ -294,123 +170,6 @@ func sortEndpointsByField(endpoints []portainer.Endpoint, endpointGroups []porta
}
}
func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, searchCriteria string) bool {
if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) {
return true
}
if strings.Contains(strings.ToLower(endpoint.URL), searchCriteria) {
return true
}
if endpoint.Status == portainer.EndpointStatusUp && searchCriteria == "up" {
return true
} else if endpoint.Status == portainer.EndpointStatusDown && searchCriteria == "down" {
return true
}
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
return true
}
}
return false
}
func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
for _, group := range endpointGroups {
if group.ID == endpoint.GroupID {
if strings.Contains(strings.ToLower(group.Name), searchCriteria) {
return true
}
tags := convertTagIDsToTags(tagsMap, group.TagIDs)
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
return true
}
}
}
}
return false
}
func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []int) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
typeSet := map[portainer.EndpointType]bool{}
for _, endpointType := range endpointTypes {
typeSet[portainer.EndpointType(endpointType)] = true
}
for _, endpoint := range endpoints {
if typeSet[endpoint.Type] {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDeviceFilter string) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if shouldReturnEdgeDevice(endpoint, edgeDeviceFilter) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceFilter string) bool {
// none - return all endpoints that are not edge devices
if edgeDeviceFilter == EdgeDeviceFilterNone && !endpoint.IsEdgeDevice {
return true
}
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return false
}
switch edgeDeviceFilter {
case EdgeDeviceFilterAll:
return true
case EdgeDeviceFilterTrusted:
return endpoint.UserTrusted
case EdgeDeviceFilterUntrusted:
return !endpoint.UserTrusted
}
return false
}
func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
tags := make([]string, 0)
for _, tagID := range tagIDs {
tags = append(tags, tagsMap[tagID])
}
return tags
}
func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
endpointGroup := getEndpointGroup(endpoint.GroupID, endpointGroups)
endpointMatched := false
if partialMatch {
endpointMatched = endpointPartialMatchTags(endpoint, endpointGroup, tagIDs)
} else {
endpointMatched = endpointFullMatchTags(endpoint, endpointGroup, tagIDs)
}
if endpointMatched {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.EndpointGroup) portainer.EndpointGroup {
var endpointGroup portainer.EndpointGroup
for _, group := range groups {
@ -421,72 +180,3 @@ func getEndpointGroup(groupID portainer.EndpointGroupID, groups []portainer.Endp
}
return endpointGroup
}
func endpointPartialMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
tagSet := make(map[portainer.TagID]bool)
for _, tagID := range tagIDs {
tagSet[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if tagSet[tagID] {
return true
}
}
for _, tagID := range endpointGroup.TagIDs {
if tagSet[tagID] {
return true
}
}
return false
}
func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
missingTags := make(map[portainer.TagID]bool)
for _, tagID := range tagIDs {
missingTags[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
for _, tagID := range endpointGroup.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
return len(missingTags) == 0
}
func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.EndpointID) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
idsSet := make(map[portainer.EndpointID]bool)
for _, id := range ids {
idsSet[id] = true
}
for _, endpoint := range endpoints {
if idsSet[endpoint.ID] {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint {
if name == "" {
return endpoints
}
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if endpoint.Name == name {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}

View file

@ -16,66 +16,64 @@ import (
"github.com/stretchr/testify/assert"
)
type endpointListEdgeDeviceTest struct {
type endpointListTest struct {
title string
expected []portainer.EndpointID
filter string
}
func Test_endpointList(t *testing.T) {
var err error
is := assert.New(t)
func Test_endpointList_edgeDeviceFilter(t *testing.T) {
_, store, teardown := datastore.MustNewTestStore(true, true)
defer teardown()
trustedEndpoint := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEndpoint := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.DockerEnvironment}
endpoints := []portainer.Endpoint{
trustedEndpoint,
untrustedEndpoint,
handler, teardown := setup(t, []portainer.Endpoint{
trustedEdgeDevice,
untrustedEdgeDevice,
regularUntrustedEdgeEndpoint,
regularTrustedEdgeEndpoint,
regularEndpoint,
})
defer teardown()
type endpointListEdgeDeviceTest struct {
endpointListTest
edgeDevice *bool
edgeDeviceUntrusted bool
}
for _, endpoint := range endpoints {
err = store.Endpoint().Create(&endpoint)
is.NoError(err, "error creating environment")
}
err = store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
is.NoError(err, "error creating a user")
bouncer := helper.NewTestRequestBouncer()
h := NewHandler(bouncer, nil)
h.DataStore = store
h.ComposeStackManager = testhelpers.NewComposeStackManager()
tests := []endpointListEdgeDeviceTest{
{
"should show all edge endpoints",
[]portainer.EndpointID{trustedEndpoint.ID, untrustedEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
EdgeDeviceFilterAll,
endpointListTest: endpointListTest{
"should show all endpoints expect of the untrusted devices",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID, regularEndpoint.ID},
},
edgeDevice: nil,
},
{
"should show only trusted edge devices",
[]portainer.EndpointID{trustedEndpoint.ID, regularTrustedEdgeEndpoint.ID},
EdgeDeviceFilterTrusted,
endpointListTest: endpointListTest{
"should show only trusted edge devices and regular endpoints",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
},
edgeDevice: BoolAddr(true),
},
{
"should show only untrusted edge devices",
[]portainer.EndpointID{untrustedEndpoint.ID, regularUntrustedEdgeEndpoint.ID},
EdgeDeviceFilterUntrusted,
endpointListTest: endpointListTest{
"should show only untrusted edge devices and regular endpoints",
[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
},
edgeDevice: BoolAddr(true),
edgeDeviceUntrusted: true,
},
{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
EdgeDeviceFilterNone,
endpointListTest: endpointListTest{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
},
edgeDevice: BoolAddr(false),
},
}
@ -83,8 +81,13 @@ func Test_endpointList(t *testing.T) {
t.Run(test.title, func(t *testing.T) {
is := assert.New(t)
req := buildEndpointListRequest(test.filter)
resp, err := doEndpointListRequest(req, h, is)
query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted)
if test.edgeDevice != nil {
query += fmt.Sprintf("edgeDevice=%v&", *test.edgeDevice)
}
req := buildEndpointListRequest(query)
resp, err := doEndpointListRequest(req, handler, is)
is.NoError(err)
is.Equal(len(test.expected), len(resp))
@ -100,8 +103,28 @@ func Test_endpointList(t *testing.T) {
}
}
func buildEndpointListRequest(filter string) *http.Request {
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/endpoints?edgeDeviceFilter=%s", filter), nil)
func setup(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, teardown func()) {
is := assert.New(t)
_, store, teardown := datastore.MustNewTestStore(true, true)
for _, endpoint := range endpoints {
err := store.Endpoint().Create(&endpoint)
is.NoError(err, "error creating environment")
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
is.NoError(err, "error creating a user")
bouncer := helper.NewTestRequestBouncer()
handler = NewHandler(bouncer, nil)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()
return handler, teardown
}
func buildEndpointListRequest(query string) *http.Request {
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/endpoints?%s", query), nil)
ctx := security.StoreTokenData(req, &portainer.TokenData{ID: 1, Username: "admin", Role: 1})
req = req.WithContext(ctx)

View file

@ -0,0 +1,415 @@
package endpoints
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
"github.com/portainer/libhttp/request"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/endpointutils"
"golang.org/x/exp/slices"
)
type EnvironmentsQuery struct {
search string
types []portainer.EndpointType
tagIds []portainer.TagID
endpointIds []portainer.EndpointID
tagsPartialMatch bool
groupIds []portainer.EndpointGroupID
status []portainer.EndpointStatus
edgeDevice *bool
edgeDeviceUntrusted bool
name string
}
func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
search, _ := request.RetrieveQueryParameter(r, "search", true)
if search != "" {
search = strings.ToLower(search)
}
status, err := getNumberArrayQueryParameter[portainer.EndpointStatus](r, "status")
if err != nil {
return EnvironmentsQuery{}, err
}
groupIDs, err := getNumberArrayQueryParameter[portainer.EndpointGroupID](r, "groupIds")
if err != nil {
return EnvironmentsQuery{}, err
}
endpointTypes, err := getNumberArrayQueryParameter[portainer.EndpointType](r, "types")
if err != nil {
return EnvironmentsQuery{}, err
}
tagIDs, err := getNumberArrayQueryParameter[portainer.TagID](r, "tagIds")
if err != nil {
return EnvironmentsQuery{}, err
}
tagsPartialMatch, _ := request.RetrieveBooleanQueryParameter(r, "tagsPartialMatch", true)
endpointIDs, err := getNumberArrayQueryParameter[portainer.EndpointID](r, "endpointIds")
if err != nil {
return EnvironmentsQuery{}, err
}
name, _ := request.RetrieveQueryParameter(r, "name", true)
edgeDeviceParam, _ := request.RetrieveQueryParameter(r, "edgeDevice", true)
var edgeDevice *bool
if edgeDeviceParam != "" {
edgeDevice = BoolAddr(edgeDeviceParam == "true")
}
edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true)
return EnvironmentsQuery{
search: search,
types: endpointTypes,
tagIds: tagIDs,
endpointIds: endpointIDs,
tagsPartialMatch: tagsPartialMatch,
groupIds: groupIDs,
status: status,
edgeDevice: edgeDevice,
edgeDeviceUntrusted: edgeDeviceUntrusted,
name: name,
}, nil
}
func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.Endpoint, query EnvironmentsQuery, groups []portainer.EndpointGroup, settings *portainer.Settings) ([]portainer.Endpoint, int, error) {
totalAvailableEndpoints := len(filteredEndpoints)
if len(query.endpointIds) > 0 {
filteredEndpoints = filteredEndpointsByIds(filteredEndpoints, query.endpointIds)
}
if len(query.groupIds) > 0 {
filteredEndpoints = filterEndpointsByGroupIDs(filteredEndpoints, query.groupIds)
}
if query.name != "" {
filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
}
if query.edgeDevice != nil {
filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, *query.edgeDevice, query.edgeDeviceUntrusted)
} else {
// If the edgeDevice parameter is not set, we need to filter out the untrusted edge devices
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return !endpoint.IsEdgeDevice || endpoint.UserTrusted
})
}
if len(query.status) > 0 {
filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings)
}
if query.search != "" {
tags, err := handler.DataStore.Tag().Tags()
if err != nil {
return nil, 0, errors.WithMessage(err, "Unable to retrieve tags from the database")
}
tagsMap := make(map[portainer.TagID]string)
for _, tag := range tags {
tagsMap[tag.ID] = tag.Name
}
filteredEndpoints = filterEndpointsBySearchCriteria(filteredEndpoints, groups, tagsMap, query.search)
}
if len(query.types) > 0 {
filteredEndpoints = filterEndpointsByTypes(filteredEndpoints, query.types)
}
if len(query.tagIds) > 0 {
filteredEndpoints = filteredEndpointsByTags(filteredEndpoints, query.tagIds, groups, query.tagsPartialMatch)
}
return filteredEndpoints, totalAvailableEndpoints, nil
}
func filterEndpointsByGroupIDs(endpoints []portainer.Endpoint, endpointGroupIDs []portainer.EndpointGroupID) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if slices.Contains(endpointGroupIDs, endpoint.GroupID) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsBySearchCriteria(endpoints []portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
endpointTags := convertTagIDsToTags(tagsMap, endpoint.TagIDs)
if endpointMatchSearchCriteria(&endpoint, endpointTags, searchCriteria) {
filteredEndpoints = append(filteredEndpoints, endpoint)
continue
}
if endpointGroupMatchSearchCriteria(&endpoint, endpointGroups, tagsMap, searchCriteria) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portainer.EndpointStatus, settings *portainer.Settings) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
status := endpoint.Status
if endpointutils.IsEdgeEndpoint(&endpoint) {
isCheckValid := false
edgeCheckinInterval := endpoint.EdgeCheckinInterval
if endpoint.EdgeCheckinInterval == 0 {
edgeCheckinInterval = settings.EdgeAgentCheckinInterval
}
if edgeCheckinInterval != 0 && endpoint.LastCheckInDate != 0 {
isCheckValid = time.Now().Unix()-endpoint.LastCheckInDate <= int64(edgeCheckinInterval*EdgeDeviceIntervalMultiplier+EdgeDeviceIntervalAdd)
}
status = portainer.EndpointStatusDown // Offline
if isCheckValid {
status = portainer.EndpointStatusUp // Online
}
}
if slices.Contains(statuses, status) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, searchCriteria string) bool {
if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) {
return true
}
if strings.Contains(strings.ToLower(endpoint.URL), searchCriteria) {
return true
}
if endpoint.Status == portainer.EndpointStatusUp && searchCriteria == "up" {
return true
} else if endpoint.Status == portainer.EndpointStatusDown && searchCriteria == "down" {
return true
}
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
return true
}
}
return false
}
func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
for _, group := range endpointGroups {
if group.ID == endpoint.GroupID {
if strings.Contains(strings.ToLower(group.Name), searchCriteria) {
return true
}
tags := convertTagIDsToTags(tagsMap, group.TagIDs)
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
return true
}
}
}
}
return false
}
func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []portainer.EndpointType) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
typeSet := map[portainer.EndpointType]bool{}
for _, endpointType := range endpointTypes {
typeSet[portainer.EndpointType(endpointType)] = true
}
for _, endpoint := range endpoints {
if typeSet[endpoint.Type] {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDevice bool, untrusted bool) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if shouldReturnEdgeDevice(endpoint, edgeDevice, untrusted) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceParam bool, untrustedParam bool) bool {
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
if !edgeDeviceParam {
return !endpoint.IsEdgeDevice
}
return endpoint.IsEdgeDevice && endpoint.UserTrusted == !untrustedParam
}
func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
tags := make([]string, 0)
for _, tagID := range tagIDs {
tags = append(tags, tagsMap[tagID])
}
return tags
}
func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
endpointGroup := getEndpointGroup(endpoint.GroupID, endpointGroups)
endpointMatched := false
if partialMatch {
endpointMatched = endpointPartialMatchTags(endpoint, endpointGroup, tagIDs)
} else {
endpointMatched = endpointFullMatchTags(endpoint, endpointGroup, tagIDs)
}
if endpointMatched {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func endpointPartialMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
tagSet := make(map[portainer.TagID]bool)
for _, tagID := range tagIDs {
tagSet[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if tagSet[tagID] {
return true
}
}
for _, tagID := range endpointGroup.TagIDs {
if tagSet[tagID] {
return true
}
}
return false
}
func endpointFullMatchTags(endpoint portainer.Endpoint, endpointGroup portainer.EndpointGroup, tagIDs []portainer.TagID) bool {
missingTags := make(map[portainer.TagID]bool)
for _, tagID := range tagIDs {
missingTags[tagID] = true
}
for _, tagID := range endpoint.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
for _, tagID := range endpointGroup.TagIDs {
if missingTags[tagID] {
delete(missingTags, tagID)
}
}
return len(missingTags) == 0
}
func filteredEndpointsByIds(endpoints []portainer.Endpoint, ids []portainer.EndpointID) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
idsSet := make(map[portainer.EndpointID]bool)
for _, id := range ids {
idsSet[id] = true
}
for _, endpoint := range endpoints {
if idsSet[endpoint.ID] {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filterEndpointsByName(endpoints []portainer.Endpoint, name string) []portainer.Endpoint {
if name == "" {
return endpoints
}
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if endpoint.Name == name {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func filter(endpoints []portainer.Endpoint, predicate func(endpoint portainer.Endpoint) bool) []portainer.Endpoint {
filteredEndpoints := make([]portainer.Endpoint, 0)
for _, endpoint := range endpoints {
if predicate(endpoint) {
filteredEndpoints = append(filteredEndpoints, endpoint)
}
}
return filteredEndpoints
}
func getArrayQueryParameter(r *http.Request, parameter string) []string {
list, exists := r.Form[fmt.Sprintf("%s[]", parameter)]
if !exists {
list = []string{}
}
return list
}
func getNumberArrayQueryParameter[T ~int](r *http.Request, parameter string) ([]T, error) {
list := getArrayQueryParameter(r, parameter)
if list == nil {
return []T{}, nil
}
var result []T
for _, item := range list {
number, err := strconv.Atoi(item)
if err != nil {
return nil, errors.Wrapf(err, "Unable to parse parameter %s", parameter)
}
result = append(result, T(number))
}
return result, nil
}

View file

@ -0,0 +1,119 @@
package endpoints
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
helper "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
)
type filterTest struct {
title string
expected []portainer.EndpointID
query EnvironmentsQuery
}
func Test_Filter_edgeDeviceFilter(t *testing.T) {
trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
endpoints := []portainer.Endpoint{
trustedEdgeDevice,
untrustedEdgeDevice,
regularUntrustedEdgeEndpoint,
regularTrustedEdgeEndpoint,
regularEndpoint,
}
handler, teardown := setupFilterTest(t, endpoints)
defer teardown()
tests := []filterTest{
{
"should show all edge endpoints except of the untrusted devices",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
EnvironmentsQuery{
types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment},
},
},
{
"should show only trusted edge devices and other regular endpoints",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(true),
},
},
{
"should show only untrusted edge devices and other regular endpoints",
[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(true),
edgeDeviceUntrusted: true,
},
},
{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(false),
},
},
}
runTests(tests, t, handler, endpoints)
}
func runTests(tests []filterTest, t *testing.T, handler *Handler, endpoints []portainer.Endpoint) {
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {
runTest(t, test, handler, endpoints)
})
}
}
func runTest(t *testing.T, test filterTest, handler *Handler, endpoints []portainer.Endpoint) {
is := assert.New(t)
filteredEndpoints, _, err := handler.filterEndpointsByQuery(endpoints, test.query, []portainer.EndpointGroup{}, &portainer.Settings{})
is.NoError(err)
is.Equal(len(test.expected), len(filteredEndpoints))
respIds := []portainer.EndpointID{}
for _, endpoint := range filteredEndpoints {
respIds = append(respIds, endpoint.ID)
}
is.ElementsMatch(test.expected, respIds)
}
func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, teardown func()) {
is := assert.New(t)
_, store, teardown := datastore.MustNewTestStore(true, true)
for _, endpoint := range endpoints {
err := store.Endpoint().Create(&endpoint)
is.NoError(err, "error creating environment")
}
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
is.NoError(err, "error creating a user")
bouncer := helper.NewTestRequestBouncer()
handler = NewHandler(bouncer, nil)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()
return handler, teardown
}

View file

@ -0,0 +1,6 @@
package endpoints
func BoolAddr(b bool) *bool {
boolVar := b
return &boolVar
}