Docker/portainer
Docker/portainer
1
0
Fork 0
mirror of https://github.com/portainer/portainer.git synced 2025-07-25 00:09:40 +02:00
Code Issues Releases Activity

feat(environments): create async edge [EE-4480] (#8527)

Browse source
This commit is contained in:
Chaim Lev-Ari 2023-03-01 20:33:05 +02:00 • committed by GitHub
parent bc6a667a6b
commit c819d4e7f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
59 changed files with 880 additions and 586 deletions
Download patch file Download diff file Expand all files Collapse all files

4
api/http/handler/endpoints/endpoint_create.go
View file

@ -164,9 +164,6 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
checkinInterval, _ := request.RetrieveNumericMultiPartFormValue(r, "CheckinInterval", true)
payload.EdgeCheckinInterval = checkinInterval
isEdgeDevice, _ := request.RetrieveBooleanMultiPartFormValue(r, "IsEdgeDevice", true)
payload.IsEdgeDevice = isEdgeDevice
return nil
}
@ -196,7 +193,6 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
// @param TagIDs formData []int false "List of tag identifiers to which this environment(endpoint) is associated"
// @param EdgeCheckinInterval formData int false "The check in interval for edge agent (in seconds)"
// @param EdgeTunnelServerAddress formData string true "URL or IP address that will be used to establish a reverse tunnel"
// @param IsEdgeDevice formData bool false "Is Edge Device"
// @param Gpus formData array false "List of GPUs"
// @success 200 {object} portainer.Endpoint "Success"
// @failure 400 "Invalid request"

4
api/http/handler/endpoints/endpoint_list.go
View file

@ -42,8 +42,8 @@ const (
// @param endpointIds query []int false "will return only these environments(endpoints)"
// @param provisioned query bool false "If true, will return environment(endpoint) that were provisioned"
// @param agentVersions query []string false "will return only environments with on of these agent versions"
// @param edgeDevice query bool false "if exists true show only edge devices, false show only regular edge endpoints. if missing, will show both types (relevant only for edge endpoints)"
// @param edgeDeviceUntrusted query bool false "if true, show only untrusted endpoints, if false show only trusted (relevant only for edge devices, and if edgeDevice is true)"
// @param edgeAsync query bool false "if exists true show only edge async agents, false show only standard edge agents. if missing, will show both types (relevant only for edge agents)"
// @param edgeDeviceUntrusted query bool false "if true, show only untrusted edge agents, if false show only trusted edge agents (relevant only for edge agents)"
// @param name query string false "will return only environments(endpoints) with this name"
// @success 200 {array} portainer.Endpoint "Endpoints"
// @failure 500 "Server error"

52
api/http/handler/endpoints/endpoint_list_test.go
View file

@ -13,7 +13,6 @@ import (
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/snapshot"
"github.com/portainer/portainer/api/internal/testhelpers"
helper "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
)
@ -104,59 +103,58 @@ func Test_EndpointList_AgentVersion(t *testing.T) {
}
}
func Test_endpointList_edgeDeviceFilter(t *testing.T) {
func Test_endpointList_edgeFilter(t *testing.T) {
trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.DockerEnvironment}
trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
handler, teardown := setup(t, []portainer.Endpoint{
trustedEdgeDevice,
untrustedEdgeDevice,
regularUntrustedEdgeEndpoint,
regularTrustedEdgeEndpoint,
trustedEdgeAsync,
untrustedEdgeAsync,
regularUntrustedEdgeStandard,
regularTrustedEdgeStandard,
regularEndpoint,
})
defer teardown()
type endpointListEdgeDeviceTest struct {
type endpointListEdgeTest struct {
endpointListTest
edgeDevice *bool
edgeAsync *bool
edgeDeviceUntrusted bool
}
tests := []endpointListEdgeDeviceTest{
tests := []endpointListEdgeTest{
{
endpointListTest: endpointListTest{
"should show all endpoints except of the untrusted devices",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID, regularEndpoint.ID},
"should show all endpoints expect of the untrusted devices",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID, regularEndpoint.ID},
},
edgeDevice: nil,
},
{
endpointListTest: endpointListTest{
"should show only trusted edge devices and regular endpoints",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
"should show only trusted edge async agents and regular endpoints",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
},
edgeDevice: BoolAddr(true),
edgeAsync: BoolAddr(true),
},
{
endpointListTest: endpointListTest{
"should show only untrusted edge devices and regular endpoints",
[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
},
edgeDevice: BoolAddr(true),
edgeAsync: BoolAddr(true),
edgeDeviceUntrusted: true,
},
{
endpointListTest: endpointListTest{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
},
edgeDevice: BoolAddr(false),
edgeAsync: BoolAddr(false),
},
}
@ -165,8 +163,8 @@ func Test_endpointList_edgeDeviceFilter(t *testing.T) {
is := assert.New(t)
query := fmt.Sprintf("edgeDeviceUntrusted=%v&", test.edgeDeviceUntrusted)
if test.edgeDevice != nil {
query += fmt.Sprintf("edgeDevice=%v&", *test.edgeDevice)
if test.edgeAsync != nil {
query += fmt.Sprintf("edgeAsync=%v&", *test.edgeAsync)
}
req := buildEndpointListRequest(query)
@ -198,7 +196,7 @@ func setup(t *testing.T, endpoints []portainer.Endpoint) (handler *Handler, tear
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
is.NoError(err, "error creating a user")
bouncer := helper.NewTestRequestBouncer()
bouncer := testhelpers.NewTestRequestBouncer()
handler = NewHandler(bouncer, nil)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()

73
api/http/handler/endpoints/filter.go
View file

@ -15,14 +15,15 @@ import (
)
type EnvironmentsQuery struct {
search string
types []portainer.EndpointType
tagIds []portainer.TagID
endpointIds []portainer.EndpointID
tagsPartialMatch bool
groupIds []portainer.EndpointGroupID
status []portainer.EndpointStatus
edgeDevice *bool
search string
types []portainer.EndpointType
tagIds []portainer.TagID
endpointIds []portainer.EndpointID
tagsPartialMatch bool
groupIds []portainer.EndpointGroupID
status []portainer.EndpointStatus
// if edgeAsync not nil, will filter edge endpoints based on this value
edgeAsync *bool
edgeDeviceUntrusted bool
excludeSnapshots bool
name string
@ -66,11 +67,10 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
name, _ := request.RetrieveQueryParameter(r, "name", true)
edgeDeviceParam, _ := request.RetrieveQueryParameter(r, "edgeDevice", true)
var edgeDevice *bool
if edgeDeviceParam != "" {
edgeDevice = BoolAddr(edgeDeviceParam == "true")
var edgeAsync *bool
edgeAsyncParam, _ := request.RetrieveQueryParameter(r, "edgeAsync", true)
if edgeAsyncParam != "" {
edgeAsync = BoolAddr(edgeAsyncParam == "true")
}
edgeDeviceUntrusted, _ := request.RetrieveBooleanQueryParameter(r, "edgeDeviceUntrusted", true)
@ -85,7 +85,7 @@ func parseQuery(r *http.Request) (EnvironmentsQuery, error) {
tagsPartialMatch: tagsPartialMatch,
groupIds: groupIDs,
status: status,
edgeDevice: edgeDevice,
edgeAsync: edgeAsync,
edgeDeviceUntrusted: edgeDeviceUntrusted,
excludeSnapshots: excludeSnapshots,
name: name,
@ -108,15 +108,26 @@ func (handler *Handler) filterEndpointsByQuery(filteredEndpoints []portainer.End
filteredEndpoints = filterEndpointsByName(filteredEndpoints, query.name)
}
if query.edgeDevice != nil {
filteredEndpoints = filterEndpointsByEdgeDevice(filteredEndpoints, *query.edgeDevice, query.edgeDeviceUntrusted)
} else {
// If the edgeDevice parameter is not set, we need to filter out the untrusted edge devices
// filter async edge environments
if query.edgeAsync != nil {
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
return !endpoint.IsEdgeDevice || endpoint.UserTrusted
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
return endpoint.Edge.AsyncMode == *query.edgeAsync
})
}
// filter edge environments by trusted/untrusted
filteredEndpoints = filter(filteredEndpoints, func(endpoint portainer.Endpoint) bool {
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
return endpoint.UserTrusted == !query.edgeDeviceUntrusted
})
if len(query.status) > 0 {
filteredEndpoints = filterEndpointsByStatuses(filteredEndpoints, query.status, settings)
}
@ -274,30 +285,6 @@ func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []port
return endpoints[:n]
}
func filterEndpointsByEdgeDevice(endpoints []portainer.Endpoint, edgeDevice bool, untrusted bool) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
if shouldReturnEdgeDevice(endpoint, edgeDevice, untrusted) {
endpoints[n] = endpoint
n++
}
}
return endpoints[:n]
}
func shouldReturnEdgeDevice(endpoint portainer.Endpoint, edgeDeviceParam bool, untrustedParam bool) bool {
if !endpointutils.IsEdgeEndpoint(&endpoint) {
return true
}
if !edgeDeviceParam {
return !endpoint.IsEdgeDevice
}
return endpoint.IsEdgeDevice && endpoint.UserTrusted == !untrustedParam
}
func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
tags := make([]string, 0, len(tagIDs))

37
api/http/handler/endpoints/filter_test.go
View file

@ -6,7 +6,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
"github.com/portainer/portainer/api/internal/testhelpers"
helper "github.com/portainer/portainer/api/internal/testhelpers"
"github.com/stretchr/testify/assert"
)
@ -74,19 +73,19 @@ func Test_Filter_AgentVersion(t *testing.T) {
runTests(tests, t, handler, endpoints)
}
func Test_Filter_edgeDeviceFilter(t *testing.T) {
func Test_Filter_edgeFilter(t *testing.T) {
trustedEdgeDevice := portainer.Endpoint{ID: 1, UserTrusted: true, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeDevice := portainer.Endpoint{ID: 2, UserTrusted: false, IsEdgeDevice: true, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeEndpoint := portainer.Endpoint{ID: 3, UserTrusted: false, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeEndpoint := portainer.Endpoint{ID: 4, UserTrusted: true, IsEdgeDevice: false, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
trustedEdgeAsync := portainer.Endpoint{ID: 1, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
untrustedEdgeAsync := portainer.Endpoint{ID: 2, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: true}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularUntrustedEdgeStandard := portainer.Endpoint{ID: 3, UserTrusted: false, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularTrustedEdgeStandard := portainer.Endpoint{ID: 4, UserTrusted: true, Edge: portainer.EnvironmentEdgeSettings{AsyncMode: false}, GroupID: 1, Type: portainer.EdgeAgentOnDockerEnvironment}
regularEndpoint := portainer.Endpoint{ID: 5, GroupID: 1, Type: portainer.DockerEnvironment}
endpoints := []portainer.Endpoint{
trustedEdgeDevice,
untrustedEdgeDevice,
regularUntrustedEdgeEndpoint,
regularTrustedEdgeEndpoint,
trustedEdgeAsync,
untrustedEdgeAsync,
regularUntrustedEdgeStandard,
regularTrustedEdgeStandard,
regularEndpoint,
}
@ -96,32 +95,32 @@ func Test_Filter_edgeDeviceFilter(t *testing.T) {
tests := []filterTest{
{
"should show all edge endpoints except of the untrusted devices",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
"should show all edge endpoints except of the untrusted edge",
[]portainer.EndpointID{trustedEdgeAsync.ID, regularTrustedEdgeStandard.ID},
EnvironmentsQuery{
types: []portainer.EndpointType{portainer.EdgeAgentOnDockerEnvironment, portainer.EdgeAgentOnKubernetesEnvironment},
},
},
{
"should show only trusted edge devices and other regular endpoints",
[]portainer.EndpointID{trustedEdgeDevice.ID, regularEndpoint.ID},
[]portainer.EndpointID{trustedEdgeAsync.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(true),
edgeAsync: BoolAddr(true),
},
},
{
"should show only untrusted edge devices and other regular endpoints",
[]portainer.EndpointID{untrustedEdgeDevice.ID, regularEndpoint.ID},
[]portainer.EndpointID{untrustedEdgeAsync.ID, regularEndpoint.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(true),
edgeAsync: BoolAddr(true),
edgeDeviceUntrusted: true,
},
},
{
"should show no edge devices",
[]portainer.EndpointID{regularEndpoint.ID, regularUntrustedEdgeEndpoint.ID, regularTrustedEdgeEndpoint.ID},
[]portainer.EndpointID{regularEndpoint.ID, regularTrustedEdgeStandard.ID},
EnvironmentsQuery{
edgeDevice: BoolAddr(false),
edgeAsync: BoolAddr(false),
},
},
}
@ -168,7 +167,7 @@ func setupFilterTest(t *testing.T, endpoints []portainer.Endpoint) (handler *Han
err := store.User().Create(&portainer.User{Username: "admin", Role: portainer.AdministratorRole})
is.NoError(err, "error creating a user")
bouncer := helper.NewTestRequestBouncer()
bouncer := testhelpers.NewTestRequestBouncer()
handler = NewHandler(bouncer, nil)
handler.DataStore = store
handler.ComposeStackManager = testhelpers.NewComposeStackManager()