feat(api-key/backend): introducing support for api-key based auth EE-978 (#6079)

* feat(access-token): Multi-auth middleware support EE-1891 (#5936) * AnyAuth middleware initial implementation with tests * using mux.MiddlewareFunc instead of custom definition * removed redundant comments * - ExtractBearerToken bouncer func made private - changed helm token handling functionality to use jwt service to convert token to jwt string - updated tests - fixed helm list broken test due to missing token in request context * rename mwCheckAuthentication -> mwCheckJWTAuthentication * - introduce initial api-key auth support using X-API-KEY header - added tests to validate x-api-key request header presence * updated core mwAuthenticatedUser middleware to support multiple auth paradigms * - simplified anyAuth middleware - enforcing authmiddleware to implement verificationFunc interface - created tests for middleware * simplify bouncer Co-authored-by: Dmitry Salakhov <to@dimasalakhov.com> * feat(api-key): user-access-token generation endpoint EE-1889 EE-1888 EE-1895 (#6012) * user-access-token generation endpoint * fix comment * - introduction of apikey service - seperation of repository from service logic - called in handler * fixed tests * - fixed api key prefix - added tests * added another test for digest matching * updated swagger spec for access token creation * api key response returns raw key and struct - easing testability * test for api key prefix length * added another TODO to middleware * - api-key prefix rune -> string (rune does not auto-encode when response sent back to client) - digest -> pointer as we want to allow nil values and omit digest in responses (when nil) * - updated apikey struct - updated apikey service to support all common operations - updated apikey repo - integration of apikey service into bouncer - added test for all apikey service functions - boilerplate code for apikey service integration * - user access token generation tests - apiKeyLookup updated to support query params - added api-key tests for query params - added api-key tests for apiKeyLookup * get and remove access token handlers * get and remove access token handler tests * - delete user deletes all associated api keys - tests for this functionality * removed redundant []byte cast * automatic api-key eviction set within cache for 1 hour * fixed bug with loop var using final value * fixed service comment * ignore bolt error responses * case-insensitive query param check * simplified query var assignment * - added GetAPIKey func to get by unique id - updated DeleteAPIKey func to not require user ID - updated tests * GenerateRandomKey helper func from github.com/gorilla/securecookie moved to codebase * json response casing for api-keys fixed * updating api-key will update the cache * updated golang LRU cache * using hashicorps golang-LRU cache for api keys * simplified jwt check in create user access token * fixed api-key update logic on cache miss * Prefix generated api-keys with `ptr_` (#6067) * prefix api-keys with 'ptr_' * updated apikey description * refactor Co-authored-by: Dmitry Salakhov <to@dimasalakhov.com> * helm list test refactor * fixed user delete test * reduce test nil pointer errors * using correct http 201 created status code for token creation; updated tests * fixed swagger doc user id path param for user access token based endpoints * added api-key security openapi spec to existing jwt secured endpoints (#6091) * fixed flaky test * apikey datecreated and lastused attrs converted to unix timestamp * feat(user): added access token datatable. (#6124) * feat(user): added access token datatable. * feat(tokens): only display lastUsed time when it is not the default date * Update app/portainer/views/account/accountController.js Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * Update app/portainer/views/account/accountController.js Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * Update app/portainer/views/account/accountController.js Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * Update app/portainer/components/datatables/access-tokens-datatable/accessTokensDatatableController.js Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * Update app/portainer/services/api/userService.js Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * feat(improvements): proposed datatable improvements to speed up dev time (#6138) * modal code update * updated datatable filenames, updated controller to be default class export * fix(access-token): code improvement. Co-authored-by: zees-dev <63374656+zees-dev@users.noreply.github.com> * feat(apikeys): create access token view initial implementation EE-1886 (#6129) * CopyButton implementation * Code component implementation * ToolTip component migration to another folder * TextTip component implementation - continued * form Heading component * Button component updated to be more dynamic * copybutton - small size * form control pass tip error * texttip small text * CreateAccessToken react feature initial implementation * create user access token angularjs view implementation * registration of CreateAccessToken component in AngularJS * user token generation API request moved to angular service, method passed down instead * consistent naming of access token operations; clustered similar code together * any user can add access token * create access token page routing * moved code component to the correct location * removed isadmin check as all functionality applicable to all users * create access token angular view moved up a level * fixed PR issues, updated PR * addressed PR issues/improvements * explicit hr for horizontal line * fixed merge conflict storybook build breaking * - apikey test - cache test * addressed testing issues: - description validations - remove token description link on table * fix(api-keys): user role change evicts user keys in cache EE-2113 (#6168) * user role change evicts user api keys in cache * EvictUserKeyCache -> InvalidateUserKeyCache * godoc for InvalidateUserKeyCache func * additional test line * disable add access token button after adding token to prevent spam Co-authored-by: Dmitry Salakhov <to@dimasalakhov.com> Co-authored-by: fhanportainer <79428273+fhanportainer@users.noreply.github.com>
2025-08-02 20:35:25 +02:00 · 2021-11-30 15:31:16 +13:00 · 2021-11-30 15:31:16 +13:00 · 69c17986d9
commit 69c17986d9
parent 120584909c
197 changed files with 3137 additions and 91 deletions
--- a/api/http/security/bouncer.go
+++ b/api/http/security/bouncer.go
@ -4,9 +4,11 @@ import (
 	"errors"
 	"net/http"
 	"strings"
+	"time"

 	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/apikey"
 	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	httperrors "github.com/portainer/portainer/api/http/errors"
 )
@ -14,8 +16,9 @@ import (
 type (
 	// RequestBouncer represents an entity that manages API request accesses
 	RequestBouncer struct {
-		dataStore  portainer.DataStore
-		jwtService portainer.JWTService
+		dataStore     portainer.DataStore
+		jwtService    portainer.JWTService
+		apiKeyService apikey.APIKeyService
 	}

 	// RestrictedRequestContext is a data structure containing information
@ -26,13 +29,19 @@ type (
 		UserID          portainer.UserID
 		UserMemberships []portainer.TeamMembership
 	}
+
+	// tokenLookup looks up a token in the request
+	tokenLookup func(*http.Request) *portainer.TokenData
 )

+const apiKeyHeader = "X-API-KEY"
+
 // NewRequestBouncer initializes a new RequestBouncer
-func NewRequestBouncer(dataStore portainer.DataStore, jwtService portainer.JWTService) *RequestBouncer {
+func NewRequestBouncer(dataStore portainer.DataStore, jwtService portainer.JWTService, apiKeyService apikey.APIKeyService) *RequestBouncer {
 	return &RequestBouncer{
-		dataStore:  dataStore,
-		jwtService: jwtService,
+		dataStore:     dataStore,
+		jwtService:    jwtService,
+		apiKeyService: apiKeyService,
 	}
 }

@ -128,11 +137,14 @@ func (bouncer *RequestBouncer) AuthorizedEdgeEndpointOperation(r *http.Request,
 	return nil
 }

-// handlers are applied backwards to the incoming request:
-// - add secure handlers to the response
-// - parse the JWT token and put it into the http context.
+// mwAuthenticatedUser authenticates a request by
+// - adding a secure handlers to the response
+// - authenticating the request with a valid token
 func (bouncer *RequestBouncer) mwAuthenticatedUser(h http.Handler) http.Handler {
-	h = bouncer.mwCheckAuthentication(h)
+	h = bouncer.mwAuthenticateFirst([]tokenLookup{
+		bouncer.JWTAuthLookup,
+		bouncer.apiKeyLookup,
+	}, h)
 	h = mwSecureHeaders(h)
 	return h
 }
@ -193,42 +205,90 @@ func (bouncer *RequestBouncer) mwUpgradeToRestrictedRequest(next http.Handler) h
 	})
 }

-// mwCheckAuthentication provides Authentication middleware for handlers
-//
-// It parses the JWT token and adds the parsed token data to the http context
-func (bouncer *RequestBouncer) mwCheckAuthentication(next http.Handler) http.Handler {
+// mwAuthenticateFirst authenticates a request an auth token.
+// A result of a first succeded token lookup would be used for the authentication.
+func (bouncer *RequestBouncer) mwAuthenticateFirst(tokenLookups []tokenLookup, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		var tokenData *portainer.TokenData
+		var token *portainer.TokenData

-		// get token from the Authorization header or query parameter
-		token, err := ExtractBearerToken(r)
-		if err != nil {
-			httperror.WriteError(w, http.StatusUnauthorized, "Unauthorized", err)
+		for _, lookup := range tokenLookups {
+			token = lookup(r)
+
+			if token != nil {
+				break
+			}
+		}
+
+		if token == nil {
+			httperror.WriteError(w, http.StatusUnauthorized, "A valid authorisation token is missing", httperrors.ErrUnauthorized)
 			return
 		}

-		tokenData, err = bouncer.jwtService.ParseAndVerifyToken(token)
-		if err != nil {
-			httperror.WriteError(w, http.StatusUnauthorized, "Invalid JWT token", err)
+		user, _ := bouncer.dataStore.User().User(token.ID)
+		if user == nil {
+			httperror.WriteError(w, http.StatusUnauthorized, "An authorisation token is invalid", httperrors.ErrUnauthorized)
 			return
 		}

-		_, err = bouncer.dataStore.User().User(tokenData.ID)
-		if err != nil && err == bolterrors.ErrObjectNotFound {
-			httperror.WriteError(w, http.StatusUnauthorized, "Unauthorized", httperrors.ErrUnauthorized)
-			return
-		} else if err != nil {
-			httperror.WriteError(w, http.StatusInternalServerError, "Unable to retrieve user details from the database", err)
-			return
-		}
-
-		ctx := StoreTokenData(r, tokenData)
+		ctx := StoreTokenData(r, token)
 		next.ServeHTTP(w, r.WithContext(ctx))
 	})
 }

-// ExtractBearerToken extracts the Bearer token from the request header or query parameter and returns the token.
-func ExtractBearerToken(r *http.Request) (string, error) {
+// JWTAuthLookup looks up a valid bearer in the request.
+func (bouncer *RequestBouncer) JWTAuthLookup(r *http.Request) *portainer.TokenData {
+	// get token from the Authorization header or query parameter
+	token, err := extractBearerToken(r)
+	if err != nil {
+		return nil
+	}
+
+	tokenData, err := bouncer.jwtService.ParseAndVerifyToken(token)
+	if err != nil {
+		return nil
+	}
+
+	return tokenData
+}
+
+// apiKeyLookup looks up an verifies an api-key by:
+// - computing the digest of the raw api-key
+// - verifying it exists in cache/database
+// - matching the key to a user (ID, Role)
+// If the key is valid/verified, the last updated time of the key is updated.
+// Successful verification of the key will return a TokenData object - since the downstream handlers
+// utilise the token injected in the request context.
+func (bouncer *RequestBouncer) apiKeyLookup(r *http.Request) *portainer.TokenData {
+	rawAPIKey, ok := extractAPIKey(r)
+	if !ok {
+		return nil
+	}
+
+	digest := bouncer.apiKeyService.HashRaw(rawAPIKey)
+
+	user, apiKey, err := bouncer.apiKeyService.GetDigestUserAndKey(digest)
+	if err != nil {
+		return nil
+	}
+
+	tokenData := &portainer.TokenData{
+		ID:       user.ID,
+		Username: user.Username,
+		Role:     user.Role,
+	}
+	if _, err := bouncer.jwtService.GenerateToken(tokenData); err != nil {
+		return nil
+	}
+
+	// update the last used time of the key
+	apiKey.LastUsed = time.Now().UTC().Unix()
+	bouncer.apiKeyService.UpdateAPIKey(&apiKey)
+
+	return tokenData
+}
+
+// extractBearerToken extracts the Bearer token from the request header or query parameter and returns the token.
+func extractBearerToken(r *http.Request) (string, error) {
 	// Optionally, token might be set via the "token" query parameter.
 	// For example, in websocket requests
 	token := r.URL.Query().Get("token")
@ -244,6 +304,26 @@ func ExtractBearerToken(r *http.Request) (string, error) {
 	return token, nil
 }

+// extractAPIKey extracts the api key from the api key request header or query params.
+func extractAPIKey(r *http.Request) (apikey string, ok bool) {
+	// extract the API key from the request header
+	apikey = r.Header.Get(apiKeyHeader)
+	if apikey != "" {
+		return apikey, true
+	}
+
+	// extract the API key from query params.
+	// Case-insensitive check for the "X-API-KEY" query param.
+	query := r.URL.Query()
+	for k, v := range query {
+		if strings.EqualFold(k, apiKeyHeader) {
+			return v[0], true
+		}
+	}
+
+	return "", false
+}
+
 // mwSecureHeaders provides secure headers middleware for handlers.
 func mwSecureHeaders(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
--- a/api/http/security/bouncer_test.go
+++ b/api/http/security/bouncer_test.go
@ -0,0 +1,334 @@
+package security
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/apikey"
+	"github.com/portainer/portainer/api/bolt"
+	httperrors "github.com/portainer/portainer/api/http/errors"
+	"github.com/portainer/portainer/api/jwt"
+	"github.com/stretchr/testify/assert"
+)
+
+// testHandler200 is a simple handler which returns HTTP status 200 OK
+var testHandler200 = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+})
+
+func tokenLookupSucceed(dataStore portainer.DataStore, jwtService portainer.JWTService) tokenLookup {
+	return func(r *http.Request) *portainer.TokenData {
+		uid := portainer.UserID(1)
+		dataStore.User().CreateUser(&portainer.User{ID: uid})
+		jwtService.GenerateToken(&portainer.TokenData{ID: uid})
+		return &portainer.TokenData{ID: 1}
+	}
+}
+
+func tokenLookupFail(r *http.Request) *portainer.TokenData {
+	return nil
+}
+
+func Test_mwAuthenticateFirst(t *testing.T) {
+	is := assert.New(t)
+
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	jwtService, err := jwt.NewService("1h", store)
+	assert.NoError(t, err, "failed to create a copy of service")
+
+	apiKeyService := apikey.NewAPIKeyService(nil, nil)
+
+	bouncer := NewRequestBouncer(store, jwtService, apiKeyService)
+
+	tests := []struct {
+		name                   string
+		verificationMiddlwares []tokenLookup
+		wantStatusCode         int
+	}{
+		{
+			name:                   "mwAuthenticateFirst middleware passes with no middleware",
+			verificationMiddlwares: nil,
+			wantStatusCode:         http.StatusUnauthorized,
+		},
+		{
+			name: "mwAuthenticateFirst middleware succeeds with passing middleware",
+			verificationMiddlwares: []tokenLookup{
+				tokenLookupSucceed(store, jwtService),
+			},
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name: "mwAuthenticateFirst fails with failing middleware",
+			verificationMiddlwares: []tokenLookup{
+				tokenLookupFail,
+			},
+			wantStatusCode: http.StatusUnauthorized,
+		},
+		{
+			name: "mwAuthenticateFirst succeeds if first middleware successfully handles request",
+			verificationMiddlwares: []tokenLookup{
+				tokenLookupSucceed(store, jwtService),
+				tokenLookupFail,
+			},
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name: "mwAuthenticateFirst succeeds if last middleware successfully handles request",
+			verificationMiddlwares: []tokenLookup{
+				tokenLookupFail,
+				tokenLookupSucceed(store, jwtService),
+			},
+			wantStatusCode: http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rr := httptest.NewRecorder()
+
+			h := bouncer.mwAuthenticateFirst(tt.verificationMiddlwares, testHandler200)
+			h.ServeHTTP(rr, req)
+
+			is.Equal(tt.wantStatusCode, rr.Code, fmt.Sprintf("Status should be %d", tt.wantStatusCode))
+		})
+	}
+}
+
+func Test_extractBearerToken(t *testing.T) {
+	is := assert.New(t)
+
+	tt := []struct {
+		name               string
+		requestHeader      string
+		requestHeaderValue string
+		wantToken          string
+		succeeds           bool
+	}{
+		{
+			name:               "missing request header",
+			requestHeader:      "",
+			requestHeaderValue: "",
+			wantToken:          "",
+			succeeds:           false,
+		},
+		{
+			name:               "invalid authorization request header",
+			requestHeader:      "authorisation", // note: `s`
+			requestHeaderValue: "abc",
+			wantToken:          "",
+			succeeds:           false,
+		},
+		{
+			name:               "valid authorization request header",
+			requestHeader:      "AUTHORIZATION",
+			requestHeaderValue: "abc",
+			wantToken:          "abc",
+			succeeds:           true,
+		},
+		{
+			name:               "valid authorization request header case-insensitive canonical check",
+			requestHeader:      "authorization",
+			requestHeaderValue: "def",
+			wantToken:          "def",
+			succeeds:           true,
+		},
+	}
+
+	for _, test := range tt {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Set(test.requestHeader, test.requestHeaderValue)
+		apiKey, err := extractBearerToken(req)
+		is.Equal(test.wantToken, apiKey)
+		if !test.succeeds {
+			is.Error(err, "Should return error")
+			is.ErrorIs(err, httperrors.ErrUnauthorized)
+		} else {
+			is.NoError(err)
+		}
+	}
+}
+
+func Test_extractAPIKeyHeader(t *testing.T) {
+	is := assert.New(t)
+
+	tt := []struct {
+		name               string
+		requestHeader      string
+		requestHeaderValue string
+		wantApiKey         string
+		succeeds           bool
+	}{
+		{
+			name:               "missing request header",
+			requestHeader:      "",
+			requestHeaderValue: "",
+			wantApiKey:         "",
+			succeeds:           false,
+		},
+		{
+			name:               "invalid api-key request header",
+			requestHeader:      "api-key",
+			requestHeaderValue: "abc",
+			wantApiKey:         "",
+			succeeds:           false,
+		},
+		{
+			name:               "valid api-key request header",
+			requestHeader:      apiKeyHeader,
+			requestHeaderValue: "abc",
+			wantApiKey:         "abc",
+			succeeds:           true,
+		},
+		{
+			name:               "valid api-key request header case-insensitive canonical check",
+			requestHeader:      "x-api-key",
+			requestHeaderValue: "def",
+			wantApiKey:         "def",
+			succeeds:           true,
+		},
+	}
+
+	for _, test := range tt {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Set(test.requestHeader, test.requestHeaderValue)
+		apiKey, ok := extractAPIKey(req)
+		is.Equal(test.wantApiKey, apiKey)
+		is.Equal(test.succeeds, ok)
+	}
+}
+
+func Test_extractAPIKeyQueryParam(t *testing.T) {
+	is := assert.New(t)
+
+	tt := []struct {
+		name            string
+		queryParam      string
+		queryParamValue string
+		wantApiKey      string
+		succeeds        bool
+	}{
+		{
+			name:            "missing request header",
+			queryParam:      "",
+			queryParamValue: "",
+			wantApiKey:      "",
+			succeeds:        false,
+		},
+		{
+			name:            "invalid api-key request header",
+			queryParam:      "api-key",
+			queryParamValue: "abc",
+			wantApiKey:      "",
+			succeeds:        false,
+		},
+		{
+			name:            "valid api-key request header",
+			queryParam:      apiKeyHeader,
+			queryParamValue: "abc",
+			wantApiKey:      "abc",
+			succeeds:        true,
+		},
+		{
+			name:            "valid api-key request header case-insensitive canonical check",
+			queryParam:      "x-api-key",
+			queryParamValue: "def",
+			wantApiKey:      "def",
+			succeeds:        true,
+		},
+	}
+
+	for _, test := range tt {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		q := req.URL.Query()
+		q.Add(test.queryParam, test.queryParamValue)
+		req.URL.RawQuery = q.Encode()
+
+		apiKey, ok := extractAPIKey(req)
+		is.Equal(test.wantApiKey, apiKey)
+		is.Equal(test.succeeds, ok)
+	}
+}
+
+func Test_apiKeyLookup(t *testing.T) {
+	is := assert.New(t)
+
+	store, teardown := bolt.MustNewTestStore(true)
+	defer teardown()
+
+	// create standard user
+	user := &portainer.User{ID: 2, Username: "standard", Role: portainer.StandardUserRole}
+	err := store.User().CreateUser(user)
+	is.NoError(err, "error creating user")
+
+	// setup services
+	jwtService, err := jwt.NewService("1h", store)
+	is.NoError(err, "Error initiating jwt service")
+	apiKeyService := apikey.NewAPIKeyService(store.APIKeyRepository(), store.User())
+	bouncer := NewRequestBouncer(store, jwtService, apiKeyService)
+
+	t.Run("missing x-api-key header fails api-key lookup", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		// req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
+		token := bouncer.apiKeyLookup(req)
+		is.Nil(token)
+	})
+
+	t.Run("invalid x-api-key header fails api-key lookup", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Add("x-api-key", "random-failing-api-key")
+		token := bouncer.apiKeyLookup(req)
+		is.Nil(token)
+	})
+
+	t.Run("valid x-api-key header succeeds api-key lookup", func(t *testing.T) {
+		rawAPIKey, _, err := apiKeyService.GenerateApiKey(*user, "test")
+		is.NoError(err)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Add("x-api-key", rawAPIKey)
+
+		token := bouncer.apiKeyLookup(req)
+
+		expectedToken := &portainer.TokenData{ID: user.ID, Username: user.Username, Role: portainer.StandardUserRole}
+		is.Equal(expectedToken, token)
+	})
+
+	t.Run("valid x-api-key header succeeds api-key lookup", func(t *testing.T) {
+		rawAPIKey, apiKey, err := apiKeyService.GenerateApiKey(*user, "test")
+		is.NoError(err)
+		defer apiKeyService.DeleteAPIKey(apiKey.ID)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Add("x-api-key", rawAPIKey)
+
+		token := bouncer.apiKeyLookup(req)
+
+		expectedToken := &portainer.TokenData{ID: user.ID, Username: user.Username, Role: portainer.StandardUserRole}
+		is.Equal(expectedToken, token)
+	})
+
+	t.Run("successful api-key lookup updates token last used time", func(t *testing.T) {
+		rawAPIKey, apiKey, err := apiKeyService.GenerateApiKey(*user, "test")
+		is.NoError(err)
+		defer apiKeyService.DeleteAPIKey(apiKey.ID)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Add("x-api-key", rawAPIKey)
+
+		token := bouncer.apiKeyLookup(req)
+
+		expectedToken := &portainer.TokenData{ID: user.ID, Username: user.Username, Role: portainer.StandardUserRole}
+		is.Equal(expectedToken, token)
+
+		_, apiKeyUpdated, err := apiKeyService.GetDigestUserAndKey(apiKey.Digest)
+		is.NoError(err)
+
+		is.True(apiKeyUpdated.LastUsed > apiKey.LastUsed)
+	})
+}