feat(analytics): add apis for event tracking (#5298)

* feat(analytics): add apis for event tracking

feat(api): fetch instanceID

feat(state): set instance id and version on matomo

refactor(state): export validation of app state

feat(analytics): update dimensions

refactor(analytics): move matomo to module

feat(analytics): disable analytics on non production

feat(analytics): track event metadata

refactor(analytics): clean push function

refactor(analytics): rename init function

feat(analytics): track user role

feat(analytics): track user global role

fix(stacks): remove event tracking for stack create

* style(analytics): remove TODO

* feat(build): add testing env

api/cmd/portainer/main.go

@@ -181,9 +181,10 @@ func initSnapshotService(snapshotInterval string, dataStore portainer.DataStore,
 	return snapshotService, nil
 }
 
-func initStatus(flags *portainer.CLIFlags) *portainer.Status {
+func initStatus(instanceID string) *portainer.Status {
 	return &portainer.Status{
-		Version: portainer.APIVersion,
+		Version:    portainer.APIVersion,
+		InstanceID: instanceID,
 	}
 }
 
@@ -471,7 +472,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		log.Fatalf("failed loading edge jobs from database: %v", err)
 	}
 
-	applicationStatus := initStatus(flags)
+	applicationStatus := initStatus(instanceID)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {

api/portainer.go

@@ -761,6 +761,8 @@ type (
 	Status struct {
 		// Portainer API version
 		Version string `json:"Version" example:"2.0.0"`
+		// Server Instance ID
+		InstanceID string `example:"299ab403-70a8-4c05-92f7-bf7a994d50df"`
 	}
 
 	// Tag represents a tag that can be associated to a resource

app/__module.js

@@ -4,7 +4,7 @@ import '@babel/polyfill';
 import angular from 'angular';
 
 import './matomo-setup';
-import './assets/js/angulartics-matomo';
+import analyticsModule from './angulartics.matomo';
 
 import './agent';
 import './azure/_module';
@@ -39,7 +39,7 @@ angular.module('portainer', [
   'rzModule',
   'moment-picker',
   'angulartics',
-  'angulartics.matomo',
+  analyticsModule,
 ]);
 
 if (require) {

app/angulartics.matomo/index.js

@@ -0,0 +1,192 @@
+import angular from 'angular';
+
+const basePath = 'http://portainer-ce.app';
+
+const dimensions = {
+  PortainerVersion: 1,
+  PortainerInstanceID: 2,
+  PortainerUserRole: 3,
+  PortainerEndpointUserRole: 4,
+};
+
+const categories = ['docker', 'kubernetes', 'aci', 'portainer', 'edge'];
+
+// forked from https://github.com/angulartics/angulartics-piwik/blob/master/src/angulartics-piwik.js
+
+/**
+ * @ngdoc overview
+ * @name angulartics.piwik
+ * Enables analytics support for Piwik/Matomo (http://piwik.org/docs/tracking-api/)
+ */
+export default angular.module('angulartics.matomo', ['angulartics']).config(config).name;
+
+/* @ngInject */
+function config($analyticsProvider, $windowProvider) {
+  const $window = $windowProvider.$get();
+
+  $analyticsProvider.settings.pageTracking.trackRelativePath = true;
+
+  $analyticsProvider.api.setPortainerStatus = setPortainerStatus;
+
+  $analyticsProvider.api.setUserRole = setUserRole;
+  $analyticsProvider.api.clearUserRole = clearUserRole;
+
+  $analyticsProvider.api.setUserEndpointRole = setUserEndpointRole;
+  $analyticsProvider.api.clearUserEndpointRole = clearUserEndpointRole;
+
+  // scope: visit or page. Defaults to 'page'
+  $analyticsProvider.api.setCustomVariable = function (varIndex, varName, value, scope = 'page') {
+    push(['setCustomVariable', varIndex, varName, value, scope]);
+  };
+
+  // scope: visit or page. Defaults to 'page'
+  $analyticsProvider.api.deleteCustomVariable = function (varIndex, scope = 'page') {
+    $window._paq.push(['deleteCustomVariable', varIndex, scope]);
+  };
+
+  // trackSiteSearch(keyword, category, [searchCount])
+  $analyticsProvider.api.trackSiteSearch = function (keyword, category, searchCount) {
+    // keyword is required
+    if (keyword) {
+      const params = ['trackSiteSearch', keyword, category || false];
+
+      // searchCount is optional
+      if (angular.isDefined(searchCount)) {
+        params.push(searchCount);
+      }
+
+      push(params);
+    }
+  };
+
+  // logs a conversion for goal 1. revenue is optional
+  // trackGoal(goalID, [revenue]);
+  $analyticsProvider.api.trackGoal = function (goalID, revenue) {
+    push(['trackGoal', goalID, revenue || 0]);
+  };
+
+  // track outlink or download
+  // linkType is 'link' or 'download', 'link' by default
+  // trackLink(url, [linkType]);
+  $analyticsProvider.api.trackLink = function (url, linkType) {
+    const type = linkType || 'link';
+    push(['trackLink', url, type]);
+  };
+
+  // Set default angulartics page and event tracking
+
+  $analyticsProvider.registerSetUsername(function (username) {
+    push(['setUserId', username]);
+  });
+
+  // locationObj is the angular $location object
+  $analyticsProvider.registerPageTrack(function (path) {
+    push(['setDocumentTitle', $window.document.title]);
+    push(['setReferrerUrl', '']);
+    push(['setCustomUrl', basePath + path]);
+    push(['trackPageView']);
+  });
+
+  /**
+   * @name eventTrack
+   * Track a basic event in Piwik, or send an ecommerce event.
+   *
+   * @param {string} action A string corresponding to the type of event that needs to be tracked.
+   * @param {object} properties The properties that need to be logged with the event.
+   */
+  $analyticsProvider.registerEventTrack(function trackEvent(action, properties = {}) {
+    /**
+     * @description Logs an event with an event category (Videos, Music, Games...), an event
+     * action (Play, Pause, Duration, Add Playlist, Downloaded, Clicked...), and an optional
+     * event name and optional numeric value.
+     *
+     * @link https://piwik.org/docs/event-tracking/
+     * @link https://developer.piwik.org/api-reference/tracking-javascript#using-the-tracker-object
+     *
+     * @property {string} category
+     * @property {string} action
+     * @property {object} metadata
+     * @property value (optional)
+     * @property dimensions (optional)
+     */
+
+    let { category, metadata, value, dimensions } = properties;
+
+    // PAQ requires that eventValue be an integer, see: http://piwik.org/docs/event-tracking
+    if (value) {
+      const parsed = parseInt(properties.value, 10);
+      properties.value = isNaN(parsed) ? 0 : parsed;
+    }
+
+    if (!category) {
+      throw new Error('missing category');
+    }
+    category = category.toLowerCase();
+
+    if (!categories.includes(category)) {
+      throw new Error('unsupported category');
+    }
+
+    action = action.toLowerCase();
+
+    let metadataString = '';
+    if (metadata) {
+      metadataString = JSON.stringify(metadata).toLowerCase();
+    }
+
+    push([
+      'trackEvent',
+      category,
+      action,
+      metadataString, // Changed in favour of Piwik documentation. Added fallback so it's backwards compatible.
+      value,
+      dimensions || {},
+    ]);
+  });
+
+  /**
+   * @name exceptionTrack
+   * Sugar on top of the eventTrack method for easily handling errors
+   *
+   * @param {object} error An Error object to track: error.toString() used for event 'action', error.stack used for event 'label'.
+   * @param {object} cause The cause of the error given from $exceptionHandler, not used.
+   */
+  $analyticsProvider.registerExceptionTrack(function (error) {
+    push(['trackEvent', 'Exceptions', error.toString(), error.stack, 0]);
+  });
+
+  function push(args) {
+    if ($window._paq) {
+      $window._paq.push(args);
+    }
+  }
+
+  function setPortainerStatus(instanceID, version) {
+    setCustomDimension(dimensions.PortainerInstanceID, instanceID);
+    setCustomDimension(dimensions.PortainerVersion, version);
+  }
+
+  function setUserRole(role) {
+    setCustomDimension(dimensions.PortainerUserRole, role);
+  }
+
+  function clearUserRole() {
+    deleteCustomDimension(dimensions.PortainerUserRole);
+  }
+
+  function setUserEndpointRole(role) {
+    setCustomDimension(dimensions.PortainerEndpointUserRole, role);
+  }
+
+  function clearUserEndpointRole() {
+    deleteCustomDimension(dimensions.PortainerEndpointUserRole);
+  }
+
+  function setCustomDimension(dimensionId, value) {
+    push(['setCustomDimension', dimensionId, value]);
+  }
+
+  function deleteCustomDimension(dimensionId) {
+    push(['deleteCustomDimension', dimensionId]);
+  }
+}

app/assets/js/angulartics-matomo.js

@@ -1,223 +0,0 @@
-import angular from 'angular';
-
-// forked from https://github.com/angulartics/angulartics-piwik/blob/master/src/angulartics-piwik.js
-
-/* global _paq */
-/**
- * @ngdoc overview
- * @name angulartics.piwik
- * Enables analytics support for Piwik/Matomo (http://piwik.org/docs/tracking-api/)
- */
-angular.module('angulartics.matomo', ['angulartics']).config([
-  '$analyticsProvider',
-  '$windowProvider',
-  function ($analyticsProvider, $windowProvider) {
-    var $window = $windowProvider.$get();
-
-    $analyticsProvider.settings.pageTracking.trackRelativePath = true;
-
-    // Add piwik specific trackers to angulartics API
-
-    // Requires the CustomDimensions plugin for Piwik.
-    $analyticsProvider.api.setCustomDimension = function (dimensionId, value) {
-      if ($window._paq) {
-        $window._paq.push(['setCustomDimension', dimensionId, value]);
-      }
-    };
-
-    // Requires the CustomDimensions plugin for Piwik.
-    $analyticsProvider.api.deleteCustomDimension = function (dimensionId) {
-      if ($window._paq) {
-        $window._paq.push(['deleteCustomDimension', dimensionId]);
-      }
-    };
-
-    // scope: visit or page. Defaults to 'page'
-    $analyticsProvider.api.setCustomVariable = function (varIndex, varName, value, scope) {
-      if ($window._paq) {
-        scope = scope || 'page';
-        $window._paq.push(['setCustomVariable', varIndex, varName, value, scope]);
-      }
-    };
-
-    // scope: visit or page. Defaults to 'page'
-    $analyticsProvider.api.deleteCustomVariable = function (varIndex, scope) {
-      if ($window._paq) {
-        scope = scope || 'page';
-        $window._paq.push(['deleteCustomVariable', varIndex, scope]);
-      }
-    };
-
-    // trackSiteSearch(keyword, category, [searchCount])
-    $analyticsProvider.api.trackSiteSearch = function (keyword, category, searchCount) {
-      // keyword is required
-      if ($window._paq && keyword) {
-        var params = ['trackSiteSearch', keyword, category || false];
-
-        // searchCount is optional
-        if (angular.isDefined(searchCount)) {
-          params.push(searchCount);
-        }
-
-        $window._paq.push(params);
-      }
-    };
-
-    // logs a conversion for goal 1. revenue is optional
-    // trackGoal(goalID, [revenue]);
-    $analyticsProvider.api.trackGoal = function (goalID, revenue) {
-      if ($window._paq) {
-        _paq.push(['trackGoal', goalID, revenue || 0]);
-      }
-    };
-
-    // track outlink or download
-    // linkType is 'link' or 'download', 'link' by default
-    // trackLink(url, [linkType]);
-    $analyticsProvider.api.trackLink = function (url, linkType) {
-      var type = linkType || 'link';
-      if ($window._paq) {
-        $window._paq.push(['trackLink', url, type]);
-      }
-    };
-
-    // Set default angulartics page and event tracking
-
-    $analyticsProvider.registerSetUsername(function (username) {
-      if ($window._paq) {
-        $window._paq.push(['setUserId', username]);
-      }
-    });
-
-    // locationObj is the angular $location object
-    $analyticsProvider.registerPageTrack(function (path) {
-      if ($window._paq) {
-        $window._paq.push(['setDocumentTitle', $window.document.title]);
-        $window._paq.push(['setReferrerUrl', '']);
-        $window._paq.push(['setCustomUrl', 'http://portainer-ce.app' + path]);
-        $window._paq.push(['trackPageView']);
-      }
-    });
-
-    /**
-     * @name eventTrack
-     * Track a basic event in Piwik, or send an ecommerce event.
-     *
-     * @param {string} action A string corresponding to the type of event that needs to be tracked.
-     * @param {object} properties The properties that need to be logged with the event.
-     */
-    $analyticsProvider.registerEventTrack(function (action, properties) {
-      if ($window._paq) {
-        properties = properties || {};
-
-        switch (action) {
-          /**
-           * @description Sets the current page view as a product or category page view. When you call
-           * setEcommerceView it must be followed by a call to trackPageView to record the product or
-           * category page view.
-           *
-           * @link https://piwik.org/docs/ecommerce-analytics/#tracking-product-page-views-category-page-views-optional
-           * @link https://developer.piwik.org/api-reference/tracking-javascript#ecommerce
-           *
-           * @property productSKU (required) SKU: Product unique identifier
-           * @property productName (optional) Product name
-           * @property categoryName (optional) Product category, or array of up to 5 categories
-           * @property price (optional) Product Price as displayed on the page
-           */
-          case 'setEcommerceView':
-            $window._paq.push(['setEcommerceView', properties.productSKU, properties.productName, properties.categoryName, properties.price]);
-            break;
-
-          /**
-           * @description Adds a product into the ecommerce order. Must be called for each product in
-           * the order.
-           *
-           * @link https://piwik.org/docs/ecommerce-analytics/#tracking-ecommerce-orders-items-purchased-required
-           * @link https://developer.piwik.org/api-reference/tracking-javascript#ecommerce
-           *
-           * @property productSKU (required) SKU: Product unique identifier
-           * @property productName (optional) Product name
-           * @property categoryName (optional) Product category, or array of up to 5 categories
-           * @property price (recommended) Product price
-           * @property quantity (optional, default to 1) Product quantity
-           */
-          case 'addEcommerceItem':
-            $window._paq.push(['addEcommerceItem', properties.productSKU, properties.productName, properties.productCategory, properties.price, properties.quantity]);
-            break;
-
-          /**
-           * @description Tracks a shopping cart. Call this javascript function every time a user is
-           * adding, updating or deleting a product from the cart.
-           *
-           * @link https://piwik.org/docs/ecommerce-analytics/#tracking-add-to-cart-items-added-to-the-cart-optional
-           * @link https://developer.piwik.org/api-reference/tracking-javascript#ecommerce
-           *
-           * @property grandTotal (required) Cart amount
-           */
-          case 'trackEcommerceCartUpdate':
-            $window._paq.push(['trackEcommerceCartUpdate', properties.grandTotal]);
-            break;
-
-          /**
-           * @description Tracks an Ecommerce order, including any ecommerce item previously added to
-           * the order. orderId and grandTotal (ie. revenue) are required parameters.
-           *
-           * @link https://piwik.org/docs/ecommerce-analytics/#tracking-ecommerce-orders-items-purchased-required
-           * @link https://developer.piwik.org/api-reference/tracking-javascript#ecommerce
-           *
-           * @property orderId (required) Unique Order ID
-           * @property grandTotal (required) Order Revenue grand total (includes tax, shipping, and subtracted discount)
-           * @property subTotal (optional) Order sub total (excludes shipping)
-           * @property tax (optional) Tax amount
-           * @property shipping (optional) Shipping amount
-           * @property discount (optional) Discount offered (set to false for unspecified parameter)
-           */
-          case 'trackEcommerceOrder':
-            $window._paq.push(['trackEcommerceOrder', properties.orderId, properties.grandTotal, properties.subTotal, properties.tax, properties.shipping, properties.discount]);
-            break;
-
-          /**
-           * @description Logs an event with an event category (Videos, Music, Games...), an event
-           * action (Play, Pause, Duration, Add Playlist, Downloaded, Clicked...), and an optional
-           * event name and optional numeric value.
-           *
-           * @link https://piwik.org/docs/event-tracking/
-           * @link https://developer.piwik.org/api-reference/tracking-javascript#using-the-tracker-object
-           *
-           * @property category
-           * @property action
-           * @property name (optional, recommended)
-           * @property value (optional)
-           */
-          default:
-            // PAQ requires that eventValue be an integer, see: http://piwik.org/docs/event-tracking
-            if (properties.value) {
-              var parsed = parseInt(properties.value, 10);
-              properties.value = isNaN(parsed) ? 0 : parsed;
-            }
-
-            $window._paq.push([
-              'trackEvent',
-              properties.category,
-              action,
-              properties.name || properties.label, // Changed in favour of Piwik documentation. Added fallback so it's backwards compatible.
-              properties.value,
-            ]);
-        }
-      }
-    });
-
-    /**
-     * @name exceptionTrack
-     * Sugar on top of the eventTrack method for easily handling errors
-     *
-     * @param {object} error An Error object to track: error.toString() used for event 'action', error.stack used for event 'label'.
-     * @param {object} cause The cause of the error given from $exceptionHandler, not used.
-     */
-    $analyticsProvider.registerExceptionTrack(function (error) {
-      if ($window._paq) {
-        $window._paq.push(['trackEvent', 'Exceptions', error.toString(), error.stack, 0]);
-      }
-    });
-  },
-]);

app/portainer/models/status.js

@@ -2,6 +2,8 @@ export function StatusViewModel(data) {
   this.Authentication = data.Authentication;
   this.Snapshot = data.Snapshot;
   this.Version = data.Version;
+  this.Edition = data.Edition;
+  this.InstanceID = data.InstanceID;
 }
 
 export function StatusVersionViewModel(data) {

app/portainer/services/stateManager.js

@@ -2,29 +2,16 @@ import moment from 'moment';
 
 angular.module('portainer.app').factory('StateManager', [
   '$q',
+  '$async',
   'SystemService',
   'InfoHelper',
-  'EndpointProvider',
   'LocalStorage',
   'SettingsService',
   'StatusService',
   'APPLICATION_CACHE_VALIDITY',
   'AgentPingService',
   '$analytics',
-  function StateManagerFactory(
+  function StateManagerFactory($q, $async, SystemService, InfoHelper, LocalStorage, SettingsService, StatusService, APPLICATION_CACHE_VALIDITY, AgentPingService, $analytics) {
-    $q,
-    SystemService,
-    InfoHelper,
-    EndpointProvider,
-    LocalStorage,
-    SettingsService,
-    StatusService,
-    APPLICATION_CACHE_VALIDITY,
-    AgentPingService,
-    $analytics
-  ) {
-    'use strict';
-
     var manager = {};
 
     var state = {
@@ -85,6 +72,9 @@ angular.module('portainer.app').factory('StateManager', [
 
     function assignStateFromStatusAndSettings(status, settings) {
       state.application.version = status.Version;
+      state.application.edition = status.Edition;
+      state.application.instanceId = status.InstanceID;
+
       state.application.enableTelemetry = settings.EnableTelemetry;
       state.application.logo = settings.LogoURL;
       state.application.snapshotInterval = settings.SnapshotInterval;
@@ -103,55 +93,51 @@ angular.module('portainer.app').factory('StateManager', [
           var status = data.status;
           var settings = data.settings;
           assignStateFromStatusAndSettings(status, settings);
-          $analytics.setOptOut(!settings.EnableTelemetry);
           LocalStorage.storeApplicationState(state.application);
           deferred.resolve(state);
         })
         .catch(function error(err) {
           deferred.reject({ msg: 'Unable to retrieve server settings and status', err: err });
-        })
-        .finally(function final() {
-          state.loading = false;
         });
 
       return deferred.promise;
     }
 
-    manager.initialize = function () {
+    manager.initialize = initialize;
-      var deferred = $q.defer();
+    async function initialize() {
-
+      return $async(async () => {
-      var UIState = LocalStorage.getUIState();
+        const UIState = LocalStorage.getUIState();
-      if (UIState) {
+        if (UIState) {
-        state.UI = UIState;
+          state.UI = UIState;
-      }
-
-      var endpointState = LocalStorage.getEndpointState();
-      if (endpointState) {
-        state.endpoint = endpointState;
-      }
-
-      var applicationState = LocalStorage.getApplicationState();
-      if (applicationState && applicationState.validity) {
-        var now = moment().unix();
-        var cacheValidity = now - applicationState.validity;
-        if (cacheValidity > APPLICATION_CACHE_VALIDITY) {
-          loadApplicationState()
-            .then(() => deferred.resolve(state))
-            .catch((err) => deferred.reject(err));
-        } else {
-          state.application = applicationState;
-          state.loading = false;
-          $analytics.setOptOut(!state.application.enableTelemetry);
-          deferred.resolve(state);
         }
-      } else {
-        loadApplicationState()
-          .then(() => deferred.resolve(state))
-          .catch((err) => deferred.reject(err));
-      }
 
-      return deferred.promise;
+        const endpointState = LocalStorage.getEndpointState();
-    };
+        if (endpointState) {
+          state.endpoint = endpointState;
+        }
+
+        const applicationState = LocalStorage.getApplicationState();
+        if (isAppStateValid(applicationState)) {
+          state.application = applicationState;
+        } else {
+          await loadApplicationState();
+        }
+
+        state.loading = false;
+        $analytics.setPortainerStatus(state.application.instanceId, state.application.version);
+        $analytics.setOptOut(!state.application.enableTelemetry);
+        return state;
+      });
+    }
+
+    function isAppStateValid(appState) {
+      if (!appState || !appState.validity) {
+        return false;
+      }
+      const now = moment().unix();
+      const cacheValidity = now - appState.validity;
+      return cacheValidity < APPLICATION_CACHE_VALIDITY;
+    }
 
     function assignExtensions(endpointExtensions) {
       var extensions = [];

app/portainer/views/auth/authController.js

@@ -5,6 +5,7 @@ class AuthenticationController {
   /* @ngInject */
   constructor(
     $async,
+    $analytics,
     $scope,
     $state,
     $stateParams,
@@ -20,6 +21,7 @@ class AuthenticationController {
     StatusService
   ) {
     this.$async = $async;
+    this.$analytics = $analytics;
     this.$scope = $scope;
     this.$state = $state;
     this.$stateParams = $stateParams;
@@ -150,6 +152,10 @@ class AuthenticationController {
 
   async postLoginSteps() {
     await this.StateManager.initialize();
+
+    const isAdmin = this.Authentication.isAdmin();
+    this.$analytics.setUserRole(isAdmin ? 'admin' : 'standard-user');
+
     await this.checkForEndpointsAsync();
     await this.checkForLatestVersionAsync();
   }

gruntfile.js

@@ -73,10 +73,10 @@ module.exports = function (grunt) {
     ]);
   });
 
-  grunt.task.registerTask('devopsbuild', 'devopsbuild:<platform>:<arch>', function (p, a) {
+  grunt.task.registerTask('devopsbuild', 'devopsbuild:<platform>:<arch>:env', function (p, a, env = 'prod') {
     grunt.task.run([
       'config:prod',
-      'env:prod',
+      `env:${env}`,
       'clean:all',
       'copy:assets',
       'shell:build_binary_azuredevops:' + p + ':' + a,
@@ -99,6 +99,9 @@ gruntfile_cfg.env = {
   prod: {
     NODE_ENV: 'production',
   },
+  testing: {
+    NODE_ENV: 'testing',
+  },
 };
 
 gruntfile_cfg.webpack = {