Add the ability to "rollup" values in a time series (#554)

* Clean up time series models

* Add value group rollup class for summarizing hierarchical data

* Integrate new classes

* Update UI to use new patterns

* Update D3 charts to expect new data format

* Clean up account model

* More cleanup

* Money improvements

* Use new money fields

* Remove invalid fixture data to avoid orphaned accountables

* Update time series to work better with collections

* Fix tests and UI bugs

app/models/account.rb

@@ -26,10 +26,8 @@ class Account < ApplicationRecord
     %w[name]
   end
 
-  def trend(period = Period.all)
-    first = balances.in_period(period).order(:date).first
-    last = balances.in_period(period).order(date: :desc).first
-    Trend.new(current: last.balance, previous: first.balance, type: classification)
+  def balance_on(date)
+    balances.where("date <= ?", date).order(date: :desc).first&.balance
   end
 
   def self.by_provider
@@ -41,55 +39,28 @@ class Account < ApplicationRecord
     exists?(status: "syncing")
   end
 
-  # TODO: We will need a better way to encapsulate large queries & transformation logic, but leaving all in one spot until
-  # we have a better understanding of the requirements
-  def self.by_group(period = Period.all)
-    ranked_balances_cte = active.joins(:balances)
-        .select("
-          account_balances.account_id,
-          account_balances.balance,
-          account_balances.date,
-          ROW_NUMBER() OVER (PARTITION BY account_balances.account_id ORDER BY date ASC) AS rn_asc,
-          ROW_NUMBER() OVER (PARTITION BY account_balances.account_id ORDER BY date DESC) AS rn_desc
-        ")
+  def series(period = Period.all)
+    TimeSeries.from_collection(balances.in_period(period), :balance_money)
+  end
 
-    if period.date_range
-      ranked_balances_cte = ranked_balances_cte.where("account_balances.date BETWEEN ? AND ?", period.date_range.begin, period.date_range.end)
+  def self.by_group(period = Period.all)
+    grouped_accounts = { assets: ValueGroup.new("Assets"), liabilities: ValueGroup.new("Liabilities") }
+
+    Accountable.by_classification.each do |classification, types|
+      types.each do |type|
+        group = grouped_accounts[classification.to_sym].add_child_node(type)
+        Accountable.from_type(type).includes(:account).each do |accountable|
+          account = accountable.account
+          value_node = group.add_value_node(account)
+          value_node.attach_series(account.series(period))
+        end
+      end
     end
 
-    accounts_with_period_balances = AccountBalance.with(
-      ranked_balances: ranked_balances_cte
-    )
-      .from("ranked_balances AS rb")
-      .joins("JOIN accounts a ON a.id = rb.account_id")
-      .select("
-        a.name,
-        a.accountable_type,
-        a.classification,
-        SUM(CASE WHEN rb.rn_asc = 1 THEN rb.balance ELSE 0 END) AS start_balance,
-        MAX(CASE WHEN rb.rn_asc = 1 THEN rb.date ELSE NULL END) as start_date,
-        SUM(CASE WHEN rb.rn_desc = 1 THEN rb.balance ELSE 0 END) AS end_balance,
-        MAX(CASE WHEN rb.rn_desc = 1 THEN rb.date ELSE NULL END) as end_date
-      ")
-      .where("rb.rn_asc = 1 OR rb.rn_desc = 1")
-      .group("a.id")
-      .order("end_balance")
-      .to_a
-
-    assets = accounts_with_period_balances.select { |row| row.classification == "asset" }
-    liabilities = accounts_with_period_balances.select { |row| row.classification == "liability" }
-
-    total_assets = assets.sum(&:end_balance)
-    total_liabilities = liabilities.sum(&:end_balance)
-
-    {
-      asset: build_group_summary(assets, "asset"),
-      liability: build_group_summary(liabilities, "liability")
-    }
+    grouped_accounts
   end
 
   private
-
     def check_currency
       if self.currency == self.family.currency
         self.converted_balance = self.balance
@@ -99,34 +70,4 @@ class Account < ApplicationRecord
         self.converted_currency = self.family.currency
       end
     end
-
-    def self.build_group_summary(accounts, classification)
-      total_balance = accounts.sum(&:end_balance)
-      {
-        total: total_balance,
-        groups: accounts.group_by(&:accountable_type).transform_values do |rows|
-          build_account_summary(rows, total_balance, classification)
-        end
-      }
-    end
-
-    def self.build_account_summary(accounts, total_balance, classification)
-      end_balance = accounts.sum(&:end_balance)
-      start_balance = accounts.sum(&:start_balance)
-      {
-        start_balance: start_balance,
-        end_balance: end_balance,
-        allocation: (end_balance / total_balance * 100).round(2),
-        trend: Trend.new(current: end_balance, previous: start_balance, type: classification),
-        accounts: accounts.map do |account|
-          {
-            name: account.name,
-            start_balance: account.start_balance,
-            end_balance: account.end_balance,
-            allocation: (account.end_balance / total_balance * 100).round(2),
-            trend: Trend.new(current: account.end_balance, previous: account.start_balance, type: classification)
-          }
-        end
-      }
-    end
 end

app/models/account_balance.rb

@@ -6,11 +6,4 @@ class AccountBalance < ApplicationRecord
   monetize :balance
 
   scope :in_period, ->(period) { period.date_range.nil? ? all : where(date: period.date_range) }
-
-  def self.to_series(account, period = Period.all)
-    MoneySeries.new(
-      in_period(period).order(:date),
-      { trend_type: account.classification }
-    )
-  end
 end

app/models/concerns/accountable.rb

@@ -1,15 +1,26 @@
 module Accountable
   extend ActiveSupport::Concern
 
-  TYPES = %w[ Account::Credit Account::Depository Account::Investment Account::Loan Account::OtherAsset Account::OtherLiability Account::Property Account::Vehicle ]
+  ASSET_TYPES = %w[ Account::Depository Account::Investment Account::OtherAsset Account::Property Account::Vehicle ]
+  LIABILITY_TYPES = %w[ Account::Credit Account::Loan Account::OtherLiability ]
+  TYPES = ASSET_TYPES + LIABILITY_TYPES
 
   def self.from_type(type)
     return nil unless types.include?(type) || TYPES.include?(type)
     "Account::#{type.demodulize}".constantize
   end
 
-  def self.types
-    TYPES.map { |type| type.demodulize }
+  def self.by_classification
+    { assets: ASSET_TYPES, liabilities: LIABILITY_TYPES }
+  end
+
+  def self.types(classification = nil)
+    types = classification ? (classification.to_sym == :asset ? ASSET_TYPES : LIABILITY_TYPES) : TYPES
+    types.map { |type| type.demodulize }
+  end
+
+  def self.classification(type)
+    ASSET_TYPES.include?(type) ? :asset : :liability
   end
 
   included do

app/models/family.rb

@@ -23,10 +23,12 @@ class Family < ApplicationRecord
 
     query = query.where("account_balances.date BETWEEN ? AND ?", period.date_range.begin, period.date_range.end) if period.date_range
 
+    result = query.to_a
+
     {
-      asset_series: MoneySeries.new(query, { trend_type: :asset, amount_accessor: "assets" }),
-      liability_series: MoneySeries.new(query, { trend_type: :liability, amount_accessor: "liabilities" }),
-      net_worth_series: MoneySeries.new(query, { trend_type: :asset, amount_accessor: "net_worth" })
+      asset_series: TimeSeries.new(result.map { |r| { date: r.date, value: Money.new(r.assets, r.currency) } }),
+      liability_series: TimeSeries.new(result.map { |r| { date: r.date, value: Money.new(r.liabilities, r.currency) } }),
+      net_worth_series: TimeSeries.new(result.map { |r| { date: r.date, value: Money.new(r.net_worth, r.currency) } })
     }
   end
 
@@ -35,7 +37,7 @@ class Family < ApplicationRecord
   end
 
   def net_worth
-    accounts.active.sum("CASE WHEN classification = 'asset' THEN balance ELSE -balance END")
+    Money.new(accounts.active.sum("CASE WHEN classification = 'asset' THEN balance ELSE -balance END"), currency)
   end
 
   def assets
@@ -43,6 +45,6 @@ class Family < ApplicationRecord
   end
 
   def liabilities
-    accounts.active.liabilities.sum(:balance)
+    Money.new(accounts.active.liabilities.sum(:balance), currency)
   end
 end

app/models/money_series.rb

@@ -1,60 +0,0 @@
-class MoneySeries
-    def initialize(series, options = {})
-        @trend_type = options[:trend_type] || "asset" # Defines whether a positive trend is good or bad
-        @accessor = options[:amount_accessor] || :balance
-        @series = series
-    end
-
-    def valid?
-        @series.length > 1
-    end
-
-    def data
-        [ nil, *@series ].each_cons(2).map do |previous, current|
-            {
-                raw: current,
-                date: current.date,
-                value: Money.new(current.send(@accessor), current.currency),
-                trend: Trend.new(
-                    current: current.send(@accessor),
-                    previous: previous&.send(@accessor),
-                    type: @trend_type
-                )
-            }
-        end
-    end
-
-    def trend
-        return Trend.new(current: 0, type: @trend_type) unless valid?
-
-        Trend.new(
-            current: @series.last.send(@accessor),
-            previous: @series.first&.send(@accessor),
-            type: @trend_type
-        )
-    end
-
-    def serialize_for_d3_chart
-        {
-            data: data.map do |datum|
-                {
-                    date: datum[:date],
-                    amount: datum[:value].amount,
-                    currency: datum[:value].currency,
-                    trend: {
-                        amount: datum[:trend].amount,
-                        percent: datum[:trend].percent,
-                        direction: datum[:trend].direction,
-                        type: datum[:trend].type
-                    }
-                }
-            end,
-            trend: {
-                amount: trend.amount,
-                percent: trend.percent,
-                direction: trend.direction,
-                type: trend.type
-            }
-        }.to_json
-    end
-end

app/models/time_series.rb

@@ -0,0 +1,83 @@
+
+class TimeSeries
+    attr_reader :type
+
+    def self.from_collection(collection, value_method, options = {})
+        data = collection.map do |obj|
+            { date: obj.date, value: obj.public_send(value_method), original: obj }
+        end
+        new(data, options)
+    end
+
+    def initialize(data, options = {})
+        @type = options[:type] || :normal
+        initialize_series_data(data)
+    end
+
+    def values
+        @values ||= add_trends_to_series
+    end
+
+    def first
+        values.first
+    end
+
+    def last
+        values.last
+    end
+
+    def on(date)
+        values.find { |v| v.date == date }
+    end
+
+    def trend
+        TimeSeries::Trend.new(
+            current: last&.value,
+            previous: first&.value,
+            type: @type
+        )
+    end
+
+    # Data shape that frontend expects for D3 charts
+    def to_json(*_args)
+        {
+            values: values.map do |v|
+                {
+                    date: v.date,
+                    value: JSON.parse(v.value.to_json),
+                    trend: {
+                        type: v.trend.type,
+                        direction: v.trend.direction,
+                        value: JSON.parse(v.trend.value.to_json),
+                        percent: v.trend.percent
+                    }
+                }
+            end,
+            trend: {
+                type: @type,
+                direction: trend.direction,
+                value: JSON.parse(trend.value.to_json),
+                percent: trend.percent
+            },
+            type: @type
+        }.to_json
+    end
+
+    private
+        def initialize_series_data(data)
+            @series_data = data.nil? || data.empty? ? [] : data.map { |d| TimeSeries::Value.new(d) }.sort_by(&:date)
+        end
+
+        def add_trends_to_series
+            [ nil, *@series_data ].each_cons(2).map do |previous, current|
+                unless current.trend
+                    current.trend = TimeSeries::Trend.new(
+                        current: current.value,
+                        previous: previous&.value,
+                        type: @type
+                    )
+                end
+                current
+            end
+        end
+end

app/models/time_series/trend.rb

@@ -0,0 +1,48 @@
+class TimeSeries::Trend
+  attr_reader :type
+
+  # Tells us whether an increasing/decreasing trend is good or bad (i.e. a liability decreasing is good)
+  TYPES = %i[normal inverse].freeze
+
+  def initialize(current: nil, previous: nil, type: :normal)
+    validate_data_types(current, previous)
+    validate_type(type)
+    @current = current
+    @previous = previous
+    @type = type
+  end
+
+  def direction
+    return "flat" if @previous.nil? || @current == @previous
+    return "up" if @current && @current > @previous
+    "down"
+  end
+
+  def value
+    return Money.new(0) if @previous.nil? && @current.is_a?(Money)
+    return 0 if @previous.nil?
+    @current - @previous
+  end
+
+  def percent
+    return 0.0 if @previous.nil?
+    return Float::INFINITY if @previous == 0
+    ((extract_numeric(@current) - extract_numeric(@previous)).abs / extract_numeric(@previous).abs.to_f * 100).round(1).to_f
+  end
+
+  private
+    def validate_type(type)
+      raise ArgumentError, "Invalid type" unless TYPES.include?(type)
+    end
+
+    def validate_data_types(current, previous)
+      return if previous.nil? || current.nil?
+      raise ArgumentError, "Current and previous values must be of the same type" unless current.class == previous.class
+      raise ArgumentError, "Current and previous values must be of type Money or Numeric" unless current.is_a?(Money) || current.is_a?(Numeric)
+    end
+
+    def extract_numeric(obj)
+      return obj.amount if obj.is_a? Money
+      obj
+    end
+end

app/models/time_series/value.rb

@@ -0,0 +1,32 @@
+class TimeSeries::Value
+    include Comparable
+
+    attr_accessor :trend
+    attr_reader :value, :date, :original
+
+    def initialize(obj)
+        @original = obj[:original] || obj
+
+        if obj.is_a?(Hash)
+            @date = obj[:date]
+            @value = obj[:value]
+        else
+            @date = obj.date
+            @value = obj.value
+        end
+
+        validate_input
+    end
+
+    def <=>(other)
+        result = date <=> other.date
+        result = value <=> other.value if result == 0
+        result
+    end
+
+    private
+        def validate_input
+            raise ArgumentError, "Date is required" unless @date.is_a?(Date)
+            raise ArgumentError, "Money or Numeric value is required" unless @value.is_a?(Money) || @value.is_a?(Numeric)
+        end
+end

app/models/trend.rb

@@ -1,26 +0,0 @@
-class Trend
-  attr_reader :current, :previous, :type
-
-  def initialize(current: nil, previous: nil, type: "asset")
-    @current = current
-    @previous = previous
-    @type = type # asset means positive trend is good, liability means negative trend is good
-  end
-
-  def direction
-    return "flat" if @current == @previous
-    return "up" if @previous.nil? || (@current && @current > @previous)
-    "down"
-  end
-
-  def amount
-    return 0 if @previous.nil?
-    @current - @previous
-  end
-
-  def percent
-    return 0 if @previous.nil?
-    return Float::INFINITY if @previous == 0
-    ((@current - @previous).abs / @previous.abs.to_f * 100).round(1)
-  end
-end

app/models/valuation.rb

@@ -8,11 +8,8 @@ class Valuation < ApplicationRecord
 
   scope :in_period, ->(period) { period.date_range.nil? ? all : where(date: period.date_range) }
 
-  def self.to_series(account, period = Period.all)
-    MoneySeries.new(
-      in_period(period).order(:date),
-      { trend_type: account.classification, amount_accessor: :value }
-    )
+  def self.to_series
+    TimeSeries.from_collection all, :value_money
   end
 
   private

app/models/value_group.rb

@@ -0,0 +1,100 @@
+  class ValueGroup
+    attr_accessor :parent
+    attr_reader :name, :children, :value, :original
+
+    def initialize(name = "Root", value: nil, original: nil)
+        @name = name
+        @value = value
+        @children = []
+        @original = original
+    end
+
+    def sum
+        return value if is_value_node?
+        return 0 if children.empty? && value.nil?
+        children.sum(&:sum)
+    end
+
+    def avg
+        return value if is_value_node?
+        return 0 if children.empty? && value.nil?
+        leaf_values = value_nodes.map(&:value)
+        leaf_values.compact.sum.to_f / leaf_values.compact.size
+    end
+
+    def series
+        return @raw_series || TimeSeries.new([]) if is_value_node?
+        summed_by_date = children.each_with_object(Hash.new(0)) do |child, acc|
+            child.series.values.each do |series_value|
+                acc[series_value.date] += series_value.value
+            end
+        end
+
+        summed_series = summed_by_date.map { |date, value| { date: date, value: value } }
+        TimeSeries.new(summed_series)
+    end
+
+    def value_nodes
+        return [ self ] unless value.nil?
+        children.flat_map { |child| child.value_nodes }
+    end
+
+    def percent_of_total
+        return 100 if parent.nil?
+        ((sum / parent.sum) * 100).round(1)
+    end
+
+    def leaf?
+        children.empty?
+    end
+
+    def add_child_node(name)
+        raise "Cannot add subgroup to node with a value" if is_value_node?
+        child = self.class.new(name)
+        child.parent = self
+        @children << child
+        child
+    end
+
+    def add_value_node(obj)
+        raise "Cannot add value node to a non-leaf node" unless can_add_value_node?
+        child = create_value_node(obj)
+        child.parent = self
+        @children << child
+        child
+    end
+
+    def attach_series(raw_series)
+        validate_attached_series(raw_series)
+        @raw_series = raw_series
+    end
+
+    def is_value_node?
+        value.present?
+    end
+
+    private
+        def can_add_value_node?
+            return false if is_value_node?
+            children.empty? || children.all?(&:is_value_node?)
+        end
+
+        def create_value_node(obj)
+            value = if obj.respond_to?(:value)
+                obj.value
+            elsif obj.respond_to?(:balance)
+                obj.balance
+            elsif obj.respond_to?(:amount)
+                obj.amount
+            else
+                raise ArgumentError, "Object must have a value, balance, or amount"
+            end
+
+            self.class.new(obj.name, value: value, original: obj)
+        end
+
+        def validate_attached_series(series)
+            raise "Cannot add series to a node without a value" unless is_value_node?
+            raise "Attached series must be a TimeSeries" unless series.is_a?(TimeSeries)
+        end
+  end