Additional cache columns on balances for activity view breakdowns (#2505)

* Initial schema iteration

* Add new balance components

* Add existing data migrator to backfill components

* Update calculator test assertions for new balance components

* Update flow assertions for forward calculator

* Update reverse calculator flows assumptions

* Forward calculator tests passing

* Get all calculator tests passing

* Assert flows factor

app/models/balance/base_calculator.rb

@@ -15,8 +15,8 @@ class Balance::BaseCalculator
     end
 
     def holdings_value_for_date(date)
-      holdings = sync_cache.get_holdings(date)
-      holdings.sum(&:amount)
+      @holdings_value_for_date ||= {}
+      @holdings_value_for_date[date] ||= sync_cache.get_holdings(date).sum(&:amount)
     end
 
     def derive_cash_balance_on_date_from_total(total_balance:, date:)
@@ -29,6 +29,67 @@ class Balance::BaseCalculator
       end
     end
 
+    def cash_adjustments_for_date(start_cash, net_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type != :non_cash
+
+      valuation.amount - start_cash - net_cash_flows
+    end
+
+    def non_cash_adjustments_for_date(start_non_cash, non_cash_flows, valuation)
+      return 0 unless valuation && account.balance_type == :non_cash
+
+      valuation.amount - start_non_cash - non_cash_flows
+    end
+
+    # If holdings value goes from $100 -> $200 (change_holdings_value is $100)
+    # And non-cash flows (i.e. "buys") for day are +$50 (net_buy_sell_value is $50)
+    # That means value increased by $100, where $50 of that is due to the change in holdings value, and $50 is due to the buy/sell
+    def market_value_change_on_date(date, flows)
+      return 0 unless account.balance_type == :investment
+
+      start_of_day_holdings_value = holdings_value_for_date(date.prev_day)
+      end_of_day_holdings_value = holdings_value_for_date(date)
+
+      change_holdings_value = end_of_day_holdings_value - start_of_day_holdings_value
+      net_buy_sell_value = flows[:non_cash_inflows] - flows[:non_cash_outflows]
+
+      change_holdings_value - net_buy_sell_value
+    end
+
+    def flows_for_date(date)
+      entries = sync_cache.get_entries(date)
+
+      cash_inflows = 0
+      cash_outflows = 0
+      non_cash_inflows = 0
+      non_cash_outflows = 0
+
+      txn_inflow_sum = entries.select { |e| e.amount < 0 && e.transaction? }.sum(&:amount)
+      txn_outflow_sum = entries.select { |e| e.amount >= 0 && e.transaction? }.sum(&:amount)
+
+      trade_cash_inflow_sum = entries.select { |e| e.amount < 0 && e.trade? }.sum(&:amount)
+      trade_cash_outflow_sum = entries.select { |e| e.amount >= 0 && e.trade? }.sum(&:amount)
+
+      if account.balance_type == :non_cash && account.accountable_type == "Loan"
+        non_cash_inflows = txn_inflow_sum.abs
+        non_cash_outflows = txn_outflow_sum
+      elsif account.balance_type != :non_cash
+        cash_inflows = txn_inflow_sum.abs + trade_cash_inflow_sum.abs
+        cash_outflows = txn_outflow_sum + trade_cash_outflow_sum
+
+        # Trades are inverse (a "buy" is outflow of cash, but "inflow" of non-cash, aka "holdings")
+        non_cash_outflows = trade_cash_inflow_sum.abs
+        non_cash_inflows = trade_cash_outflow_sum
+      end
+
+      {
+        cash_inflows: cash_inflows,
+        cash_outflows: cash_outflows,
+        non_cash_inflows: non_cash_inflows,
+        non_cash_outflows: non_cash_outflows
+      }
+    end
+
     def derive_cash_balance(cash_balance, date)
       entries = sync_cache.get_entries(date)
 
@@ -57,13 +118,23 @@ class Balance::BaseCalculator
       raise NotImplementedError, "Directional calculators must implement this method"
     end
 
-    def build_balance(date:, cash_balance:, non_cash_balance:)
+    def build_balance(date:, **args)
       Balance.new(
         account_id: account.id,
+        currency: account.currency,
         date: date,
-        balance: non_cash_balance + cash_balance,
-        cash_balance: cash_balance,
-        currency: account.currency
+        balance: args[:balance],
+        cash_balance: args[:cash_balance],
+        start_cash_balance: args[:start_cash_balance] || 0,
+        start_non_cash_balance: args[:start_non_cash_balance] || 0,
+        cash_inflows: args[:cash_inflows] || 0,
+        cash_outflows: args[:cash_outflows] || 0,
+        non_cash_inflows: args[:non_cash_inflows] || 0,
+        non_cash_outflows: args[:non_cash_outflows] || 0,
+        cash_adjustments: args[:cash_adjustments] || 0,
+        non_cash_adjustments: args[:non_cash_adjustments] || 0,
+        net_market_flows: args[:net_market_flows] || 0,
+        flows_factor: account.classification == "asset" ? 1 : -1
       )
     end
 end

app/models/balance/forward_calculator.rb

@@ -2,13 +2,13 @@ class Balance::ForwardCalculator < Balance::BaseCalculator
   def calculate
     Rails.logger.tagged("Balance::ForwardCalculator") do
       start_cash_balance = derive_cash_balance_on_date_from_total(
-        total_balance: account.opening_anchor_balance,
+        total_balance: 0,
         date: account.opening_anchor_date
       )
-      start_non_cash_balance = account.opening_anchor_balance - start_cash_balance
+      start_non_cash_balance = 0
 
       calc_start_date.upto(calc_end_date).map do |date|
-        valuation = sync_cache.get_reconciliation_valuation(date)
+        valuation = sync_cache.get_valuation(date)
 
         if valuation
           end_cash_balance = derive_cash_balance_on_date_from_total(
@@ -21,10 +21,22 @@ class Balance::ForwardCalculator < Balance::BaseCalculator
           end_non_cash_balance = derive_end_non_cash_balance(start_non_cash_balance: start_non_cash_balance, date: date)
         end
 
+        flows = flows_for_date(date)
+        market_value_change = market_value_change_on_date(date, flows)
+
         output_balance = build_balance(
           date: date,
+          balance: end_cash_balance + end_non_cash_balance,
           cash_balance: end_cash_balance,
-          non_cash_balance: end_non_cash_balance
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          cash_adjustments: cash_adjustments_for_date(start_cash_balance, flows[:cash_inflows] - flows[:cash_outflows], valuation),
+          non_cash_adjustments: non_cash_adjustments_for_date(start_non_cash_balance, flows[:non_cash_inflows] - flows[:non_cash_outflows], valuation),
+          net_market_flows: market_value_change
         )
 
         # Set values for the next iteration

app/models/balance/reverse_calculator.rb

@@ -11,6 +11,8 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
 
       # Calculates in reverse-chronological order (End of day -> Start of day)
       account.current_anchor_date.downto(account.opening_anchor_date).map do |date|
+        flows = flows_for_date(date)
+
         if use_opening_anchor_for_date?(date)
           end_cash_balance = derive_cash_balance_on_date_from_total(
             total_balance: account.opening_anchor_balance,
@@ -20,29 +22,30 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
 
           start_cash_balance = end_cash_balance
           start_non_cash_balance = end_non_cash_balance
-
-          build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
+          market_value_change = 0
         else
           start_cash_balance = derive_start_cash_balance(end_cash_balance: end_cash_balance, date: date)
           start_non_cash_balance = derive_start_non_cash_balance(end_non_cash_balance: end_non_cash_balance, date: date)
-
-          # Even though we've just calculated "start" balances, we set today equal to end of day, then use those
-          # in our next iteration (slightly confusing, but just the nature of a "reverse" sync)
-          output_balance = build_balance(
-            date: date,
-            cash_balance: end_cash_balance,
-            non_cash_balance: end_non_cash_balance
-          )
-
-          end_cash_balance = start_cash_balance
-          end_non_cash_balance = start_non_cash_balance
-
-          output_balance
+          market_value_change = market_value_change_on_date(date, flows)
         end
+
+        output_balance = build_balance(
+          date: date,
+          balance: end_cash_balance + end_non_cash_balance,
+          cash_balance: end_cash_balance,
+          start_cash_balance: start_cash_balance,
+          start_non_cash_balance: start_non_cash_balance,
+          cash_inflows: flows[:cash_inflows],
+          cash_outflows: flows[:cash_outflows],
+          non_cash_inflows: flows[:non_cash_inflows],
+          non_cash_outflows: flows[:non_cash_outflows],
+          net_market_flows: market_value_change
+        )
+
+        end_cash_balance = start_cash_balance
+        end_non_cash_balance = start_non_cash_balance
+
+        output_balance
       end
     end
   end
@@ -58,13 +61,6 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
       account.asset? ? entry_flows : -entry_flows
     end
 
-    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
-    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
-    # explanation, see the test suite.
-    def use_opening_anchor_for_date?(date)
-      account.has_opening_anchor? && date == account.opening_anchor_date
-    end
-
     # Alias method, for algorithmic clarity
     # Derives cash balance, starting from the end-of-day, applying entries in reverse to get the start-of-day balance
     def derive_start_cash_balance(end_cash_balance:, date:)
@@ -76,4 +72,11 @@ class Balance::ReverseCalculator < Balance::BaseCalculator
     def derive_start_non_cash_balance(end_non_cash_balance:, date:)
       derive_non_cash_balance(end_non_cash_balance, date, direction: :reverse)
     end
+
+    # Reverse syncs are a bit different than forward syncs because we do not allow "reconciliation" valuations
+    # to be used at all. This is primarily to keep the code and the UI easy to understand. For a more detailed
+    # explanation, see the test suite.
+    def use_opening_anchor_for_date?(date)
+      account.has_opening_anchor? && date == account.opening_anchor_date
+    end
 end

app/models/balance/sync_cache.rb

@@ -3,8 +3,8 @@ class Balance::SyncCache
     @account = account
   end
 
-  def get_reconciliation_valuation(date)
-    converted_entries.find { |e| e.date == date && e.valuation? && e.valuation.reconciliation? }
+  def get_valuation(date)
+    converted_entries.find { |e| e.date == date && e.valuation? }
   end
 
   def get_holdings(date)