Improve account sync performance, handle concurrent market data syncing (#2236)

* PlaidConnectable concern

* Remove bad abstraction

* Put sync implementations in own concerns

* Sync strategies

* Move sync orchestration to Sync class

* Clean up sync class, add state machine

* Basic market data sync cron

* Fix price sync

* Improve sync window column names, add timestamps

* 30 day syncs by default

* Clean up market data methods

* Report high duplicate sync counts to Sentry

* Add sync states throughout app

* account tab session

* Persistent account tab selections

* Remove manual sleep

* Add migration to clear stale syncs on self hosted apps

* Tweak sync states

* Sync completion event broadcasts

* Fix timezones in tests

* Cleanup

* More cleanup

* Plaid item UI broadcasts for sync

* Fix account ID namespace conflict

* Sync broadcasters

* Smoother account sync refreshes

* Remove test sync delay

app/models/holding/forward_calculator.rb

@@ -1,10 +1,12 @@
-class Holding::ForwardCalculator < Holding::BaseCalculator
-  private
-    def portfolio_cache
-      @portfolio_cache ||= Holding::PortfolioCache.new(account)
-    end
+class Holding::ForwardCalculator
+  attr_reader :account
 
-    def calculate_holdings
+  def initialize(account)
+    @account = account
+  end
+
+  def calculate
+    Rails.logger.tagged("Holding::ForwardCalculator") do
       current_portfolio = generate_starting_portfolio
       next_portfolio = {}
       holdings = []
@@ -16,6 +18,55 @@ class Holding::ForwardCalculator < Holding::BaseCalculator
         current_portfolio = next_portfolio
       end
 
-      holdings
+      Holding.gapfill(holdings)
+    end
+  end
+
+  private
+    def portfolio_cache
+      @portfolio_cache ||= Holding::PortfolioCache.new(account)
+    end
+
+    def empty_portfolio
+      securities = portfolio_cache.get_securities
+      securities.each_with_object({}) { |security, hash| hash[security.id] = 0 }
+    end
+
+    def generate_starting_portfolio
+      empty_portfolio
+    end
+
+    def transform_portfolio(previous_portfolio, trade_entries, direction: :forward)
+      new_quantities = previous_portfolio.dup
+
+      trade_entries.each do |trade_entry|
+        trade = trade_entry.entryable
+        security_id = trade.security_id
+        qty_change = trade.qty
+        qty_change = qty_change * -1 if direction == :reverse
+        new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change
+      end
+
+      new_quantities
+    end
+
+    def build_holdings(portfolio, date, price_source: nil)
+      portfolio.map do |security_id, qty|
+        price = portfolio_cache.get_price(security_id, date, source: price_source)
+
+        if price.nil?
+          next
+        end
+
+        Holding.new(
+          account_id: account.id,
+          security_id: security_id,
+          date: date,
+          qty: qty,
+          price: price.price,
+          currency: price.currency,
+          amount: qty * price.price
+        )
+      end.compact
     end
 end