Improve account sync performance, handle concurrent market data syncing (#2236)

* PlaidConnectable concern

* Remove bad abstraction

* Put sync implementations in own concerns

* Sync strategies

* Move sync orchestration to Sync class

* Clean up sync class, add state machine

* Basic market data sync cron

* Fix price sync

* Improve sync window column names, add timestamps

* 30 day syncs by default

* Clean up market data methods

* Report high duplicate sync counts to Sentry

* Add sync states throughout app

* account tab session

* Persistent account tab selections

* Remove manual sleep

* Add migration to clear stale syncs on self hosted apps

* Tweak sync states

* Sync completion event broadcasts

* Fix timezones in tests

* Cleanup

* More cleanup

* Plaid item UI broadcasts for sync

* Fix account ID namespace conflict

* Sync broadcasters

* Smoother account sync refreshes

* Remove test sync delay

app/models/family/plaid_connectable.rb

@@ -0,0 +1,51 @@
+module Family::PlaidConnectable
+  extend ActiveSupport::Concern
+
+  included do
+    has_many :plaid_items, dependent: :destroy
+  end
+
+  def create_plaid_item!(public_token:, item_name:, region:)
+    provider = plaid_provider_for_region(region)
+
+    public_token_response = provider.exchange_public_token(public_token)
+
+    plaid_item = plaid_items.create!(
+      name: item_name,
+      plaid_id: public_token_response.item_id,
+      access_token: public_token_response.access_token,
+      plaid_region: region
+    )
+
+    plaid_item.sync_later
+
+    plaid_item
+  end
+
+  def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil)
+    return nil unless plaid_us || plaid_eu
+
+    provider = plaid_provider_for_region(region)
+
+    provider.get_link_token(
+      user_id: self.id,
+      webhooks_url: webhooks_url,
+      redirect_url: redirect_url,
+      accountable_type: accountable_type,
+      access_token: access_token
+    ).link_token
+  end
+
+  private
+    def plaid_us
+      @plaid ||= Provider::Registry.get_provider(:plaid_us)
+    end
+
+    def plaid_eu
+      @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu)
+    end
+
+    def plaid_provider_for_region(region)
+      region.to_sym == :eu ? plaid_eu : plaid_us
+    end
+end

app/models/family/subscribeable.rb

@@ -72,10 +72,9 @@ module Family::Subscribeable
     (1 - days_left_in_trial.to_f / Subscription::TRIAL_DAYS) * 100
   end
 
-  private
-    def sync_trial_status!
-      if subscription&.status == "trialing" && days_left_in_trial < 0
-        subscription.update!(status: "paused")
-      end
+  def sync_trial_status!
+    if subscription&.status == "trialing" && days_left_in_trial < 0
+      subscription.update!(status: "paused")
     end
+  end
 end

app/models/family/sync_complete_event.rb

@@ -0,0 +1,21 @@
+class Family::SyncCompleteEvent
+  attr_reader :family
+
+  def initialize(family)
+    @family = family
+  end
+
+  def broadcast
+    family.broadcast_replace(
+      target: "balance-sheet",
+      partial: "pages/dashboard/balance_sheet",
+      locals: { balance_sheet: family.balance_sheet }
+    )
+
+    family.broadcast_replace(
+      target: "net-worth-chart",
+      partial: "pages/dashboard/net_worth_chart",
+      locals: { balance_sheet: family.balance_sheet, period: Period.last_30_days }
+    )
+  end
+end

app/models/family/syncer.rb

@@ -0,0 +1,31 @@
+class Family::Syncer
+  attr_reader :family
+
+  def initialize(family)
+    @family = family
+  end
+
+  def perform_sync(sync)
+    # We don't rely on this value to guard the app, but keep it eventually consistent
+    family.sync_trial_status!
+
+    Rails.logger.info("Applying rules for family #{family.id}")
+    family.rules.each do |rule|
+      rule.apply_later
+    end
+
+    # Schedule child syncs
+    child_syncables.each do |syncable|
+      syncable.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date)
+    end
+  end
+
+  def perform_post_sync
+    family.auto_match_transfers!
+  end
+
+  private
+    def child_syncables
+      family.plaid_items + family.accounts.manual
+    end
+end