Improve account sync performance, handle concurrent market data syncing (#2236)

* PlaidConnectable concern

* Remove bad abstraction

* Put sync implementations in own concerns

* Sync strategies

* Move sync orchestration to Sync class

* Clean up sync class, add state machine

* Basic market data sync cron

* Fix price sync

* Improve sync window column names, add timestamps

* 30 day syncs by default

* Clean up market data methods

* Report high duplicate sync counts to Sentry

* Add sync states throughout app

* account tab session

* Persistent account tab selections

* Remove manual sleep

* Add migration to clear stale syncs on self hosted apps

* Tweak sync states

* Sync completion event broadcasts

* Fix timezones in tests

* Cleanup

* More cleanup

* Plaid item UI broadcasts for sync

* Fix account ID namespace conflict

* Sync broadcasters

* Smoother account sync refreshes

* Remove test sync delay

app/models/family.rb

@@ -1,5 +1,5 @@
 class Family < ApplicationRecord
-  include Syncable, AutoTransferMatchable, Subscribeable
+  include PlaidConnectable, Syncable, AutoTransferMatchable, Subscribeable
 
   DATE_FORMATS = [
     [ "MM-DD-YYYY", "%m-%d-%Y" ],
@@ -15,7 +15,6 @@ class Family < ApplicationRecord
 
   has_many :users, dependent: :destroy
   has_many :accounts, dependent: :destroy
-  has_many :plaid_items, dependent: :destroy
   has_many :invitations, dependent: :destroy
 
   has_many :imports, dependent: :destroy
@@ -36,6 +35,15 @@ class Family < ApplicationRecord
   validates :locale, inclusion: { in: I18n.available_locales.map(&:to_s) }
   validates :date_format, inclusion: { in: DATE_FORMATS.map(&:last) }
 
+  # If any accounts or plaid items are syncing, the family is also syncing, even if a formal "Family Sync" is not running.
+  def syncing?
+    Sync.joins("LEFT JOIN plaid_items ON plaid_items.id = syncs.syncable_id AND syncs.syncable_type = 'PlaidItem'")
+        .joins("LEFT JOIN accounts ON accounts.id = syncs.syncable_id AND syncs.syncable_type = 'Account'")
+        .where("syncs.syncable_id = ? OR accounts.family_id = ? OR plaid_items.family_id = ?", id, id, id)
+        .incomplete
+        .exists?
+  end
+
   def assigned_merchants
     merchant_ids = transactions.where.not(merchant_id: nil).pluck(:merchant_id).uniq
     Merchant.where(id: merchant_ids)
@@ -65,64 +73,10 @@ class Family < ApplicationRecord
     @income_statement ||= IncomeStatement.new(self)
   end
 
-  def sync_data(sync, start_date: nil)
-    # We don't rely on this value to guard the app, but keep it eventually consistent
-    sync_trial_status!
-
-    Rails.logger.info("Syncing accounts for family #{id}")
-    accounts.manual.each do |account|
-      account.sync_later(start_date: start_date, parent_sync: sync)
-    end
-
-    Rails.logger.info("Applying rules for family #{id}")
-    rules.each do |rule|
-      rule.apply_later
-    end
-  end
-
-  def remove_syncing_notice!
-    broadcast_remove target: "syncing-notice"
-  end
-
-  def post_sync(sync)
-    auto_match_transfers!
-    broadcast_refresh
-  end
-
-  # If family has any syncs pending/syncing within the last 10 minutes, we show a persistent "syncing" notice.
-  # Ignore syncs older than 10 minutes as they are considered "stale"
-  def syncing?
-    Sync.where(
-      "(syncable_type = 'Family' AND syncable_id = ?) OR
-       (syncable_type = 'Account' AND syncable_id IN (SELECT id FROM accounts WHERE family_id = ? AND plaid_account_id IS NULL)) OR
-       (syncable_type = 'PlaidItem' AND syncable_id IN (SELECT id FROM plaid_items WHERE family_id = ?))",
-      id, id, id
-    ).where(status: [ "pending", "syncing" ], created_at: 10.minutes.ago..).exists?
-  end
-
   def eu?
     country != "US" && country != "CA"
   end
 
-  def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil)
-    provider = if region.to_sym == :eu
-      Provider::Registry.get_provider(:plaid_eu)
-    else
-      Provider::Registry.get_provider(:plaid_us)
-    end
-
-    # early return when no provider
-    return nil unless provider
-
-    provider.get_link_token(
-      user_id: id,
-      webhooks_url: webhooks_url,
-      redirect_url: redirect_url,
-      accountable_type: accountable_type,
-      access_token: access_token
-    ).link_token
-  end
-
   def requires_data_provider?
     # If family has any trades, they need a provider for historical prices
     return true if trades.any?