Improve account sync performance, handle concurrent market data syncing (#2236)

* PlaidConnectable concern

* Remove bad abstraction

* Put sync implementations in own concerns

* Sync strategies

* Move sync orchestration to Sync class

* Clean up sync class, add state machine

* Basic market data sync cron

* Fix price sync

* Improve sync window column names, add timestamps

* 30 day syncs by default

* Clean up market data methods

* Report high duplicate sync counts to Sentry

* Add sync states throughout app

* account tab session

* Persistent account tab selections

* Remove manual sleep

* Add migration to clear stale syncs on self hosted apps

* Tweak sync states

* Sync completion event broadcasts

* Fix timezones in tests

* Cleanup

* More cleanup

* Plaid item UI broadcasts for sync

* Fix account ID namespace conflict

* Sync broadcasters

* Smoother account sync refreshes

* Remove test sync delay

app/models/sync.rb

@@ -1,4 +1,6 @@
 class Sync < ApplicationRecord
+  include AASM
+
   Error = Class.new(StandardError)
 
   belongs_to :syncable, polymorphic: true
@@ -6,12 +8,31 @@ class Sync < ApplicationRecord
   belongs_to :parent, class_name: "Sync", optional: true
   has_many :children, class_name: "Sync", foreign_key: :parent_id, dependent: :destroy
 
-  enum :status, { pending: "pending", syncing: "syncing", completed: "completed", failed: "failed" }
-
   scope :ordered, -> { order(created_at: :desc) }
+  scope :incomplete, -> { where(status: [ :pending, :syncing ]) }
 
-  def child?
-    parent_id.present?
+  validate :window_valid
+
+  # Sync state machine
+  aasm column: :status, timestamps: true do
+    state :pending, initial: true
+    state :syncing
+    state :completed
+    state :failed
+
+    after_all_transitions :log_status_change
+
+    event :start, after_commit: :report_warnings do
+      transitions from: :pending, to: :syncing
+    end
+
+    event :complete do
+      transitions from: :syncing, to: :completed
+    end
+
+    event :fail do
+      transitions from: :syncing, to: :failed
+    end
   end
 
   def perform
@@ -19,43 +40,83 @@ class Sync < ApplicationRecord
       start!
 
       begin
-        syncable.sync_data(self, start_date: start_date)
-
-        complete!
-        Rails.logger.info("Sync completed, starting post-sync")
-        syncable.post_sync(self)
-        Rails.logger.info("Post-sync completed")
-      rescue StandardError => error
-        fail! error, report_error: true
+        syncable.perform_sync(self)
+      rescue => e
+        fail!
+        update(error: e.message)
+        report_error(e)
+      ensure
+        finalize_if_all_children_finalized
       end
     end
   end
 
-  private
-    def start!
-      Rails.logger.info("Starting sync")
-      update! status: :syncing
-    end
+  # Finalizes the current sync AND parent (if it exists)
+  def finalize_if_all_children_finalized
+    Sync.transaction do
+      lock!
 
-    def complete!
-      Rails.logger.info("Sync completed")
-      update! status: :completed, last_ran_at: Time.current
-    end
+      # If this is the "parent" and there are still children running, don't finalize.
+      return unless all_children_finalized?
 
-    def fail!(error, report_error: false)
-      Rails.logger.error("Sync failed: #{error.message}")
-
-      if report_error
-        Sentry.capture_exception(error) do |scope|
-          scope.set_context("sync", { id: id, syncable_type: syncable_type, syncable_id: syncable_id })
-          scope.set_tags(sync_id: id)
+      if syncing?
+        if has_failed_children?
+          fail!
+        else
+          complete!
         end
       end
 
-      update!(
-        status: :failed,
-        error: error.message,
-        last_ran_at: Time.current
-      )
+      # If we make it here, the sync is finalized.  Run post-sync, regardless of failure/success.
+      perform_post_sync
+    end
+
+    # If this sync has a parent, try to finalize it so the child status propagates up the chain.
+    parent&.finalize_if_all_children_finalized
+  end
+
+  private
+    def log_status_change
+      Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})")
+    end
+
+    def has_failed_children?
+      children.failed.any?
+    end
+
+    def all_children_finalized?
+      children.incomplete.empty?
+    end
+
+    def perform_post_sync
+      Rails.logger.info("Performing post-sync for #{syncable_type} (#{syncable.id})")
+      syncable.perform_post_sync
+      syncable.broadcast_sync_complete
+    rescue => e
+      Rails.logger.error("Error performing post-sync for #{syncable_type} (#{syncable.id}): #{e.message}")
+      report_error(e)
+    end
+
+    def report_error(error)
+      Sentry.capture_exception(error) do |scope|
+        scope.set_tags(sync_id: id)
+      end
+    end
+
+    def report_warnings
+      todays_sync_count = syncable.syncs.where(created_at: Date.current.all_day).count
+
+      if todays_sync_count > 10
+        Sentry.capture_exception(
+          Error.new("#{syncable_type} (#{syncable.id}) has exceeded 10 syncs today (count: #{todays_sync_count})"),
+          level: :warning
+        )
+      end
+    end
+
+    def window_valid
+      if window_start_date && window_end_date && window_start_date > window_end_date
+        errors.add(:window_end_date, "must be greater than window_start_date")
+      end
     end
 end