Improve account sync performance, handle concurrent market data syncing (#2236)

* PlaidConnectable concern

* Remove bad abstraction

* Put sync implementations in own concerns

* Sync strategies

* Move sync orchestration to Sync class

* Clean up sync class, add state machine

* Basic market data sync cron

* Fix price sync

* Improve sync window column names, add timestamps

* 30 day syncs by default

* Clean up market data methods

* Report high duplicate sync counts to Sentry

* Add sync states throughout app

* account tab session

* Persistent account tab selections

* Remove manual sleep

* Add migration to clear stale syncs on self hosted apps

* Tweak sync states

* Sync completion event broadcasts

* Fix timezones in tests

* Cleanup

* More cleanup

* Plaid item UI broadcasts for sync

* Fix account ID namespace conflict

* Sync broadcasters

* Smoother account sync refreshes

* Remove test sync delay

db/migrate/20250512171654_update_sync_timestamps.rb

@@ -0,0 +1,65 @@
+class UpdateSyncTimestamps < ActiveRecord::Migration[7.2]
+  def change
+    # Timestamps, managed by aasm
+    add_column :syncs, :pending_at, :datetime
+    add_column :syncs, :syncing_at, :datetime
+    add_column :syncs, :completed_at, :datetime
+    add_column :syncs, :failed_at, :datetime
+
+    add_column :syncs, :window_start_date, :date
+    add_column :syncs, :window_end_date, :date
+
+    reversible do |dir|
+      dir.up do
+        execute <<-SQL
+          UPDATE syncs
+          SET
+            completed_at = CASE
+              WHEN status = 'completed' THEN last_ran_at
+              ELSE NULL
+            END,
+            failed_at = CASE
+              WHEN status = 'failed' THEN last_ran_at
+              ELSE NULL
+            END
+        SQL
+
+        execute <<-SQL
+          UPDATE syncs
+          SET window_start_date = start_date
+        SQL
+
+        # Due to some recent bugs, some self hosters have syncs that are stuck.
+        # This manually fails those syncs so they stop seeing syncing UI notices.
+        if Rails.application.config.app_mode.self_hosted?
+          puts "Self hosted: Fail syncs older than 2 hours"
+          execute <<-SQL
+            UPDATE syncs
+            SET status = 'failed'
+            WHERE (
+              status = 'syncing' AND
+              created_at < NOW() - INTERVAL '2 hours'
+            )
+          SQL
+        end
+      end
+
+      dir.down do
+        execute <<-SQL
+          UPDATE syncs
+          SET
+            last_ran_at = COALESCE(completed_at, failed_at)
+        SQL
+
+        execute <<-SQL
+          UPDATE syncs
+          SET start_date = window_start_date
+        SQL
+      end
+    end
+
+    remove_column :syncs, :start_date, :date
+    remove_column :syncs, :last_ran_at, :datetime
+    remove_column :syncs, :error_backtrace, :text, array: true
+  end
+end

db/migrate/20250514214242_add_metadata_to_session.rb

@@ -0,0 +1,5 @@
+class AddMetadataToSession < ActiveRecord::Migration[7.2]
+  def change
+    add_column :sessions, :data, :jsonb, default: {}
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_05_13_122703) do
+ActiveRecord::Schema[7.2].define(version: 2025_05_14_214242) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -537,6 +537,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_13_122703) do
     t.uuid "active_impersonator_session_id"
     t.datetime "subscribed_at"
     t.jsonb "prev_transaction_page_params", default: {}
+    t.jsonb "data", default: {}
     t.index ["active_impersonator_session_id"], name: "index_sessions_on_active_impersonator_session_id"
     t.index ["user_id"], name: "index_sessions_on_user_id"
   end
@@ -587,15 +588,18 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_13_122703) do
   create_table "syncs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.string "syncable_type", null: false
     t.uuid "syncable_id", null: false
-    t.datetime "last_ran_at"
-    t.date "start_date"
     t.string "status", default: "pending"
     t.string "error"
     t.jsonb "data"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.text "error_backtrace", array: true
     t.uuid "parent_id"
+    t.datetime "pending_at"
+    t.datetime "syncing_at"
+    t.datetime "completed_at"
+    t.datetime "failed_at"
+    t.date "window_start_date"
+    t.date "window_end_date"
     t.index ["parent_id"], name: "index_syncs_on_parent_id"
     t.index ["syncable_type", "syncable_id"], name: "index_syncs_on_syncable"
   end