From 7a8ac82823cc6df3bfc9fcd921789e333b933ca5 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Mon, 26 May 2025 05:39:14 -0400 Subject: [PATCH] Partial sync interface --- app/models/concerns/syncable.rb | 23 ++++++++----------- app/models/entry.rb | 3 ++- app/models/family/syncer.rb | 2 +- app/models/plaid_item.rb | 9 +++----- app/models/plaid_item/syncer.rb | 6 +---- app/models/sync.rb | 23 ------------------- .../20250526093332_add_synced_through.rb | 7 ++++++ db/schema.rb | 5 +++- test/interfaces/syncable_interface_test.rb | 18 +-------------- test/jobs/sync_job_test.rb | 2 +- test/models/account/entry_test.rb | 6 ++--- test/models/family/syncer_test.rb | 4 ++-- test/models/sync_test.rb | 20 ---------------- 13 files changed, 34 insertions(+), 94 deletions(-) create mode 100644 db/migrate/20250526093332_add_synced_through.rb diff --git a/app/models/concerns/syncable.rb b/app/models/concerns/syncable.rb index 72556bf7..209e9f5d 100644 --- a/app/models/concerns/syncable.rb +++ b/app/models/concerns/syncable.rb @@ -9,26 +9,17 @@ module Syncable raise NotImplementedError, "Subclasses must implement the syncing? method" end - # Schedules a sync for syncable. If there is an existing sync pending/syncing for this syncable, - # we do not create a new sync, and attempt to expand the sync window if needed. - def sync_later(parent_sync: nil, window_start_date: nil, window_end_date: nil) + def sync_later(parent_sync: nil) Sync.transaction do with_lock do sync = self.syncs.incomplete.first - if sync - Rails.logger.info("There is an existing sync, expanding window if needed (#{sync.id})") - sync.expand_window_if_needed(window_start_date, window_end_date) - else - sync = self.syncs.create!( - parent: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - - SyncJob.perform_later(sync) + unless sync + sync = self.syncs.create!(parent: parent_sync) end + SyncJob.perform_later(sync) + sync end end @@ -58,6 +49,10 @@ module Syncable latest_sync&.created_at end + def needs_sync? + data_synced_through.nil? || data_synced_through < Date.current + end + private def latest_sync syncs.ordered.first diff --git a/app/models/entry.rb b/app/models/entry.rb index 0426c9f5..13cf179a 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -44,8 +44,9 @@ class Entry < ApplicationRecord end def sync_account_later + # TODO: sync_start_date = [ date_previously_was, date ].compact.min unless destroyed? - account.sync_later(window_start_date: sync_start_date) + account.sync_later end def entryable_name_short diff --git a/app/models/family/syncer.rb b/app/models/family/syncer.rb index 30ce2ad5..7ca9539e 100644 --- a/app/models/family/syncer.rb +++ b/app/models/family/syncer.rb @@ -16,7 +16,7 @@ class Family::Syncer # Schedule child syncs child_syncables.each do |syncable| - syncable.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date) + syncable.sync_later(parent_sync: sync) end end diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 25aa3e18..7931bdc7 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -68,13 +68,10 @@ class PlaidItem < ApplicationRecord end # Once all the data is fetched, we can schedule account syncs to calculate historical balances - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) + # TODO: + def schedule_account_syncs(parent_sync: nil) accounts.each do |account| - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) + account.sync_later(parent_sync: parent_sync) end end diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb index b76c37b6..bea78508 100644 --- a/app/models/plaid_item/syncer.rb +++ b/app/models/plaid_item/syncer.rb @@ -13,11 +13,7 @@ class PlaidItem::Syncer plaid_item.process_accounts # All data is synced, so we can now run an account sync to calculate historical balances and more - plaid_item.schedule_account_syncs( - parent_sync: sync, - window_start_date: sync.window_start_date, - window_end_date: sync.window_end_date - ) + plaid_item.schedule_account_syncs(parent_sync: sync) end def perform_post_sync diff --git a/app/models/sync.rb b/app/models/sync.rb index 37c05dfa..ae8816d5 100644 --- a/app/models/sync.rb +++ b/app/models/sync.rb @@ -101,29 +101,6 @@ class Sync < ApplicationRecord parent&.finalize_if_all_children_finalized end - # If a sync is pending, we can adjust the window if new syncs are created with a wider window. - def expand_window_if_needed(new_window_start_date, new_window_end_date) - return unless pending? - return if self.window_start_date.nil? && self.window_end_date.nil? # already as wide as possible - - earliest_start_date = if self.window_start_date && new_window_start_date - [ self.window_start_date, new_window_start_date ].min - else - nil - end - - latest_end_date = if self.window_end_date && new_window_end_date - [ self.window_end_date, new_window_end_date ].max - else - nil - end - - update( - window_start_date: earliest_start_date, - window_end_date: latest_end_date - ) - end - private def log_status_change Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})") diff --git a/db/migrate/20250526093332_add_synced_through.rb b/db/migrate/20250526093332_add_synced_through.rb new file mode 100644 index 00000000..2345baab --- /dev/null +++ b/db/migrate/20250526093332_add_synced_through.rb @@ -0,0 +1,7 @@ +class AddSyncedThrough < ActiveRecord::Migration[7.2] + def change + add_column :families, :data_synced_through, :date + add_column :plaid_items, :data_synced_through, :date + add_column :accounts, :data_synced_through, :date + end +end diff --git a/db/schema.rb b/db/schema.rb index b5f41eab..6ed2af16 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do +ActiveRecord::Schema[7.2].define(version: 2025_05_26_093332) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -36,6 +36,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do t.boolean "scheduled_for_deletion", default: false t.decimal "cash_balance", precision: 19, scale: 4, default: "0.0" t.jsonb "locked_attributes", default: {} + t.date "data_synced_through" t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type" t.index ["accountable_type"], name: "index_accounts_on_accountable_type" t.index ["family_id", "accountable_type"], name: "index_accounts_on_family_id_and_accountable_type" @@ -228,6 +229,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do t.boolean "data_enrichment_enabled", default: false t.boolean "early_access", default: false t.boolean "auto_sync_on_login", default: true, null: false + t.date "data_synced_through" end create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -456,6 +458,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do t.string "status", default: "good", null: false t.jsonb "raw_payload", default: {} t.jsonb "raw_institution_payload", default: {} + t.date "data_synced_through" t.index ["family_id"], name: "index_plaid_items_on_family_id" t.index ["plaid_id"], name: "index_plaid_items_on_plaid_id", unique: true end diff --git a/test/interfaces/syncable_interface_test.rb b/test/interfaces/syncable_interface_test.rb index df142052..fc7a29cf 100644 --- a/test/interfaces/syncable_interface_test.rb +++ b/test/interfaces/syncable_interface_test.rb @@ -7,7 +7,7 @@ module SyncableInterfaceTest test "can sync later" do assert_difference "@syncable.syncs.count", 1 do assert_enqueued_with job: SyncJob do - @syncable.sync_later(window_start_date: 2.days.ago.to_date) + @syncable.sync_later end end end @@ -17,20 +17,4 @@ module SyncableInterfaceTest @syncable.class.any_instance.expects(:perform_sync).with(mock_sync).once @syncable.perform_sync(mock_sync) end - - test "second sync request widens existing pending window" do - later_start = 2.days.ago.to_date - first_sync = @syncable.sync_later(window_start_date: later_start, window_end_date: later_start) - - earlier_start = 5.days.ago.to_date - wider_end = Date.current - - assert_no_difference "@syncable.syncs.count" do - @syncable.sync_later(window_start_date: earlier_start, window_end_date: wider_end) - end - - first_sync.reload - assert_equal earlier_start, first_sync.window_start_date - assert_equal wider_end, first_sync.window_end_date - end end diff --git a/test/jobs/sync_job_test.rb b/test/jobs/sync_job_test.rb index b392b3a3..8cebb12a 100644 --- a/test/jobs/sync_job_test.rb +++ b/test/jobs/sync_job_test.rb @@ -4,7 +4,7 @@ class SyncJobTest < ActiveJob::TestCase test "sync is performed" do syncable = accounts(:depository) - sync = syncable.syncs.create!(window_start_date: 2.days.ago.to_date) + sync = syncable.syncs.create! sync.expects(:perform).once diff --git a/test/models/account/entry_test.rb b/test/models/account/entry_test.rb index edd55e68..e681771e 100644 --- a/test/models/account/entry_test.rb +++ b/test/models/account/entry_test.rb @@ -30,7 +30,7 @@ class EntryTest < ActiveSupport::TestCase prior_date = @entry.date - 1 @entry.update! date: prior_date - @entry.account.expects(:sync_later).with(window_start_date: prior_date) + @entry.account.expects(:sync_later).once @entry.sync_account_later end @@ -38,14 +38,14 @@ class EntryTest < ActiveSupport::TestCase prior_date = @entry.date @entry.update! date: @entry.date + 1 - @entry.account.expects(:sync_later).with(window_start_date: prior_date) + @entry.account.expects(:sync_later).once @entry.sync_account_later end test "triggers sync with correct start date when transaction deleted" do @entry.destroy! - @entry.account.expects(:sync_later).with(window_start_date: nil) + @entry.account.expects(:sync_later).once @entry.sync_account_later end diff --git a/test/models/family/syncer_test.rb b/test/models/family/syncer_test.rb index 7fe01a7e..55c074dc 100644 --- a/test/models/family/syncer_test.rb +++ b/test/models/family/syncer_test.rb @@ -15,12 +15,12 @@ class Family::SyncerTest < ActiveSupport::TestCase Account.any_instance .expects(:sync_later) - .with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil) + .with(parent_sync: family_sync) .times(manual_accounts_count) PlaidItem.any_instance .expects(:sync_later) - .with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil) + .with(parent_sync: family_sync) .times(items_count) syncer.perform_sync(family_sync) diff --git a/test/models/sync_test.rb b/test/models/sync_test.rb index d1182fd5..4569612e 100644 --- a/test/models/sync_test.rb +++ b/test/models/sync_test.rb @@ -199,24 +199,4 @@ class SyncTest < ActiveSupport::TestCase assert_equal "stale", stale_pending.reload.status assert_equal "stale", stale_syncing.reload.status end - - test "expand_window_if_needed widens start and end dates on a pending sync" do - initial_start = 1.day.ago.to_date - initial_end = 1.day.ago.to_date - - sync = Sync.create!( - syncable: accounts(:depository), - window_start_date: initial_start, - window_end_date: initial_end - ) - - new_start = 5.days.ago.to_date - new_end = Date.current - - sync.expand_window_if_needed(new_start, new_end) - sync.reload - - assert_equal new_start, sync.window_start_date - assert_equal new_end, sync.window_end_date - end end