mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-07-21 22:29:38 +02:00
Improve account sync performance, handle concurrent market data syncing (#2236)
* PlaidConnectable concern * Remove bad abstraction * Put sync implementations in own concerns * Sync strategies * Move sync orchestration to Sync class * Clean up sync class, add state machine * Basic market data sync cron * Fix price sync * Improve sync window column names, add timestamps * 30 day syncs by default * Clean up market data methods * Report high duplicate sync counts to Sentry * Add sync states throughout app * account tab session * Persistent account tab selections * Remove manual sleep * Add migration to clear stale syncs on self hosted apps * Tweak sync states * Sync completion event broadcasts * Fix timezones in tests * Cleanup * More cleanup * Plaid item UI broadcasts for sync * Fix account ID namespace conflict * Sync broadcasters * Smoother account sync refreshes * Remove test sync delay
This commit is contained in:
Zach Gollwitzer
GitHub
parent
9793cc74f9
commit
10dd9e061a
97 changed files with 1837 additions and 949 deletions
|
|
@ -1,34 +1,170 @@
|
|||
require "test_helper"
|
||||
|
||||
class SyncTest < ActiveSupport::TestCase
|
||||
setup do
|
||||
@sync = syncs(:account)
|
||||
@sync.update(status: "pending")
|
||||
end
|
||||
include ActiveJob::TestHelper
|
||||
|
||||
test "runs successful sync" do
|
||||
@sync.syncable.expects(:sync_data).with(@sync, start_date: @sync.start_date).once
|
||||
syncable = accounts(:depository)
|
||||
sync = Sync.create!(syncable: syncable)
|
||||
|
||||
assert_equal "pending", @sync.status
|
||||
syncable.expects(:perform_sync).with(sync).once
|
||||
|
||||
previously_ran_at = @sync.last_ran_at
|
||||
assert_equal "pending", sync.status
|
||||
|
||||
@sync.perform
|
||||
sync.perform
|
||||
|
||||
assert @sync.last_ran_at > previously_ran_at
|
||||
assert_equal "completed", @sync.status
|
||||
assert sync.completed_at < Time.now
|
||||
assert_equal "completed", sync.status
|
||||
end
|
||||
|
||||
test "handles sync errors" do
|
||||
@sync.syncable.expects(:sync_data).with(@sync, start_date: @sync.start_date).raises(StandardError.new("test sync error"))
|
||||
syncable = accounts(:depository)
|
||||
sync = Sync.create!(syncable: syncable)
|
||||
|
||||
assert_equal "pending", @sync.status
|
||||
previously_ran_at = @sync.last_ran_at
|
||||
syncable.expects(:perform_sync).with(sync).raises(StandardError.new("test sync error"))
|
||||
|
||||
@sync.perform
|
||||
assert_equal "pending", sync.status
|
||||
|
||||
assert @sync.last_ran_at > previously_ran_at
|
||||
assert_equal "failed", @sync.status
|
||||
assert_equal "test sync error", @sync.error
|
||||
sync.perform
|
||||
|
||||
assert sync.failed_at < Time.now
|
||||
assert_equal "failed", sync.status
|
||||
assert_equal "test sync error", sync.error
|
||||
end
|
||||
|
||||
test "can run nested syncs that alert the parent when complete" do
|
||||
family = families(:dylan_family)
|
||||
plaid_item = plaid_items(:one)
|
||||
account = accounts(:connected)
|
||||
|
||||
family_sync = Sync.create!(syncable: family)
|
||||
plaid_item_sync = Sync.create!(syncable: plaid_item, parent: family_sync)
|
||||
account_sync = Sync.create!(syncable: account, parent: plaid_item_sync)
|
||||
|
||||
assert_equal "pending", family_sync.status
|
||||
assert_equal "pending", plaid_item_sync.status
|
||||
assert_equal "pending", account_sync.status
|
||||
|
||||
family.expects(:perform_sync).with(family_sync).once
|
||||
|
||||
family_sync.perform
|
||||
|
||||
assert_equal "syncing", family_sync.reload.status
|
||||
|
||||
plaid_item.expects(:perform_sync).with(plaid_item_sync).once
|
||||
|
||||
plaid_item_sync.perform
|
||||
|
||||
assert_equal "syncing", family_sync.reload.status
|
||||
assert_equal "syncing", plaid_item_sync.reload.status
|
||||
|
||||
account.expects(:perform_sync).with(account_sync).once
|
||||
|
||||
# Since these are accessed through `parent`, they won't necessarily be the same
|
||||
# instance we configured above
|
||||
Account.any_instance.expects(:perform_post_sync).once
|
||||
Account.any_instance.expects(:broadcast_sync_complete).once
|
||||
PlaidItem.any_instance.expects(:perform_post_sync).once
|
||||
PlaidItem.any_instance.expects(:broadcast_sync_complete).once
|
||||
Family.any_instance.expects(:perform_post_sync).once
|
||||
Family.any_instance.expects(:broadcast_sync_complete).once
|
||||
|
||||
account_sync.perform
|
||||
|
||||
assert_equal "completed", plaid_item_sync.reload.status
|
||||
assert_equal "completed", account_sync.reload.status
|
||||
assert_equal "completed", family_sync.reload.status
|
||||
end
|
||||
|
||||
test "failures propagate up the chain" do
|
||||
family = families(:dylan_family)
|
||||
plaid_item = plaid_items(:one)
|
||||
account = accounts(:connected)
|
||||
|
||||
family_sync = Sync.create!(syncable: family)
|
||||
plaid_item_sync = Sync.create!(syncable: plaid_item, parent: family_sync)
|
||||
account_sync = Sync.create!(syncable: account, parent: plaid_item_sync)
|
||||
|
||||
assert_equal "pending", family_sync.status
|
||||
assert_equal "pending", plaid_item_sync.status
|
||||
assert_equal "pending", account_sync.status
|
||||
|
||||
family.expects(:perform_sync).with(family_sync).once
|
||||
|
||||
family_sync.perform
|
||||
|
||||
assert_equal "syncing", family_sync.reload.status
|
||||
|
||||
plaid_item.expects(:perform_sync).with(plaid_item_sync).once
|
||||
|
||||
plaid_item_sync.perform
|
||||
|
||||
assert_equal "syncing", family_sync.reload.status
|
||||
assert_equal "syncing", plaid_item_sync.reload.status
|
||||
|
||||
# This error should "bubble up" to the PlaidItem and Family sync results
|
||||
account.expects(:perform_sync).with(account_sync).raises(StandardError.new("test account sync error"))
|
||||
|
||||
# Since these are accessed through `parent`, they won't necessarily be the same
|
||||
# instance we configured above
|
||||
Account.any_instance.expects(:perform_post_sync).once
|
||||
PlaidItem.any_instance.expects(:perform_post_sync).once
|
||||
Family.any_instance.expects(:perform_post_sync).once
|
||||
|
||||
Account.any_instance.expects(:broadcast_sync_complete).once
|
||||
PlaidItem.any_instance.expects(:broadcast_sync_complete).once
|
||||
Family.any_instance.expects(:broadcast_sync_complete).once
|
||||
|
||||
account_sync.perform
|
||||
|
||||
assert_equal "failed", plaid_item_sync.reload.status
|
||||
assert_equal "failed", account_sync.reload.status
|
||||
assert_equal "failed", family_sync.reload.status
|
||||
end
|
||||
|
||||
test "parent failure should not change status if child succeeds" do
|
||||
family = families(:dylan_family)
|
||||
plaid_item = plaid_items(:one)
|
||||
account = accounts(:connected)
|
||||
|
||||
family_sync = Sync.create!(syncable: family)
|
||||
plaid_item_sync = Sync.create!(syncable: plaid_item, parent: family_sync)
|
||||
account_sync = Sync.create!(syncable: account, parent: plaid_item_sync)
|
||||
|
||||
assert_equal "pending", family_sync.status
|
||||
assert_equal "pending", plaid_item_sync.status
|
||||
assert_equal "pending", account_sync.status
|
||||
|
||||
family.expects(:perform_sync).with(family_sync).raises(StandardError.new("test family sync error"))
|
||||
|
||||
family_sync.perform
|
||||
|
||||
assert_equal "failed", family_sync.reload.status
|
||||
|
||||
plaid_item.expects(:perform_sync).with(plaid_item_sync).raises(StandardError.new("test plaid item sync error"))
|
||||
|
||||
plaid_item_sync.perform
|
||||
|
||||
assert_equal "failed", family_sync.reload.status
|
||||
assert_equal "failed", plaid_item_sync.reload.status
|
||||
|
||||
# Leaf level sync succeeds, but shouldn't change the status of the already-failed parent syncs
|
||||
account.expects(:perform_sync).with(account_sync).once
|
||||
|
||||
# Since these are accessed through `parent`, they won't necessarily be the same
|
||||
# instance we configured above
|
||||
Account.any_instance.expects(:perform_post_sync).once
|
||||
PlaidItem.any_instance.expects(:perform_post_sync).once
|
||||
Family.any_instance.expects(:perform_post_sync).once
|
||||
|
||||
Account.any_instance.expects(:broadcast_sync_complete).once
|
||||
PlaidItem.any_instance.expects(:broadcast_sync_complete).once
|
||||
Family.any_instance.expects(:broadcast_sync_complete).once
|
||||
|
||||
account_sync.perform
|
||||
|
||||
assert_equal "failed", plaid_item_sync.reload.status
|
||||
assert_equal "failed", family_sync.reload.status
|
||||
assert_equal "completed", account_sync.reload.status
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue