mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-07-18 20:59:39 +02:00
Partial sync interface
This commit is contained in:
parent
6e202bd7ec
commit
7a8ac82823
13 changed files with 34 additions and 94 deletions
|
@ -9,26 +9,17 @@ module Syncable
|
||||||
raise NotImplementedError, "Subclasses must implement the syncing? method"
|
raise NotImplementedError, "Subclasses must implement the syncing? method"
|
||||||
end
|
end
|
||||||
|
|
||||||
# Schedules a sync for syncable. If there is an existing sync pending/syncing for this syncable,
|
def sync_later(parent_sync: nil)
|
||||||
# we do not create a new sync, and attempt to expand the sync window if needed.
|
|
||||||
def sync_later(parent_sync: nil, window_start_date: nil, window_end_date: nil)
|
|
||||||
Sync.transaction do
|
Sync.transaction do
|
||||||
with_lock do
|
with_lock do
|
||||||
sync = self.syncs.incomplete.first
|
sync = self.syncs.incomplete.first
|
||||||
|
|
||||||
if sync
|
unless sync
|
||||||
Rails.logger.info("There is an existing sync, expanding window if needed (#{sync.id})")
|
sync = self.syncs.create!(parent: parent_sync)
|
||||||
sync.expand_window_if_needed(window_start_date, window_end_date)
|
|
||||||
else
|
|
||||||
sync = self.syncs.create!(
|
|
||||||
parent: parent_sync,
|
|
||||||
window_start_date: window_start_date,
|
|
||||||
window_end_date: window_end_date
|
|
||||||
)
|
|
||||||
|
|
||||||
SyncJob.perform_later(sync)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
SyncJob.perform_later(sync)
|
||||||
|
|
||||||
sync
|
sync
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -58,6 +49,10 @@ module Syncable
|
||||||
latest_sync&.created_at
|
latest_sync&.created_at
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def needs_sync?
|
||||||
|
data_synced_through.nil? || data_synced_through < Date.current
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
def latest_sync
|
def latest_sync
|
||||||
syncs.ordered.first
|
syncs.ordered.first
|
||||||
|
|
|
@ -44,8 +44,9 @@ class Entry < ApplicationRecord
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_account_later
|
def sync_account_later
|
||||||
|
# TODO: <Sync>
|
||||||
sync_start_date = [ date_previously_was, date ].compact.min unless destroyed?
|
sync_start_date = [ date_previously_was, date ].compact.min unless destroyed?
|
||||||
account.sync_later(window_start_date: sync_start_date)
|
account.sync_later
|
||||||
end
|
end
|
||||||
|
|
||||||
def entryable_name_short
|
def entryable_name_short
|
||||||
|
|
|
@ -16,7 +16,7 @@ class Family::Syncer
|
||||||
|
|
||||||
# Schedule child syncs
|
# Schedule child syncs
|
||||||
child_syncables.each do |syncable|
|
child_syncables.each do |syncable|
|
||||||
syncable.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date)
|
syncable.sync_later(parent_sync: sync)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -68,13 +68,10 @@ class PlaidItem < ApplicationRecord
|
||||||
end
|
end
|
||||||
|
|
||||||
# Once all the data is fetched, we can schedule account syncs to calculate historical balances
|
# Once all the data is fetched, we can schedule account syncs to calculate historical balances
|
||||||
def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
|
# TODO: <Sync>
|
||||||
|
def schedule_account_syncs(parent_sync: nil)
|
||||||
accounts.each do |account|
|
accounts.each do |account|
|
||||||
account.sync_later(
|
account.sync_later(parent_sync: parent_sync)
|
||||||
parent_sync: parent_sync,
|
|
||||||
window_start_date: window_start_date,
|
|
||||||
window_end_date: window_end_date
|
|
||||||
)
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -13,11 +13,7 @@ class PlaidItem::Syncer
|
||||||
plaid_item.process_accounts
|
plaid_item.process_accounts
|
||||||
|
|
||||||
# All data is synced, so we can now run an account sync to calculate historical balances and more
|
# All data is synced, so we can now run an account sync to calculate historical balances and more
|
||||||
plaid_item.schedule_account_syncs(
|
plaid_item.schedule_account_syncs(parent_sync: sync)
|
||||||
parent_sync: sync,
|
|
||||||
window_start_date: sync.window_start_date,
|
|
||||||
window_end_date: sync.window_end_date
|
|
||||||
)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def perform_post_sync
|
def perform_post_sync
|
||||||
|
|
|
@ -101,29 +101,6 @@ class Sync < ApplicationRecord
|
||||||
parent&.finalize_if_all_children_finalized
|
parent&.finalize_if_all_children_finalized
|
||||||
end
|
end
|
||||||
|
|
||||||
# If a sync is pending, we can adjust the window if new syncs are created with a wider window.
|
|
||||||
def expand_window_if_needed(new_window_start_date, new_window_end_date)
|
|
||||||
return unless pending?
|
|
||||||
return if self.window_start_date.nil? && self.window_end_date.nil? # already as wide as possible
|
|
||||||
|
|
||||||
earliest_start_date = if self.window_start_date && new_window_start_date
|
|
||||||
[ self.window_start_date, new_window_start_date ].min
|
|
||||||
else
|
|
||||||
nil
|
|
||||||
end
|
|
||||||
|
|
||||||
latest_end_date = if self.window_end_date && new_window_end_date
|
|
||||||
[ self.window_end_date, new_window_end_date ].max
|
|
||||||
else
|
|
||||||
nil
|
|
||||||
end
|
|
||||||
|
|
||||||
update(
|
|
||||||
window_start_date: earliest_start_date,
|
|
||||||
window_end_date: latest_end_date
|
|
||||||
)
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
private
|
||||||
def log_status_change
|
def log_status_change
|
||||||
Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})")
|
Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})")
|
||||||
|
|
7
db/migrate/20250526093332_add_synced_through.rb
Normal file
7
db/migrate/20250526093332_add_synced_through.rb
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
class AddSyncedThrough < ActiveRecord::Migration[7.2]
|
||||||
|
def change
|
||||||
|
add_column :families, :data_synced_through, :date
|
||||||
|
add_column :plaid_items, :data_synced_through, :date
|
||||||
|
add_column :accounts, :data_synced_through, :date
|
||||||
|
end
|
||||||
|
end
|
5
db/schema.rb
generated
5
db/schema.rb
generated
|
@ -10,7 +10,7 @@
|
||||||
#
|
#
|
||||||
# It's strongly recommended that you check this file into your version control system.
|
# It's strongly recommended that you check this file into your version control system.
|
||||||
|
|
||||||
ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do
|
ActiveRecord::Schema[7.2].define(version: 2025_05_26_093332) do
|
||||||
# These are extensions that must be enabled in order to support this database
|
# These are extensions that must be enabled in order to support this database
|
||||||
enable_extension "pgcrypto"
|
enable_extension "pgcrypto"
|
||||||
enable_extension "plpgsql"
|
enable_extension "plpgsql"
|
||||||
|
@ -36,6 +36,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do
|
||||||
t.boolean "scheduled_for_deletion", default: false
|
t.boolean "scheduled_for_deletion", default: false
|
||||||
t.decimal "cash_balance", precision: 19, scale: 4, default: "0.0"
|
t.decimal "cash_balance", precision: 19, scale: 4, default: "0.0"
|
||||||
t.jsonb "locked_attributes", default: {}
|
t.jsonb "locked_attributes", default: {}
|
||||||
|
t.date "data_synced_through"
|
||||||
t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"
|
t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"
|
||||||
t.index ["accountable_type"], name: "index_accounts_on_accountable_type"
|
t.index ["accountable_type"], name: "index_accounts_on_accountable_type"
|
||||||
t.index ["family_id", "accountable_type"], name: "index_accounts_on_family_id_and_accountable_type"
|
t.index ["family_id", "accountable_type"], name: "index_accounts_on_family_id_and_accountable_type"
|
||||||
|
@ -228,6 +229,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do
|
||||||
t.boolean "data_enrichment_enabled", default: false
|
t.boolean "data_enrichment_enabled", default: false
|
||||||
t.boolean "early_access", default: false
|
t.boolean "early_access", default: false
|
||||||
t.boolean "auto_sync_on_login", default: true, null: false
|
t.boolean "auto_sync_on_login", default: true, null: false
|
||||||
|
t.date "data_synced_through"
|
||||||
end
|
end
|
||||||
|
|
||||||
create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
|
||||||
|
@ -456,6 +458,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do
|
||||||
t.string "status", default: "good", null: false
|
t.string "status", default: "good", null: false
|
||||||
t.jsonb "raw_payload", default: {}
|
t.jsonb "raw_payload", default: {}
|
||||||
t.jsonb "raw_institution_payload", default: {}
|
t.jsonb "raw_institution_payload", default: {}
|
||||||
|
t.date "data_synced_through"
|
||||||
t.index ["family_id"], name: "index_plaid_items_on_family_id"
|
t.index ["family_id"], name: "index_plaid_items_on_family_id"
|
||||||
t.index ["plaid_id"], name: "index_plaid_items_on_plaid_id", unique: true
|
t.index ["plaid_id"], name: "index_plaid_items_on_plaid_id", unique: true
|
||||||
end
|
end
|
||||||
|
|
|
@ -7,7 +7,7 @@ module SyncableInterfaceTest
|
||||||
test "can sync later" do
|
test "can sync later" do
|
||||||
assert_difference "@syncable.syncs.count", 1 do
|
assert_difference "@syncable.syncs.count", 1 do
|
||||||
assert_enqueued_with job: SyncJob do
|
assert_enqueued_with job: SyncJob do
|
||||||
@syncable.sync_later(window_start_date: 2.days.ago.to_date)
|
@syncable.sync_later
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -17,20 +17,4 @@ module SyncableInterfaceTest
|
||||||
@syncable.class.any_instance.expects(:perform_sync).with(mock_sync).once
|
@syncable.class.any_instance.expects(:perform_sync).with(mock_sync).once
|
||||||
@syncable.perform_sync(mock_sync)
|
@syncable.perform_sync(mock_sync)
|
||||||
end
|
end
|
||||||
|
|
||||||
test "second sync request widens existing pending window" do
|
|
||||||
later_start = 2.days.ago.to_date
|
|
||||||
first_sync = @syncable.sync_later(window_start_date: later_start, window_end_date: later_start)
|
|
||||||
|
|
||||||
earlier_start = 5.days.ago.to_date
|
|
||||||
wider_end = Date.current
|
|
||||||
|
|
||||||
assert_no_difference "@syncable.syncs.count" do
|
|
||||||
@syncable.sync_later(window_start_date: earlier_start, window_end_date: wider_end)
|
|
||||||
end
|
|
||||||
|
|
||||||
first_sync.reload
|
|
||||||
assert_equal earlier_start, first_sync.window_start_date
|
|
||||||
assert_equal wider_end, first_sync.window_end_date
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -4,7 +4,7 @@ class SyncJobTest < ActiveJob::TestCase
|
||||||
test "sync is performed" do
|
test "sync is performed" do
|
||||||
syncable = accounts(:depository)
|
syncable = accounts(:depository)
|
||||||
|
|
||||||
sync = syncable.syncs.create!(window_start_date: 2.days.ago.to_date)
|
sync = syncable.syncs.create!
|
||||||
|
|
||||||
sync.expects(:perform).once
|
sync.expects(:perform).once
|
||||||
|
|
||||||
|
|
|
@ -30,7 +30,7 @@ class EntryTest < ActiveSupport::TestCase
|
||||||
prior_date = @entry.date - 1
|
prior_date = @entry.date - 1
|
||||||
@entry.update! date: prior_date
|
@entry.update! date: prior_date
|
||||||
|
|
||||||
@entry.account.expects(:sync_later).with(window_start_date: prior_date)
|
@entry.account.expects(:sync_later).once
|
||||||
@entry.sync_account_later
|
@entry.sync_account_later
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -38,14 +38,14 @@ class EntryTest < ActiveSupport::TestCase
|
||||||
prior_date = @entry.date
|
prior_date = @entry.date
|
||||||
@entry.update! date: @entry.date + 1
|
@entry.update! date: @entry.date + 1
|
||||||
|
|
||||||
@entry.account.expects(:sync_later).with(window_start_date: prior_date)
|
@entry.account.expects(:sync_later).once
|
||||||
@entry.sync_account_later
|
@entry.sync_account_later
|
||||||
end
|
end
|
||||||
|
|
||||||
test "triggers sync with correct start date when transaction deleted" do
|
test "triggers sync with correct start date when transaction deleted" do
|
||||||
@entry.destroy!
|
@entry.destroy!
|
||||||
|
|
||||||
@entry.account.expects(:sync_later).with(window_start_date: nil)
|
@entry.account.expects(:sync_later).once
|
||||||
@entry.sync_account_later
|
@entry.sync_account_later
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -15,12 +15,12 @@ class Family::SyncerTest < ActiveSupport::TestCase
|
||||||
|
|
||||||
Account.any_instance
|
Account.any_instance
|
||||||
.expects(:sync_later)
|
.expects(:sync_later)
|
||||||
.with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil)
|
.with(parent_sync: family_sync)
|
||||||
.times(manual_accounts_count)
|
.times(manual_accounts_count)
|
||||||
|
|
||||||
PlaidItem.any_instance
|
PlaidItem.any_instance
|
||||||
.expects(:sync_later)
|
.expects(:sync_later)
|
||||||
.with(parent_sync: family_sync, window_start_date: nil, window_end_date: nil)
|
.with(parent_sync: family_sync)
|
||||||
.times(items_count)
|
.times(items_count)
|
||||||
|
|
||||||
syncer.perform_sync(family_sync)
|
syncer.perform_sync(family_sync)
|
||||||
|
|
|
@ -199,24 +199,4 @@ class SyncTest < ActiveSupport::TestCase
|
||||||
assert_equal "stale", stale_pending.reload.status
|
assert_equal "stale", stale_pending.reload.status
|
||||||
assert_equal "stale", stale_syncing.reload.status
|
assert_equal "stale", stale_syncing.reload.status
|
||||||
end
|
end
|
||||||
|
|
||||||
test "expand_window_if_needed widens start and end dates on a pending sync" do
|
|
||||||
initial_start = 1.day.ago.to_date
|
|
||||||
initial_end = 1.day.ago.to_date
|
|
||||||
|
|
||||||
sync = Sync.create!(
|
|
||||||
syncable: accounts(:depository),
|
|
||||||
window_start_date: initial_start,
|
|
||||||
window_end_date: initial_end
|
|
||||||
)
|
|
||||||
|
|
||||||
new_start = 5.days.ago.to_date
|
|
||||||
new_end = Date.current
|
|
||||||
|
|
||||||
sync.expand_window_if_needed(new_start, new_end)
|
|
||||||
sync.reload
|
|
||||||
|
|
||||||
assert_equal new_start, sync.window_start_date
|
|
||||||
assert_equal new_end, sync.window_end_date
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue