diff --git a/app/models/account/syncable.rb b/app/models/account/syncable.rb deleted file mode 100644 index f4547ce9..00000000 --- a/app/models/account/syncable.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Account::Syncable - extend ActiveSupport::Concern - - include Syncable - - def sync_data(sync, start_date: nil) - Rails.logger.info("Processing balances (#{linked? ? 'reverse' : 'forward'})") - sync_balances - end - - def post_sync(sync) - family.remove_syncing_notice! - - accountable.post_sync(sync) - - unless sync.child? - family.auto_match_transfers! - end - end - - private - def sync_balances - strategy = linked? ? :reverse : :forward - Balance::Syncer.new(self, strategy: strategy).sync_balances - end -end diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb new file mode 100644 index 00000000..c35975e9 --- /dev/null +++ b/app/models/account/syncer.rb @@ -0,0 +1,28 @@ +class Account::Syncer + attr_reader :account + + def initialize(account) + @account = account + end + + def perform_sync(start_date: nil) + Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})") + sync_balances + end + + def perform_post_sync(sync) + account.family.remove_syncing_notice! + + account.accountable.post_sync(sync) + + unless sync.child? + account.family.auto_match_transfers! + end + end + + private + def sync_balances + strategy = account.linked? ? :reverse : :forward + Balance::Syncer.new(account, strategy: strategy).sync_balances + end +end diff --git a/app/models/concerns/syncable.rb b/app/models/concerns/syncable.rb index d804b992..43afed5f 100644 --- a/app/models/concerns/syncable.rb +++ b/app/models/concerns/syncable.rb @@ -18,14 +18,6 @@ module Syncable syncs.create!(start_date: start_date).perform end - def sync_data(sync, start_date: nil) - raise NotImplementedError, "Subclasses must implement the `sync_data` method" - end - - def post_sync(sync) - # no-op, syncable can optionally provide implementation - end - def sync_error latest_sync&.error end diff --git a/app/models/family.rb b/app/models/family.rb index 0df7fd12..0644776b 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -35,6 +35,21 @@ class Family < ApplicationRecord validates :locale, inclusion: { in: I18n.available_locales.map(&:to_s) } validates :date_format, inclusion: { in: DATE_FORMATS.map(&:last) } + def remove_syncing_notice! + broadcast_remove target: "syncing-notice" + end + + # If family has any syncs pending/syncing within the last 10 minutes, we show a persistent "syncing" notice. + # Ignore syncs older than 10 minutes as they are considered "stale" + def syncing? + Sync.where( + "(syncable_type = 'Family' AND syncable_id = ?) OR + (syncable_type = 'Account' AND syncable_id IN (SELECT id FROM accounts WHERE family_id = ? AND plaid_account_id IS NULL)) OR + (syncable_type = 'PlaidItem' AND syncable_id IN (SELECT id FROM plaid_items WHERE family_id = ?))", + id, id, id + ).where(status: [ "pending", "syncing" ], created_at: 10.minutes.ago..).exists? + end + def assigned_merchants merchant_ids = transactions.where.not(merchant_id: nil).pluck(:merchant_id).uniq Merchant.where(id: merchant_ids) diff --git a/app/models/family/subscribeable.rb b/app/models/family/subscribeable.rb index 4c8624b7..94ee0971 100644 --- a/app/models/family/subscribeable.rb +++ b/app/models/family/subscribeable.rb @@ -72,10 +72,9 @@ module Family::Subscribeable (1 - days_left_in_trial.to_f / Subscription::TRIAL_DAYS) * 100 end - private - def sync_trial_status! - if subscription&.status == "trialing" && days_left_in_trial < 0 - subscription.update!(status: "paused") - end + def sync_trial_status! + if subscription&.status == "trialing" && days_left_in_trial < 0 + subscription.update!(status: "paused") end + end end diff --git a/app/models/family/syncable.rb b/app/models/family/syncable.rb deleted file mode 100644 index 8615a528..00000000 --- a/app/models/family/syncable.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Family::Syncable - extend ActiveSupport::Concern - - include Syncable - - def sync_data(sync, start_date: nil) - # We don't rely on this value to guard the app, but keep it eventually consistent - sync_trial_status! - - Rails.logger.info("Syncing accounts for family #{id}") - accounts.manual.each do |account| - account.sync_later(start_date: start_date, parent_sync: sync) - end - - Rails.logger.info("Syncing plaid items for family #{id}") - plaid_items.each do |plaid_item| - plaid_item.sync_later(start_date: start_date, parent_sync: sync) - end - - Rails.logger.info("Applying rules for family #{id}") - rules.each do |rule| - rule.apply_later - end - end - - def remove_syncing_notice! - broadcast_remove target: "syncing-notice" - end - - def post_sync(sync) - auto_match_transfers! - broadcast_refresh - end - - # If family has any syncs pending/syncing within the last 10 minutes, we show a persistent "syncing" notice. - # Ignore syncs older than 10 minutes as they are considered "stale" - def syncing? - Sync.where( - "(syncable_type = 'Family' AND syncable_id = ?) OR - (syncable_type = 'Account' AND syncable_id IN (SELECT id FROM accounts WHERE family_id = ? AND plaid_account_id IS NULL)) OR - (syncable_type = 'PlaidItem' AND syncable_id IN (SELECT id FROM plaid_items WHERE family_id = ?))", - id, id, id - ).where(status: [ "pending", "syncing" ], created_at: 10.minutes.ago..).exists? - end -end diff --git a/app/models/family/syncer.rb b/app/models/family/syncer.rb new file mode 100644 index 00000000..fe35f42f --- /dev/null +++ b/app/models/family/syncer.rb @@ -0,0 +1,32 @@ +class Family::Syncer + attr_reader :family + + def initialize(family) + @family = family + end + + def perform_sync(sync, start_date: nil) + # We don't rely on this value to guard the app, but keep it eventually consistent + family.sync_trial_status! + + Rails.logger.info("Syncing accounts for family #{family.id}") + family.accounts.manual.each do |account| + account.sync_later(start_date: start_date, parent_sync: sync) + end + + Rails.logger.info("Syncing plaid items for family #{family.id}") + family.plaid_items.each do |plaid_item| + plaid_item.sync_later(start_date: start_date, parent_sync: sync) + end + + Rails.logger.info("Applying rules for family #{family.id}") + family.rules.each do |rule| + rule.apply_later + end + end + + def perform_post_sync(sync) + family.auto_match_transfers! + family.broadcast_refresh + end +end diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index c387dcc1..c2708f49 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -1,5 +1,5 @@ class PlaidItem < ApplicationRecord - include Provided, Syncable + include Syncable enum :plaid_region, { us: "us", eu: "eu" } enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -43,6 +43,10 @@ class PlaidItem < ApplicationRecord end end + def build_category_alias_matcher(user_categories) + Provider::Plaid::CategoryAliasMatcher.new(user_categories) + end + def destroy_later update!(scheduled_for_deletion: true) DestroyJob.perform_later(self) @@ -79,123 +83,11 @@ class PlaidItem < ApplicationRecord end private - def fetch_and_load_plaid_data(sync) - data = {} - - # Log what we're about to fetch - Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)" - - item = plaid_provider.get_item(access_token).item - update!(available_products: item.available_products, billed_products: item.billed_products) - - # Institution details - if item.institution_id.present? - begin - Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}" - institution = plaid_provider.get_institution(item.institution_id) - update!( - institution_id: item.institution_id, - institution_url: institution.institution.url, - institution_color: institution.institution.primary_color - ) - rescue Plaid::ApiError => e - Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}" - end - end - - # Accounts - fetched_accounts = plaid_provider.get_item_accounts(self).accounts - data[:accounts] = fetched_accounts || [] - sync.update!(data: data) - Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})" - - internal_plaid_accounts = fetched_accounts.map do |account| - internal_plaid_account = plaid_accounts.find_or_create_from_plaid_data!(account, family) - internal_plaid_account.sync_account_data!(account) - internal_plaid_account - end - - # Transactions - fetched_transactions = safe_fetch_plaid_data(:get_item_transactions) - data[:transactions] = fetched_transactions || [] - sync.update!(data: data) - - if fetched_transactions - Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id } - modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id } - removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id } - - internal_plaid_account.sync_transactions!(added:, modified:, removed:) - end - - update!(next_cursor: fetched_transactions.cursor) - end - end - - # Investments - fetched_investments = safe_fetch_plaid_data(:get_item_investments) - data[:investments] = fetched_investments || [] - sync.update!(data: data) - - if fetched_investments - Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id } - holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id } - securities = fetched_investments.securities - - internal_plaid_account.sync_investments!(transactions:, holdings:, securities:) - end - end - end - - # Liabilities - fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities) - data[:liabilities] = fetched_liabilities || [] - sync.update!(data: data) - - if fetched_liabilities - Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id } - mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id } - student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id } - - internal_plaid_account.sync_credit_data!(credit) if credit - internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage - internal_plaid_account.sync_student_loan_data!(student) if student - end - end - end - end - - def safe_fetch_plaid_data(method) - begin - plaid_provider.send(method, self) - rescue Plaid::ApiError => e - Rails.logger.warn("Error fetching #{method} for item #{id}: #{e.message}") - nil - end - end - def remove_plaid_item plaid_provider.remove_item(access_token) rescue StandardError => e Rails.logger.warn("Failed to remove Plaid item #{id}: #{e.message}") end - def handle_plaid_error(error) - error_body = JSON.parse(error.response_body) - - if error_body["error_code"] == "ITEM_LOGIN_REQUIRED" - update!(status: :requires_update) - end - end - class PlaidConnectionLostError < StandardError; end end diff --git a/app/models/plaid_item/provided.rb b/app/models/plaid_item/provided.rb deleted file mode 100644 index f2e8ee8f..00000000 --- a/app/models/plaid_item/provided.rb +++ /dev/null @@ -1,30 +0,0 @@ -module PlaidItem::Provided - extend ActiveSupport::Concern - - class_methods do - def plaid_us_provider - Provider::Registry.get_provider(:plaid_us) - end - - def plaid_eu_provider - Provider::Registry.get_provider(:plaid_eu) - end - - def plaid_provider_for_region(region) - region.to_sym == :eu ? plaid_eu_provider : plaid_us_provider - end - end - - def build_category_alias_matcher(user_categories) - Provider::Plaid::CategoryAliasMatcher.new(user_categories) - end - - private - def eu? - raise "eu? is not implemented for #{self.class.name}" - end - - def plaid_provider - eu? ? self.class.plaid_eu_provider : self.class.plaid_us_provider - end -end diff --git a/app/models/plaid_item/syncable.rb b/app/models/plaid_item/syncable.rb deleted file mode 100644 index 760887cf..00000000 --- a/app/models/plaid_item/syncable.rb +++ /dev/null @@ -1,28 +0,0 @@ -module PlaidItem::Syncable - extend ActiveSupport::Concern - - include Syncable - - def sync_data(sync, start_date: nil) - begin - Rails.logger.info("Fetching and loading Plaid data") - fetch_and_load_plaid_data(sync) - update!(status: :good) if requires_update? - - # Schedule account syncs - accounts.each do |account| - account.sync_later(start_date: start_date, parent_sync: sync) - end - - Rails.logger.info("Plaid data fetched and loaded") - rescue Plaid::ApiError => e - handle_plaid_error(e) - raise e - end - end - - def post_sync(sync) - auto_match_categories! - family.broadcast_refresh - end -end diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb new file mode 100644 index 00000000..4474c992 --- /dev/null +++ b/app/models/plaid_item/syncer.rb @@ -0,0 +1,151 @@ +class PlaidItem::Syncer + attr_reader :plaid_item + + def initialize(plaid_item) + @plaid_item = plaid_item + end + + def perform_sync(sync, start_date: nil) + begin + Rails.logger.info("Fetching and loading Plaid data") + fetch_and_load_plaid_data + plaid_item.update!(status: :good) if plaid_item.requires_update? + + # Schedule account syncs + plaid_item.accounts.each do |account| + account.sync_later(start_date: start_date, parent_sync: sync) + end + + Rails.logger.info("Plaid data fetched and loaded") + rescue Plaid::ApiError => e + handle_plaid_error(e) + raise e + end + end + + def perform_post_sync(sync) + plaid_item.auto_match_categories! + plaid_item.family.broadcast_refresh + end + + private + def plaid + plaid_item.plaid_region == "eu" ? plaid_eu : plaid_us + end + + def plaid_eu + @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu) + end + + def plaid_us + @plaid_us ||= Provider::Registry.get_provider(:plaid_us) + end + + def safe_fetch_plaid_data(method) + begin + plaid.send(method, self) + rescue Plaid::ApiError => e + Rails.logger.warn("Error fetching #{method} for item #{id}: #{e.message}") + nil + end + end + + def handle_plaid_error(error) + error_body = JSON.parse(error.response_body) + + if error_body["error_code"] == "ITEM_LOGIN_REQUIRED" + update!(status: :requires_update) + end + end + + def fetch_and_load_plaid_data + data = {} + + # Log what we're about to fetch + Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)" + + item = plaid.get_item(plaid_item.access_token).item + update!(available_products: item.available_products, billed_products: item.billed_products) + + # Institution details + if item.institution_id.present? + begin + Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}" + institution = plaid.get_institution(item.institution_id) + update!( + institution_id: item.institution_id, + institution_url: institution.institution.url, + institution_color: institution.institution.primary_color + ) + rescue Plaid::ApiError => e + Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}" + end + end + + # Accounts + fetched_accounts = plaid.get_item_accounts(plaid_item).accounts + data[:accounts] = fetched_accounts || [] + Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})" + + internal_plaid_accounts = fetched_accounts.map do |account| + internal_plaid_account = plaid_item.plaid_accounts.find_or_create_from_plaid_data!(account, plaid_item.family) + internal_plaid_account.sync_account_data!(account) + internal_plaid_account + end + + # Transactions + fetched_transactions = safe_fetch_plaid_data(:get_item_transactions) + data[:transactions] = fetched_transactions || [] + + if fetched_transactions + Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})" + transaction do + internal_plaid_accounts.each do |internal_plaid_account| + added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id } + modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id } + removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id } + + internal_plaid_account.sync_transactions!(added:, modified:, removed:) + end + + update!(next_cursor: fetched_transactions.cursor) + end + end + + # Investments + fetched_investments = safe_fetch_plaid_data(:get_item_investments) + data[:investments] = fetched_investments || [] + + if fetched_investments + Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})" + transaction do + internal_plaid_accounts.each do |internal_plaid_account| + transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id } + holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id } + securities = fetched_investments.securities + + internal_plaid_account.sync_investments!(transactions:, holdings:, securities:) + end + end + end + + # Liabilities + fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities) + data[:liabilities] = fetched_liabilities || [] + + if fetched_liabilities + Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})" + transaction do + internal_plaid_accounts.each do |internal_plaid_account| + credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id } + mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id } + student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id } + + internal_plaid_account.sync_credit_data!(credit) if credit + internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage + internal_plaid_account.sync_student_loan_data!(student) if student + end + end + end + end +end diff --git a/app/models/sync.rb b/app/models/sync.rb index 826899ce..b335eb14 100644 --- a/app/models/sync.rb +++ b/app/models/sync.rb @@ -19,12 +19,12 @@ class Sync < ApplicationRecord start! begin - syncable.sync_data(self, start_date: start_date) + syncer.perform_sync(self, start_date: start_date) unless has_pending_child_syncs? complete! Rails.logger.info("Sync completed, starting post-sync") - syncable.post_sync(self) + syncer.perform_post_sync(self) Rails.logger.info("Post-sync completed") end rescue StandardError => error @@ -51,12 +51,16 @@ class Sync < ApplicationRecord # If this sync is both a child and a parent, we need to notify the parent of completion notify_parent_of_completion! if has_parent? - syncable.post_sync(self) + syncer.perform_post_sync(self) end end end private + def syncer + "#{syncable_type}::Syncer".constantize.new(syncable) + end + def has_pending_child_syncs? children.where(status: [ :pending, :syncing ]).any? end diff --git a/app/views/accounts/_accountable_group.html.erb b/app/views/accounts/_accountable_group.html.erb index a1ab2e3a..ab10dec7 100644 --- a/app/views/accounts/_accountable_group.html.erb +++ b/app/views/accounts/_accountable_group.html.erb @@ -15,11 +15,11 @@