mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-08-05 05:25:24 +02:00
Improve account sync performance, handle concurrent market data syncing (#2236)
* PlaidConnectable concern * Remove bad abstraction * Put sync implementations in own concerns * Sync strategies * Move sync orchestration to Sync class * Clean up sync class, add state machine * Basic market data sync cron * Fix price sync * Improve sync window column names, add timestamps * 30 day syncs by default * Clean up market data methods * Report high duplicate sync counts to Sentry * Add sync states throughout app * account tab session * Persistent account tab selections * Remove manual sleep * Add migration to clear stale syncs on self hosted apps * Tweak sync states * Sync completion event broadcasts * Fix timezones in tests * Cleanup * More cleanup * Plaid item UI broadcasts for sync * Fix account ID namespace conflict * Sync broadcasters * Smoother account sync refreshes * Remove test sync delay
This commit is contained in:
Zach Gollwitzer
GitHub
parent
9793cc74f9
commit
10dd9e061a
97 changed files with 1837 additions and 949 deletions
|
|
@ -1,5 +1,5 @@
|
|||
class PlaidItem < ApplicationRecord
|
||||
include Provided, Syncable
|
||||
include Syncable
|
||||
|
||||
enum :plaid_region, { us: "us", eu: "eu" }
|
||||
enum :status, { good: "good", requires_update: "requires_update" }, default: :good
|
||||
|
|
@ -22,39 +22,6 @@ class PlaidItem < ApplicationRecord
|
|||
scope :ordered, -> { order(created_at: :desc) }
|
||||
scope :needs_update, -> { where(status: :requires_update) }
|
||||
|
||||
class << self
|
||||
def create_from_public_token(token, item_name:, region:)
|
||||
response = plaid_provider_for_region(region).exchange_public_token(token)
|
||||
|
||||
new_plaid_item = create!(
|
||||
name: item_name,
|
||||
plaid_id: response.item_id,
|
||||
access_token: response.access_token,
|
||||
plaid_region: region
|
||||
)
|
||||
|
||||
new_plaid_item.sync_later
|
||||
end
|
||||
end
|
||||
|
||||
def sync_data(sync, start_date: nil)
|
||||
begin
|
||||
Rails.logger.info("Fetching and loading Plaid data")
|
||||
fetch_and_load_plaid_data(sync)
|
||||
update!(status: :good) if requires_update?
|
||||
|
||||
# Schedule account syncs
|
||||
accounts.each do |account|
|
||||
account.sync_later(start_date: start_date, parent_sync: sync)
|
||||
end
|
||||
|
||||
Rails.logger.info("Plaid data fetched and loaded")
|
||||
rescue Plaid::ApiError => e
|
||||
handle_plaid_error(e)
|
||||
raise e
|
||||
end
|
||||
end
|
||||
|
||||
def get_update_link_token(webhooks_url:, redirect_url:)
|
||||
begin
|
||||
family.get_link_token(
|
||||
|
|
@ -76,9 +43,8 @@ class PlaidItem < ApplicationRecord
|
|||
end
|
||||
end
|
||||
|
||||
def post_sync(sync)
|
||||
auto_match_categories!
|
||||
family.broadcast_refresh
|
||||
def build_category_alias_matcher(user_categories)
|
||||
Provider::Plaid::CategoryAliasMatcher.new(user_categories)
|
||||
end
|
||||
|
||||
def destroy_later
|
||||
|
|
@ -86,6 +52,14 @@ class PlaidItem < ApplicationRecord
|
|||
DestroyJob.perform_later(self)
|
||||
end
|
||||
|
||||
def syncing?
|
||||
Sync.joins("LEFT JOIN accounts a ON a.id = syncs.syncable_id AND syncs.syncable_type = 'Account'")
|
||||
.joins("LEFT JOIN plaid_accounts pa ON pa.id = a.plaid_account_id")
|
||||
.where("syncs.syncable_id = ? OR pa.plaid_item_id = ?", id, id)
|
||||
.incomplete
|
||||
.exists?
|
||||
end
|
||||
|
||||
def auto_match_categories!
|
||||
if family.categories.none?
|
||||
family.categories.bootstrap!
|
||||
|
|
@ -117,123 +91,11 @@ class PlaidItem < ApplicationRecord
|
|||
end
|
||||
|
||||
private
|
||||
def fetch_and_load_plaid_data(sync)
|
||||
data = {}
|
||||
|
||||
# Log what we're about to fetch
|
||||
Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)"
|
||||
|
||||
item = plaid_provider.get_item(access_token).item
|
||||
update!(available_products: item.available_products, billed_products: item.billed_products)
|
||||
|
||||
# Institution details
|
||||
if item.institution_id.present?
|
||||
begin
|
||||
Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}"
|
||||
institution = plaid_provider.get_institution(item.institution_id)
|
||||
update!(
|
||||
institution_id: item.institution_id,
|
||||
institution_url: institution.institution.url,
|
||||
institution_color: institution.institution.primary_color
|
||||
)
|
||||
rescue Plaid::ApiError => e
|
||||
Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
# Accounts
|
||||
fetched_accounts = plaid_provider.get_item_accounts(self).accounts
|
||||
data[:accounts] = fetched_accounts || []
|
||||
sync.update!(data: data)
|
||||
Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})"
|
||||
|
||||
internal_plaid_accounts = fetched_accounts.map do |account|
|
||||
internal_plaid_account = plaid_accounts.find_or_create_from_plaid_data!(account, family)
|
||||
internal_plaid_account.sync_account_data!(account)
|
||||
internal_plaid_account
|
||||
end
|
||||
|
||||
# Transactions
|
||||
fetched_transactions = safe_fetch_plaid_data(:get_item_transactions)
|
||||
data[:transactions] = fetched_transactions || []
|
||||
sync.update!(data: data)
|
||||
|
||||
if fetched_transactions
|
||||
Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})"
|
||||
transaction do
|
||||
internal_plaid_accounts.each do |internal_plaid_account|
|
||||
added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id }
|
||||
modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id }
|
||||
removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id }
|
||||
|
||||
internal_plaid_account.sync_transactions!(added:, modified:, removed:)
|
||||
end
|
||||
|
||||
update!(next_cursor: fetched_transactions.cursor)
|
||||
end
|
||||
end
|
||||
|
||||
# Investments
|
||||
fetched_investments = safe_fetch_plaid_data(:get_item_investments)
|
||||
data[:investments] = fetched_investments || []
|
||||
sync.update!(data: data)
|
||||
|
||||
if fetched_investments
|
||||
Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})"
|
||||
transaction do
|
||||
internal_plaid_accounts.each do |internal_plaid_account|
|
||||
transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id }
|
||||
holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id }
|
||||
securities = fetched_investments.securities
|
||||
|
||||
internal_plaid_account.sync_investments!(transactions:, holdings:, securities:)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Liabilities
|
||||
fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities)
|
||||
data[:liabilities] = fetched_liabilities || []
|
||||
sync.update!(data: data)
|
||||
|
||||
if fetched_liabilities
|
||||
Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})"
|
||||
transaction do
|
||||
internal_plaid_accounts.each do |internal_plaid_account|
|
||||
credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id }
|
||||
mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id }
|
||||
student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id }
|
||||
|
||||
internal_plaid_account.sync_credit_data!(credit) if credit
|
||||
internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage
|
||||
internal_plaid_account.sync_student_loan_data!(student) if student
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def safe_fetch_plaid_data(method)
|
||||
begin
|
||||
plaid_provider.send(method, self)
|
||||
rescue Plaid::ApiError => e
|
||||
Rails.logger.warn("Error fetching #{method} for item #{id}: #{e.message}")
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def remove_plaid_item
|
||||
plaid_provider.remove_item(access_token)
|
||||
rescue StandardError => e
|
||||
Rails.logger.warn("Failed to remove Plaid item #{id}: #{e.message}")
|
||||
end
|
||||
|
||||
def handle_plaid_error(error)
|
||||
error_body = JSON.parse(error.response_body)
|
||||
|
||||
if error_body["error_code"] == "ITEM_LOGIN_REQUIRED"
|
||||
update!(status: :requires_update)
|
||||
end
|
||||
end
|
||||
|
||||
class PlaidConnectionLostError < StandardError; end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue