Docker/Maybe
Docker/Maybe
1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-02 20:15:22 +02:00
Code Issues Releases Activity Actions

Account::Sync model and test fixture simplifications (#968)

Browse source 
* Add sync model

* Fresh fixtures for sync tests

* Sync tests overhaul

* Fix entry tests

* Complete remaining model test updates

* Update system tests

* Update demo data task

* Add system tests back to PR checks

* More simplifications, add empty family to fixtures for easier testing
This commit is contained in:
Zach Gollwitzer 2024-07-10 11:22:59 -04:00 • committed by GitHub
parent de5a2e55b3
commit c6bdf49f10
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
60 changed files with 929 additions and 1353 deletions
Download patch file Download diff file Expand all files Collapse all files

17
app/models/account.rb
View file

@ -14,10 +14,10 @@ class Account < ApplicationRecord
has_many :valuations, through: :entries, source: :entryable, source_type: "Account::Valuation"
has_many :balances, dependent: :destroy
has_many :imports, dependent: :destroy
has_many :syncs, dependent: :destroy
monetize :balance
enum :status, { ok: "ok", syncing: "syncing", error: "error" }, validate: true
enum :classification, { asset: "asset", liability: "liability" }, validate: { allow_nil: true }
scope :active, -> { where(is_active: true) }
@ -73,24 +73,15 @@ class Account < ApplicationRecord
end
end
def balance_on(date)
balances.where("date <= ?", date).order(date: :desc).first&.balance
def alert
latest_sync = syncs.latest
[ latest_sync&.error, *latest_sync&.warnings ].compact.first
end
def favorable_direction
classification == "asset" ? "up" : "down"
end
# e.g. Wise, Revolut accounts that have transactions in multiple currencies
def multi_currency?
entries.select(:currency).distinct.count > 1
end
# e.g. Accounts denominated in currency other than family currency
def foreign_currency?
currency != family.currency
end
def series(period: Period.all, currency: self.currency)
balance_series = balances.in_period(period).where(currency: Money::Currency.new(currency).iso_code)

1
app/models/account/balance.rb
View file

@ -5,4 +5,5 @@ class Account::Balance < ApplicationRecord
validates :account, :date, :balance, presence: true
monetize :balance
scope :in_period, ->(period) { period.date_range.nil? ? all : where(date: period.date_range) }
scope :chronological, -> { order(:date) }
end

116
app/models/account/balance/calculator.rb
View file

@ -1,116 +0,0 @@
class Account::Balance::Calculator
attr_reader :errors, :warnings
def initialize(account, options = {})
@errors = []
@warnings = []
@account = account
@calc_start_date = calculate_sync_start(options[:calc_start_date])
end
def daily_balances
@daily_balances ||= calculate_daily_balances
end
private
attr_reader :calc_start_date, :account
def calculate_sync_start(provided_start_date = nil)
if account.balances.any?
[ provided_start_date, account.effective_start_date ].compact.max
else
account.effective_start_date
end
end
def calculate_daily_balances
prior_balance = nil
calculated_balances = (calc_start_date..Date.current).map do |date|
valuation_entry = find_valuation_entry(date)
if valuation_entry
current_balance = valuation_entry.amount
elsif prior_balance.nil?
current_balance = implied_start_balance
else
txn_entries = syncable_transaction_entries.select { |e| e.date == date }
txn_flows = transaction_flows(txn_entries)
current_balance = prior_balance - txn_flows
end
prior_balance = current_balance
{ date:, balance: current_balance, currency: account.currency, updated_at: Time.current }
end
if account.foreign_currency?
calculated_balances.concat(convert_balances_to_family_currency(calculated_balances))
end
calculated_balances
end
def syncable_entries
@entries ||= account.entries.where("date >= ?", calc_start_date).to_a
end
def syncable_transaction_entries
@syncable_transaction_entries ||= syncable_entries.select { |e| e.account_transaction? }
end
def find_valuation_entry(date)
syncable_entries.find { |entry| entry.date == date && entry.account_valuation? }
end
def transaction_flows(transaction_entries)
converted_entries = transaction_entries.map { |entry| convert_entry_to_account_currency(entry) }.compact
flows = converted_entries.sum(&:amount)
flows *= -1 if account.liability?
flows
end
def convert_balances_to_family_currency(balances)
rates = ExchangeRate.find_rates(
from: account.currency,
to: account.family.currency,
start_date: calc_start_date
).to_a
# Abort conversion if some required rates are missing
if rates.length != balances.length
@errors << :sync_message_missing_rates
return []
end
balances.map do |balance|
rate = rates.find { |r| r.date == balance[:date] }
converted_balance = balance[:balance] * rate&.rate
{ date: balance[:date], balance: converted_balance, currency: account.family.currency, updated_at: Time.current }
end
end
# Multi-currency accounts have transactions in many currencies
def convert_entry_to_account_currency(entry)
return entry if entry.currency == account.currency
converted_entry = entry.dup
rate = ExchangeRate.find_rate(from: entry.currency, to: account.currency, date: entry.date)
unless rate
@errors << :sync_message_missing_rates
return nil
end
converted_entry.currency = account.currency
converted_entry.amount = entry.amount * rate.rate
converted_entry
end
def implied_start_balance
transaction_entries = syncable_transaction_entries.select { |e| e.date > calc_start_date }
account.balance.to_d + transaction_flows(transaction_entries)
end
end

129
app/models/account/balance/syncer.rb Normal file
View file

@ -0,0 +1,129 @@
class Account::Balance::Syncer
attr_reader :warnings
def initialize(account, start_date: nil)
@account = account
@warnings = []
@sync_start_date = calculate_sync_start_date(start_date)
end
def run
daily_balances = calculate_daily_balances
daily_balances += calculate_converted_balances(daily_balances) if account.currency != account.family.currency
Account::Balance.transaction do
upsert_balances!(daily_balances)
purge_stale_balances!
end
end
private
attr_reader :sync_start_date, :account
def upsert_balances!(balances)
balances_to_upsert = balances.map do |balance|
{
date: balance.date,
balance: balance.balance,
currency: balance.currency,
updated_at: Time.now
}
end
account.balances.upsert_all(balances_to_upsert, unique_by: %i[account_id date currency])
end
def purge_stale_balances!
account.balances.delete_by("date < ?", account_start_date)
end
def calculate_balance_for_date(date, entries:, prior_balance:)
valuation = entries.find { |e| e.date == date && e.account_valuation? }
return valuation.amount if valuation
return derived_sync_start_balance(entries) unless prior_balance
transactions = entries.select { |e| e.date == date && e.account_transaction? }
prior_balance - net_transaction_flows(transactions)
end
def calculate_daily_balances
entries = account.entries.where("date >= ?", sync_start_date).to_a
prior_balance = find_prior_balance
daily_balances = (sync_start_date...Date.current).map do |date|
current_balance = calculate_balance_for_date(date, entries:, prior_balance:)
prior_balance = current_balance
build_balance(date, current_balance)
end
# Last balance of series is always equal to account balance
daily_balances << build_balance(Date.current, account.balance)
end
def calculate_converted_balances(balances)
from_currency = account.currency
to_currency = account.family.currency
exchange_rates = ExchangeRate.find_rates from: from_currency,
to: to_currency,
start_date: sync_start_date
balances.map do |balance|
exchange_rate = exchange_rates.find { |er| er.date == balance.date }
raise Money::ConversionError.new("missing exchange rate from #{from_currency} to #{to_currency} on date #{balance.date}") unless exchange_rate
build_balance(balance.date, exchange_rate.rate * balance.balance, to_currency)
end
rescue Money::ConversionError
@warnings << "missing exchange rates from #{from_currency} to #{to_currency}"
[]
end
def build_balance(date, balance, currency = nil)
account.balances.build \
date: date,
balance: balance,
currency: currency || account.currency
end
def derived_sync_start_balance(entries)
transactions = entries.select { |e| e.account_transaction? && e.date > sync_start_date }
account.balance + net_transaction_flows(transactions)
end
def find_prior_balance
account.balances.where("date < ?", sync_start_date).order(date: :desc).first&.balance
end
def net_transaction_flows(transactions, target_currency = account.currency)
converted_transaction_amounts = transactions.map { |t| t.amount_money.exchange_to(target_currency, date: t.date) }
flows = converted_transaction_amounts.sum(&:amount)
account.liability? ? flows * -1 : flows
end
def account_start_date
@account_start_date ||= begin
oldest_entry_date = account.entries.chronological.first.try(:date)
return Date.current unless oldest_entry_date
oldest_entry_is_valuation = account.entries.account_valuations.where(date: oldest_entry_date).exists?
oldest_entry_date -= 1 unless oldest_entry_is_valuation
oldest_entry_date
end
end
def calculate_sync_start_date(provided_start_date)
[ provided_start_date, account_start_date ].compact.max
end
end

18
app/models/account/entry.rb
View file

@ -33,7 +33,7 @@ class Account::Entry < ApplicationRecord
sync_start_date = [ date_previously_was, date ].compact.min
end
account.sync_later(sync_start_date)
account.sync_later(start_date: sync_start_date)
end
def inflow?
@ -122,19 +122,17 @@ class Account::Entry < ApplicationRecord
end
def income_total(currency = "USD")
account_transactions.includes(:entryable)
without_transfers.account_transactions.includes(:entryable)
.where("account_entries.amount <= 0")
.where("account_entries.currency = ?", currency)
.reject { |e| e.marked_as_transfer? }
.sum(&:amount_money)
.map { |e| e.amount_money.exchange_to(currency, date: e.date, fallback_rate: 0) }
.sum
end
def expense_total(currency = "USD")
account_transactions.includes(:entryable)
.where("account_entries.amount > 0")
.where("account_entries.currency = ?", currency)
.reject { |e| e.marked_as_transfer? }
.sum(&:amount_money)
without_transfers.account_transactions.includes(:entryable)
.where("account_entries.amount > 0")
.map { |e| e.amount_money.exchange_to(currency, date: e.date, fallback_rate: 0) }
.sum
end
def search(params)

51
app/models/account/sync.rb Normal file
View file

@ -0,0 +1,51 @@
class Account::Sync < ApplicationRecord
belongs_to :account
enum :status, { pending: "pending", syncing: "syncing", completed: "completed", failed: "failed" }
class << self
def for(account, start_date: nil)
create! account: account, start_date: start_date
end
def latest
order(created_at: :desc).first
end
end
def run
start!
sync_balances
complete!
rescue StandardError => error
fail! error
end
private
def sync_balances
syncer = Account::Balance::Syncer.new(account, start_date: start_date)
syncer.run
append_warnings(syncer.warnings)
end
def append_warnings(new_warnings)
update! warnings: warnings + new_warnings
end
def start!
update! status: "syncing", last_ran_at: Time.now
end
def complete!
update! status: "completed"
end
def fail!(error)
update! status: "failed", error: error.message
end
end

88
app/models/account/syncable.rb
View file

@ -1,89 +1,21 @@
module Account::Syncable
extend ActiveSupport::Concern
def sync_later(start_date = nil)
AccountSyncJob.perform_later(self, start_date)
class_methods do
def sync(start_date: nil)
all.each { |a| a.sync_later(start_date: start_date) }
end
end
def sync(start_date = nil)
update!(status: "syncing")
if multi_currency? || foreign_currency?
sync_exchange_rates
end
calculator = Account::Balance::Calculator.new(self, { calc_start_date: start_date })
self.balances.upsert_all(calculator.daily_balances, unique_by: :index_account_balances_on_account_id_date_currency_unique)
self.balances.where("date < ?", effective_start_date).delete_all
new_balance = calculator.daily_balances.select { |b| b[:currency] == self.currency }.last[:balance]
update! \
status: "ok",
last_sync_date: Date.current,
balance: new_balance,
sync_errors: calculator.errors,
sync_warnings: calculator.warnings
rescue => e
update!(status: "error", sync_errors: [ :sync_message_unknown_error ])
logger.error("Failed to sync account #{id}: #{e.message}")
def syncing?
syncs.syncing.any?
end
def can_sync?
# Skip account sync if account is not active or the sync process is already running
return false unless is_active
return false if syncing?
# If last_sync_date is blank (i.e. the account has never been synced before) allow syncing
return true if last_sync_date.blank?
# If last_sync_date is not today, allow syncing
last_sync_date != Date.today
def sync_later(start_date: nil)
AccountSyncJob.perform_later(self, start_date: start_date)
end
# The earliest date we can calculate a balance for
def effective_start_date
@effective_start_date ||= entries.order(:date).first.try(:date) || Date.current
end
# Finds all the rate pairs that are required to calculate balances for an account and syncs them
def sync_exchange_rates
rate_candidates = []
if multi_currency?
transactions_in_foreign_currency = self.entries.where.not(currency: self.currency).pluck(:currency, :date).uniq
transactions_in_foreign_currency.each do |currency, date|
rate_candidates << { date: date, from_currency: currency, to_currency: self.currency }
end
end
if foreign_currency?
(effective_start_date..Date.current).each do |date|
rate_candidates << { date: date, from_currency: self.currency, to_currency: self.family.currency }
end
end
return if rate_candidates.blank?
existing_rates = ExchangeRate.where(
from_currency: rate_candidates.map { |rc| rc[:from_currency] },
to_currency: rate_candidates.map { |rc| rc[:to_currency] },
date: rate_candidates.map { |rc| rc[:date] }
).pluck(:from_currency, :to_currency, :date)
# Convert to a set for faster lookup
existing_rates_set = existing_rates.map { |er| [ er[0], er[1], er[2].to_s ] }.to_set
rate_candidates.each do |rate_candidate|
rc_from = rate_candidate[:from_currency]
rc_to = rate_candidate[:to_currency]
rc_date = rate_candidate[:date]
next if existing_rates_set.include?([ rc_from, rc_to, rc_date.to_s ])
logger.info "Fetching exchange rate from provider for account #{self.name}: #{self.id} (#{rc_from} to #{rc_to} on #{rc_date})"
ExchangeRate.find_rate_or_fetch from: rc_from, to: rc_to, date: rc_date
end
nil
def sync(start_date: nil)
Account::Sync.for(self, start_date: start_date).run
end
end

4
app/models/family.rb
View file

@ -104,7 +104,7 @@ class Family < ApplicationRecord
Money.new(accounts.active.liabilities.map { |account| account.balance_money.exchange_to(currency, fallback_rate: 0) }.sum, currency)
end
def sync_accounts
accounts.each { |account| account.sync_later if account.can_sync? }
def sync(start_date: nil)
accounts.active.sync(start_date: start_date)
end
end