Docker/Maybe
Docker/Maybe
1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-05 05:25:24 +02:00
Code Issues Releases Activity Actions

Populate holdings for "offline" securities properly (#1958)

Browse source 
* Placeholder logic for missing prices

* Generate holdings properly for "offline" securities

* Separate forward and reverse calculators for holdings and balances

* Remove unnecessary currency conversion during sync

* Clearer sync process

* Move price caching logic to dedicated model

* Base holding calculator

* Base calculator for balances

* Finish balance calculators

* Better naming

* Logs cleanup

* Remove stale data type

* Remove stale test

* Fix price lookup logic for holdings sync

* Fix Plaid item sync regression

* Remove temp logging

* Calculate cash and holdings series

* Add holdings, cash, and balance series dropdown for investments
This commit is contained in:
Zach Gollwitzer 2025-03-07 17:35:55 -05:00 • committed by GitHub
parent 26762477a3
commit eac5d5e663
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 1109 additions and 808 deletions
Download patch file Download diff file Expand all files Collapse all files

33
app/models/account.rb
View file

@ -1,11 +1,10 @@
class Account < ApplicationRecord
include Syncable, Monetizable, Issuable, Chartable
include Syncable, Monetizable, Issuable, Chartable, Enrichable, Linkable
validates :name, :balance, :currency, presence: true
belongs_to :family
belongs_to :import, optional: true
belongs_to :plaid_account, optional: true
has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping"
has_many :entries, dependent: :destroy, class_name: "Account::Entry"
@ -75,7 +74,16 @@ class Account < ApplicationRecord
def sync_data(start_date: nil)
update!(last_synced_at: Time.current)
Syncer.new(self, start_date: start_date).run
Rails.logger.info("Auto-matching transfers")
family.auto_match_transfers!
Rails.logger.info("Processing balances (#{linked? ? 'reverse' : 'forward'})")
sync_balances
if enrichable?
Rails.logger.info("Enriching transaction data")
enrich_data
end
end
def post_sync
@ -93,10 +101,6 @@ class Account < ApplicationRecord
holdings.where(currency: currency, date: holdings.maximum(:date)).order(amount: :desc)
end
def enrich_data
DataEnricher.new(self).run
end
def update_with_sync!(attributes)
should_update_balance = attributes[:balance] && attributes[:balance].to_d != balance
@ -123,11 +127,14 @@ class Account < ApplicationRecord
end
end
def sparkline_series
cache_key = family.build_cache_key("#{id}_sparkline")
Rails.cache.fetch(cache_key) do
balance_series
end
def start_date
first_entry_date = entries.minimum(:date) || Date.current
first_entry_date - 1.day
end
private
def sync_balances
strategy = linked? ? :reverse : :forward
Balance::Syncer.new(self, strategy: strategy).sync_balances
end
end

35
app/models/account/balance/base_calculator.rb Normal file
View file

@ -0,0 +1,35 @@
class Account::Balance::BaseCalculator
attr_reader :account
def initialize(account)
@account = account
end
def calculate
Rails.logger.tagged(self.class.name) do
calculate_balances
end
end
private
def sync_cache
@sync_cache ||= Account::Balance::SyncCache.new(account)
end
def build_balance(date, cash_balance, holdings_value)
Account::Balance.new(
account_id: account.id,
date: date,
balance: holdings_value + cash_balance,
cash_balance: cash_balance,
currency: account.currency
)
end
def calculate_next_balance(prior_balance, transactions, direction: :forward)
flows = transactions.sum(&:amount)
negated = direction == :forward ? account.asset? : account.liability?
flows *= -1 if negated
prior_balance + flows
end
end

28
app/models/account/balance/forward_calculator.rb Normal file
View file

@ -0,0 +1,28 @@
class Account::Balance::ForwardCalculator < Account::Balance::BaseCalculator
private
def calculate_balances
current_cash_balance = 0
next_cash_balance = nil
@balances = []
account.start_date.upto(Date.current).each do |date|
entries = sync_cache.get_entries(date)
holdings = sync_cache.get_holdings(date)
holdings_value = holdings.sum(&:amount)
valuation = sync_cache.get_valuation(date)
next_cash_balance = if valuation
valuation.amount - holdings_value
else
calculate_next_balance(current_cash_balance, entries, direction: :forward)
end
@balances << build_balance(date, next_cash_balance, holdings_value)
current_cash_balance = next_cash_balance
end
@balances
end
end

32
app/models/account/balance/reverse_calculator.rb Normal file
View file

@ -0,0 +1,32 @@
class Account::Balance::ReverseCalculator < Account::Balance::BaseCalculator
private
def calculate_balances
current_cash_balance = account.cash_balance
previous_cash_balance = nil
@balances = []
Date.current.downto(account.start_date).map do |date|
entries = sync_cache.get_entries(date)
holdings = sync_cache.get_holdings(date)
holdings_value = holdings.sum(&:amount)
valuation = sync_cache.get_valuation(date)
previous_cash_balance = if valuation
valuation.amount - holdings_value
else
calculate_next_balance(current_cash_balance, entries, direction: :reverse)
end
if valuation.present?
@balances << build_balance(date, previous_cash_balance, holdings_value)
else
@balances << build_balance(date, current_cash_balance, holdings_value)
end
current_cash_balance = previous_cash_balance
end
@balances
end
end

46
app/models/account/balance/sync_cache.rb Normal file
View file

@ -0,0 +1,46 @@
class Account::Balance::SyncCache
def initialize(account)
@account = account
end
def get_valuation(date)
converted_entries.find { |e| e.date == date && e.account_valuation? }
end
def get_holdings(date)
converted_holdings.select { |h| h.date == date }
end
def get_entries(date)
converted_entries.select { |e| e.date == date && (e.account_transaction? || e.account_trade?) }
end
private
attr_reader :account
def converted_entries
@converted_entries ||= account.entries.order(:date).to_a.map do |e|
converted_entry = e.dup
converted_entry.amount = converted_entry.amount_money.exchange_to(
account.currency,
date: e.date,
fallback_rate: 1
).amount
converted_entry.currency = account.currency
converted_entry
end
end
def converted_holdings
@converted_holdings ||= account.holdings.map do |h|
converted_holding = h.dup
converted_holding.amount = converted_holding.amount_money.exchange_to(
account.currency,
date: h.date,
fallback_rate: 1
).amount
converted_holding.currency = account.currency
converted_holding
end
end
end

69
app/models/account/balance/syncer.rb Normal file
View file

@ -0,0 +1,69 @@
class Account::Balance::Syncer
attr_reader :account, :strategy
def initialize(account, strategy:)
@account = account
@strategy = strategy
end
def sync_balances
Account::Balance.transaction do
sync_holdings
calculate_balances
Rails.logger.info("Persisting #{@balances.size} balances")
persist_balances
purge_stale_balances
if strategy == :forward
update_account_info
end
end
end
private
def sync_holdings
@holdings = Account::Holding::Syncer.new(account, strategy: strategy).sync_holdings
end
def update_account_info
calculated_balance = @balances.sort_by(&:date).last&.balance || 0
calculated_holdings_value = @holdings.select { |h| h.date == Date.current }.sum(&:amount) || 0
calculated_cash_balance = calculated_balance - calculated_holdings_value
Rails.logger.info("Balance update: cash=#{calculated_cash_balance}, total=#{calculated_balance}")
account.update!(
balance: calculated_balance,
cash_balance: calculated_cash_balance
)
end
def calculate_balances
@balances = calculator.calculate
end
def persist_balances
current_time = Time.now
account.balances.upsert_all(
@balances.map { |b| b.attributes
.slice("date", "balance", "cash_balance", "currency")
.merge("updated_at" => current_time) },
unique_by: %i[account_id date currency]
)
end
def purge_stale_balances
deleted_count = account.balances.delete_by("date < ?", account.start_date)
Rails.logger.info("Purged #{deleted_count} stale balances") if deleted_count > 0
end
def calculator
if strategy == :reverse
Account::Balance::ReverseCalculator.new(account)
else
Account::Balance::ForwardCalculator.new(account)
end
end
end

124
app/models/account/balance_calculator.rb
View file

@ -1,124 +0,0 @@
class Account::BalanceCalculator
def initialize(account, holdings: nil)
@account = account
@holdings = holdings || []
end
def calculate(reverse: false, start_date: nil)
Rails.logger.tagged("Account::BalanceCalculator") do
Rails.logger.info("Calculating cash balances with strategy: #{reverse ? "reverse sync" : "forward sync"}")
cash_balances = reverse ? reverse_cash_balances : forward_cash_balances
cash_balances.map do |balance|
holdings_value = converted_holdings.select { |h| h.date == balance.date }.sum(&:amount)
balance.balance = balance.balance + holdings_value
balance
end.compact
end
end
private
attr_reader :account, :holdings
def oldest_date
converted_entries.first ? converted_entries.first.date - 1.day : Date.current
end
def reverse_cash_balances
prior_balance = account.cash_balance
Date.current.downto(oldest_date).map do |date|
entries_for_date = converted_entries.select { |e| e.date == date }
holdings_for_date = converted_holdings.select { |h| h.date == date }
valuation = entries_for_date.find { |e| e.account_valuation? }
current_balance = if valuation
# To get this to a cash valuation, we back out holdings value on day
valuation.amount - holdings_for_date.sum(&:amount)
else
transactions = entries_for_date.select { |e| e.account_transaction? || e.account_trade? }
calculate_balance(prior_balance, transactions)
end
balance_record = Account::Balance.new(
account: account,
date: date,
balance: valuation ? current_balance : prior_balance,
cash_balance: valuation ? current_balance : prior_balance,
currency: account.currency
)
prior_balance = current_balance
balance_record
end
end
def forward_cash_balances
prior_balance = 0
current_balance = nil
oldest_date.upto(Date.current).map do |date|
entries_for_date = converted_entries.select { |e| e.date == date }
holdings_for_date = converted_holdings.select { |h| h.date == date }
valuation = entries_for_date.find { |e| e.account_valuation? }
current_balance = if valuation
# To get this to a cash valuation, we back out holdings value on day
valuation.amount - holdings_for_date.sum(&:amount)
else
transactions = entries_for_date.select { |e| e.account_transaction? || e.account_trade? }
calculate_balance(prior_balance, transactions, inverse: true)
end
balance_record = Account::Balance.new(
account: account,
date: date,
balance: current_balance,
cash_balance: current_balance,
currency: account.currency
)
prior_balance = current_balance
balance_record
end
end
def converted_entries
@converted_entries ||= @account.entries.order(:date).to_a.map do |e|
converted_entry = e.dup
converted_entry.amount = converted_entry.amount_money.exchange_to(
account.currency,
date: e.date,
fallback_rate: 1
).amount
converted_entry.currency = account.currency
converted_entry
end
end
def converted_holdings
@converted_holdings ||= holdings.map do |h|
converted_holding = h.dup
converted_holding.amount = converted_holding.amount_money.exchange_to(
account.currency,
date: h.date,
fallback_rate: 1
).amount
converted_holding.currency = account.currency
converted_holding
end
end
def calculate_balance(prior_balance, transactions, inverse: false)
flows = transactions.sum(&:amount)
negated = inverse ? account.asset? : account.liability?
flows *= -1 if negated
prior_balance + flows
end
end

59
app/models/account/chartable.rb
View file

@ -2,7 +2,9 @@ module Account::Chartable
extend ActiveSupport::Concern
class_methods do
def balance_series(currency:, period: Period.last_30_days, favorable_direction: "up")
def balance_series(currency:, period: Period.last_30_days, favorable_direction: "up", view: :balance)
raise ArgumentError, "Invalid view type" unless [ :balance, :cash_balance, :holdings_balance ].include?(view.to_sym)
balances = Account::Balance.find_by_sql([
balance_series_query,
{
@ -21,8 +23,8 @@ module Account::Chartable
date: curr.date,
date_formatted: I18n.l(curr.date, format: :long),
trend: Trend.new(
current: Money.new(curr.balance, currency),
previous: prev.nil? ? nil : Money.new(prev.balance, currency),
current: Money.new(balance_value_for(curr, view), currency),
previous: prev.nil? ? nil : Money.new(balance_value_for(prev, view), currency),
favorable_direction: favorable_direction
)
)
@ -33,8 +35,8 @@ module Account::Chartable
end_date: period.end_date,
interval: period.interval,
trend: Trend.new(
current: Money.new(balances.last&.balance || 0, currency),
previous: Money.new(balances.first&.balance || 0, currency),
current: Money.new(balance_value_for(balances.last, view) || 0, currency),
previous: Money.new(balance_value_for(balances.first, view) || 0, currency),
favorable_direction: favorable_direction
),
values: values
@ -52,6 +54,8 @@ module Account::Chartable
SELECT
d.date,
SUM(CASE WHEN accounts.classification = 'asset' THEN ab.balance ELSE -ab.balance END * COALESCE(er.rate, 1)) as balance,
SUM(CASE WHEN accounts.classification = 'asset' THEN ab.cash_balance ELSE -ab.cash_balance END * COALESCE(er.rate, 1)) as cash_balance,
SUM(CASE WHEN accounts.classification = 'asset' THEN ab.balance - ab.cash_balance ELSE 0 END * COALESCE(er.rate, 1)) as holdings_balance,
COUNT(CASE WHEN accounts.currency <> :target_currency AND er.rate IS NULL THEN 1 END) as missing_rates
FROM dates d
LEFT JOIN accounts ON accounts.id IN (#{all.select(:id).to_sql})
@ -70,26 +74,46 @@ module Account::Chartable
SQL
end
def balance_value_for(balance_record, view)
return 0 if balance_record.nil?
case view.to_sym
when :balance then balance_record.balance
when :cash_balance then balance_record.cash_balance
when :holdings_balance then balance_record.holdings_balance
else
raise ArgumentError, "Invalid view type: #{view}"
end
end
def invert_balances(balances)
balances.map do |balance|
balance.balance = -balance.balance
balance.cash_balance = -balance.cash_balance
balance.holdings_balance = -balance.holdings_balance
balance
end
end
def gapfill_balances(balances)
gapfilled = []
prev = nil
prev_balance = nil
[ nil, *balances ].each_cons(2).each_with_index do |(prev, curr), index|
if index == 0 && curr.balance.nil?
curr.balance = 0 # Ensure all series start with a non-nil balance
elsif curr.balance.nil?
curr.balance = prev.balance
balances.each do |curr|
if prev.nil?
# Initialize first record with zeros if nil
curr.balance ||= 0
curr.cash_balance ||= 0
curr.holdings_balance ||= 0
else
# Copy previous values for nil fields
curr.balance ||= prev.balance
curr.cash_balance ||= prev.cash_balance
curr.holdings_balance ||= prev.holdings_balance
end
gapfilled << curr
prev = curr
end
gapfilled
@ -100,11 +124,20 @@ module Account::Chartable
classification == "asset" ? "up" : "down"
end
def balance_series(period: Period.last_30_days)
def balance_series(period: Period.last_30_days, view: :balance)
self.class.where(id: self.id).balance_series(
currency: currency,
period: period,
view: view,
favorable_direction: favorable_direction
)
end
def sparkline_series
cache_key = family.build_cache_key("#{id}_sparkline")
Rails.cache.fetch(cache_key) do
balance_series
end
end
end

12
app/models/account/enrichable.rb Normal file
View file

@ -0,0 +1,12 @@
module Account::Enrichable
extend ActiveSupport::Concern
def enrich_data
DataEnricher.new(self).run
end
private
def enrichable?
family.data_enrichment_enabled? || (linked? && Rails.application.config.app_mode.hosted?)
end
end

2
app/models/account/holding.rb
View file

@ -1,5 +1,5 @@
class Account::Holding < ApplicationRecord
include Monetizable
include Monetizable, Gapfillable
monetize :amount

63
app/models/account/holding/base_calculator.rb Normal file
View file

@ -0,0 +1,63 @@
class Account::Holding::BaseCalculator
attr_reader :account
def initialize(account)
@account = account
end
def calculate
Rails.logger.tagged(self.class.name) do
holdings = calculate_holdings
Account::Holding.gapfill(holdings)
end
end
private
def portfolio_cache
@portfolio_cache ||= Account::Holding::PortfolioCache.new(account)
end
def empty_portfolio
securities = portfolio_cache.get_securities
securities.each_with_object({}) { |security, hash| hash[security.id] = 0 }
end
def generate_starting_portfolio
empty_portfolio
end
def transform_portfolio(previous_portfolio, trade_entries, direction: :forward)
new_quantities = previous_portfolio.dup
trade_entries.each do |trade_entry|
trade = trade_entry.entryable
security_id = trade.security_id
qty_change = trade.qty
qty_change = qty_change * -1 if direction == :reverse
new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change
end
new_quantities
end
def build_holdings(portfolio, date)
portfolio.map do |security_id, qty|
price = portfolio_cache.get_price(security_id, date)
if price.nil?
Rails.logger.warn "No price found for security #{security_id} on #{date}"
next
end
Account::Holding.new(
account_id: account.id,
security_id: security_id,
date: date,
qty: qty,
price: price.price,
currency: price.currency,
amount: qty * price.price
)
end.compact
end
end

21
app/models/account/holding/forward_calculator.rb Normal file
View file

@ -0,0 +1,21 @@
class Account::Holding::ForwardCalculator < Account::Holding::BaseCalculator
private
def portfolio_cache
@portfolio_cache ||= Account::Holding::PortfolioCache.new(account)
end
def calculate_holdings
current_portfolio = generate_starting_portfolio
next_portfolio = {}
holdings = []
account.start_date.upto(Date.current).each do |date|
trades = portfolio_cache.get_trades(date: date)
next_portfolio = transform_portfolio(current_portfolio, trades, direction: :forward)
holdings += build_holdings(next_portfolio, date)
current_portfolio = next_portfolio
end
holdings
end
end

38
app/models/account/holding/gapfillable.rb Normal file
View file

@ -0,0 +1,38 @@
module Account::Holding::Gapfillable
extend ActiveSupport::Concern
class_methods do
def gapfill(holdings)
filled_holdings = []
holdings.group_by { |h| h.security_id }.each do |security_id, security_holdings|
next if security_holdings.empty?
sorted = security_holdings.sort_by(&:date)
previous_holding = sorted.first
sorted.first.date.upto(Date.current) do |date|
holding = security_holdings.find { |h| h.date == date }
if holding
filled_holdings << holding
previous_holding = holding
else
# Create a new holding based on the previous day's data
filled_holdings << Account::Holding.new(
account: previous_holding.account,
security: previous_holding.security,
date: date,
qty: previous_holding.qty,
price: previous_holding.price,
currency: previous_holding.currency,
amount: previous_holding.amount
)
end
end
end
filled_holdings
end
end
end

132
app/models/account/holding/portfolio_cache.rb Normal file
View file

@ -0,0 +1,132 @@
class Account::Holding::PortfolioCache
attr_reader :account, :use_holdings
class SecurityNotFound < StandardError
def initialize(security_id, account_id)
super("Security id=#{security_id} not found in portfolio cache for account #{account_id}. This should not happen unless securities were preloaded incorrectly.")
end
end
def initialize(account, use_holdings: false)
@account = account
@use_holdings = use_holdings
load_prices
end
def get_trades(date: nil)
if date.blank?
trades
else
trades.select { |t| t.date == date }
end
end
def get_price(security_id, date)
security = @security_cache[security_id]
raise SecurityNotFound.new(security_id, account.id) unless security
price = security[:prices].select { |p| p.price.date == date }.min_by(&:priority)&.price
return nil unless price
price_money = Money.new(price.price, price.currency)
converted_amount = price_money.exchange_to(account.currency, fallback_rate: 1).amount
Security::Price.new(
security_id: security_id,
date: price.date,
price: converted_amount,
currency: price.currency
)
end
def get_securities
@security_cache.map { |_, v| v[:security] }
end
private
PriceWithPriority = Data.define(:price, :priority)
def trades
@trades ||= account.entries.includes(entryable: :security).account_trades.chronological.to_a
end
def holdings
@holdings ||= account.holdings.chronological.to_a
end
def collect_unique_securities
unique_securities_from_trades = trades.map(&:entryable).map(&:security).uniq
return unique_securities_from_trades unless use_holdings
unique_securities_from_holdings = holdings.map(&:security).uniq
(unique_securities_from_trades + unique_securities_from_holdings).uniq
end
# Loads all known prices for all securities in the account with priority based on source:
# 1 - DB or provider prices
# 2 - Trade prices
# 3 - Holding prices
def load_prices
@security_cache = {}
securities = collect_unique_securities
Rails.logger.info "Preloading #{securities.size} securities for account #{account.id}"
securities.each do |security|
Rails.logger.info "Loading security: ID=#{security.id} Ticker=#{security.ticker}"
# Highest priority prices
db_or_provider_prices = Security::Price.find_prices(
security: security,
start_date: account.start_date,
end_date: Date.current
).map do |price|
PriceWithPriority.new(
price: price,
priority: 1
)
end
# Medium priority prices from trades
trade_prices = trades
.select { |t| t.entryable.security_id == security.id }
.map do |trade|
PriceWithPriority.new(
price: Security::Price.new(
security: security,
price: trade.entryable.price,
currency: trade.entryable.currency,
date: trade.date
),
priority: 2
)
end
# Low priority prices from holdings (if applicable)
holding_prices = if use_holdings
holdings.select { |h| h.security_id == security.id }.map do |holding|
PriceWithPriority.new(
price: Security::Price.new(
security: security,
price: holding.price,
currency: holding.currency,
date: holding.date
),
priority: 3
)
end
else
[]
end
@security_cache[security.id] = {
security: security,
prices: db_or_provider_prices + trade_prices + holding_prices
}
end
end
end

38
app/models/account/holding/reverse_calculator.rb Normal file
View file

@ -0,0 +1,38 @@
class Account::Holding::ReverseCalculator < Account::Holding::BaseCalculator
private
# Reverse calculators will use the existing holdings as a source of security ids and prices
# since it is common for a provider to supply "current day" holdings but not all the historical
# trades that make up those holdings.
def portfolio_cache
@portfolio_cache ||= Account::Holding::PortfolioCache.new(account, use_holdings: true)
end
def calculate_holdings
current_portfolio = generate_starting_portfolio
previous_portfolio = {}
holdings = []
Date.current.downto(account.start_date).each do |date|
today_trades = portfolio_cache.get_trades(date: date)
previous_portfolio = transform_portfolio(current_portfolio, today_trades, direction: :reverse)
holdings += build_holdings(current_portfolio, date)
current_portfolio = previous_portfolio
end
holdings
end
# Since this is a reverse sync, we start with today's holdings
def generate_starting_portfolio
holding_quantities = empty_portfolio
todays_holdings = account.holdings.where(date: Date.current)
todays_holdings.each do |holding|
holding_quantities[holding.security_id] = holding.qty
end
holding_quantities
end
end

58
app/models/account/holding/syncer.rb Normal file
View file

@ -0,0 +1,58 @@
class Account::Holding::Syncer
def initialize(account, strategy:)
@account = account
@strategy = strategy
end
def sync_holdings
calculate_holdings
Rails.logger.info("Persisting #{@holdings.size} holdings")
persist_holdings
if strategy == :forward
purge_stale_holdings
end
@holdings
end
private
attr_reader :account, :strategy
def calculate_holdings
@holdings = calculator.calculate
end
def persist_holdings
current_time = Time.now
account.holdings.upsert_all(
@holdings.map { |h| h.attributes
.slice("date", "currency", "qty", "price", "amount", "security_id")
.merge("account_id" => account.id, "updated_at" => current_time) },
unique_by: %i[account_id security_id date currency]
)
end
def purge_stale_holdings
portfolio_security_ids = account.entries.account_trades.map { |entry| entry.entryable.security_id }.uniq
# If there are no securities in the portfolio, delete all holdings
if portfolio_security_ids.empty?
Rails.logger.info("Clearing all holdings (no securities)")
account.holdings.delete_all
else
deleted_count = account.holdings.delete_by("date < ? OR security_id NOT IN (?)", account.start_date, portfolio_security_ids)
Rails.logger.info("Purged #{deleted_count} stale holdings") if deleted_count > 0
end
end
def calculator
if strategy == :reverse
Account::Holding::ReverseCalculator.new(account)
else
Account::Holding::ForwardCalculator.new(account)
end
end
end

188
app/models/account/holding_calculator.rb
View file

@ -1,188 +0,0 @@
class Account::HoldingCalculator
def initialize(account)
@account = account
@securities_cache = {}
end
def calculate(reverse: false)
Rails.logger.tagged("Account::HoldingCalculator") do
preload_securities
Rails.logger.info("Calculating holdings with strategy: #{reverse ? "reverse sync" : "forward sync"}")
calculated_holdings = reverse ? reverse_holdings : forward_holdings
gapfill_holdings(calculated_holdings)
end
end
private
attr_reader :account, :securities_cache
def reverse_holdings
current_holding_quantities = load_current_holding_quantities
prior_holding_quantities = {}
holdings = []
Date.current.downto(portfolio_start_date).map do |date|
today_trades = trades.select { |t| t.date == date }
prior_holding_quantities = calculate_portfolio(current_holding_quantities, today_trades)
holdings += generate_holding_records(current_holding_quantities, date)
current_holding_quantities = prior_holding_quantities
end
holdings
end
def forward_holdings
prior_holding_quantities = load_empty_holding_quantities
current_holding_quantities = {}
holdings = []
portfolio_start_date.upto(Date.current).map do |date|
today_trades = trades.select { |t| t.date == date }
current_holding_quantities = calculate_portfolio(prior_holding_quantities, today_trades, inverse: true)
holdings += generate_holding_records(current_holding_quantities, date)
prior_holding_quantities = current_holding_quantities
end
holdings
end
def generate_holding_records(portfolio, date)
Rails.logger.info "[HoldingCalculator] Generating holdings for #{portfolio.size} securities on #{date}"
portfolio.map do |security_id, qty|
security = securities_cache[security_id]
if security.blank?
Rails.logger.error "[HoldingCalculator] Security #{security_id} not found in cache for account #{account.id}"
next
end
price = security.dig(:prices)&.find { |p| p.date == date }
if price.blank?
Rails.logger.info "[HoldingCalculator] No price found for security #{security_id} on #{date}"
next
end
converted_price = Money.new(price.price, price.currency).exchange_to(account.currency, fallback_rate: 1).amount
account.holdings.build(
security: security.dig(:security),
date: date,
qty: qty,
price: converted_price,
currency: account.currency,
amount: qty * converted_price
)
end.compact
end
def gapfill_holdings(holdings)
filled_holdings = []
holdings.group_by { |h| h.security_id }.each do |security_id, security_holdings|
next if security_holdings.empty?
sorted = security_holdings.sort_by(&:date)
previous_holding = sorted.first
sorted.first.date.upto(Date.current) do |date|
holding = security_holdings.find { |h| h.date == date }
if holding
filled_holdings << holding
previous_holding = holding
else
# Create a new holding based on the previous day's data
filled_holdings << account.holdings.build(
security: previous_holding.security,
date: date,
qty: previous_holding.qty,
price: previous_holding.price,
currency: previous_holding.currency,
amount: previous_holding.amount
)
end
end
end
filled_holdings
end
def trades
@trades ||= account.entries.includes(entryable: :security).account_trades.chronological.to_a
end
def portfolio_start_date
trades.first ? trades.first.date - 1.day : Date.current
end
def preload_securities
# Get securities from trades and current holdings
securities = trades.map(&:entryable).map(&:security).uniq
securities += account.holdings.where(date: Date.current).map(&:security)
securities.uniq!
Rails.logger.info "[HoldingCalculator] Preloading #{securities.size} securities for account #{account.id}"
securities.each do |security|
begin
Rails.logger.info "[HoldingCalculator] Loading security: ID=#{security.id} Ticker=#{security.ticker}"
prices = Security::Price.find_prices(
security: security,
start_date: portfolio_start_date,
end_date: Date.current
)
Rails.logger.info "[HoldingCalculator] Found #{prices.size} prices for security #{security.id}"
@securities_cache[security.id] = {
security: security,
prices: prices
}
rescue => e
Rails.logger.error "[HoldingCalculator] Error processing security #{security.id}: #{e.message}"
Rails.logger.error "[HoldingCalculator] Security details: #{security.attributes}"
Rails.logger.error e.backtrace.join("\n")
next # Skip this security and continue with others
end
end
end
def calculate_portfolio(holding_quantities, today_trades, inverse: false)
new_quantities = holding_quantities.dup
today_trades.each do |trade|
security_id = trade.entryable.security_id
qty_change = inverse ? trade.entryable.qty : -trade.entryable.qty
new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change
end
new_quantities
end
def load_empty_holding_quantities
holding_quantities = {}
trades.map { |t| t.entryable.security_id }.uniq.each do |security_id|
holding_quantities[security_id] = 0
end
holding_quantities
end
def load_current_holding_quantities
holding_quantities = load_empty_holding_quantities
account.holdings.where(date: Date.current, currency: account.currency).map do |holding|
holding_quantities[holding.security_id] = holding.qty
end
holding_quantities
end
end

18
app/models/account/linkable.rb Normal file
View file

@ -0,0 +1,18 @@
module Account::Linkable
extend ActiveSupport::Concern
included do
belongs_to :plaid_account, optional: true
end
# A "linked" account gets transaction and balance data from a third party like Plaid
def linked?
plaid_account_id.present?
end
# An "offline" or "unlinked" account is one where the user tracks values and
# adds transactions manually, without the help of a data provider
def unlinked?
!linked?
end
end

162
app/models/account/syncer.rb
View file

@ -1,162 +0,0 @@
class Account::Syncer
def initialize(account, start_date: nil)
@account = account
@start_date = start_date
end
def run
Rails.logger.tagged("Account::Syncer") do
Rails.logger.info("Finding potential transfers to auto-match")
account.family.auto_match_transfers!
holdings = sync_holdings
Rails.logger.info("Calculated #{holdings.size} holdings")
balances = sync_balances(holdings)
Rails.logger.info("Calculated #{balances.size} balances")
account.reload
unless plaid_sync?
update_account_info(balances, holdings)
end
unless account.currency == account.family.currency
Rails.logger.info("Converting #{balances.size} balances and #{holdings.size} holdings from #{account.currency} to #{account.family.currency}")
convert_records_to_family_currency(balances, holdings)
end
# Enrich if user opted in or if we're syncing transactions from a Plaid account on the hosted app
if account.family.data_enrichment_enabled? || (plaid_sync? && Rails.application.config.app_mode.hosted?)
Rails.logger.info("Enriching transaction data for account #{account.name}")
account.enrich_data
else
Rails.logger.info("Data enrichment disabled for account #{account.name}")
end
end
end
private
attr_reader :account, :start_date
def account_start_date
@account_start_date ||= (account.entries.chronological.first&.date || Date.current) - 1.day
end
def update_account_info(balances, holdings)
new_balance = balances.sort_by(&:date).last.balance
new_holdings_value = holdings.select { |h| h.date == Date.current }.sum(&:amount)
new_cash_balance = new_balance - new_holdings_value
account.update!(
balance: new_balance,
cash_balance: new_cash_balance
)
end
def sync_holdings
calculator = Account::HoldingCalculator.new(account)
calculated_holdings = calculator.calculate(reverse: plaid_sync?)
Account.transaction do
load_holdings(calculated_holdings)
purge_outdated_holdings unless plaid_sync?
end
calculated_holdings
end
def sync_balances(holdings)
calculator = Account::BalanceCalculator.new(account, holdings: holdings)
calculated_balances = calculator.calculate(reverse: plaid_sync?, start_date: start_date)
Account.transaction do
load_balances(calculated_balances)
purge_outdated_balances
end
calculated_balances
end
def convert_records_to_family_currency(balances, holdings)
from_currency = account.currency
to_currency = account.family.currency
exchange_rates = ExchangeRate.find_rates(
from: from_currency,
to: to_currency,
start_date: balances.min_by(&:date).date
)
converted_balances = balances.map do |balance|
exchange_rate = exchange_rates.find { |er| er.date == balance.date }
next unless exchange_rate.present?
account.balances.build(
date: balance.date,
balance: exchange_rate.rate * balance.balance,
currency: to_currency
)
end.compact
converted_holdings = holdings.map do |holding|
exchange_rate = exchange_rates.find { |er| er.date == holding.date }
next unless exchange_rate.present?
account.holdings.build(
security: holding.security,
date: holding.date,
qty: holding.qty,
price: exchange_rate.rate * holding.price,
amount: exchange_rate.rate * holding.amount,
currency: to_currency
)
end.compact
Account.transaction do
load_balances(converted_balances)
load_holdings(converted_holdings)
end
end
def load_balances(balances = [])
current_time = Time.now
account.balances.upsert_all(
balances.map { |b| b.attributes
.slice("date", "balance", "cash_balance", "currency")
.merge("updated_at" => current_time) },
unique_by: %i[account_id date currency]
)
end
def load_holdings(holdings = [])
current_time = Time.now
account.holdings.upsert_all(
holdings.map { |h| h.attributes
.slice("date", "currency", "qty", "price", "amount", "security_id")
.merge("updated_at" => current_time) },
unique_by: %i[account_id security_id date currency]
)
end
def purge_outdated_balances
account.balances.delete_by("date < ?", account_start_date)
end
def plaid_sync?
account.plaid_account_id.present?
end
def purge_outdated_holdings
portfolio_security_ids = account.entries.account_trades.map { |entry| entry.entryable.security_id }.uniq
# If there are no securities in the portfolio, delete all holdings
if portfolio_security_ids.empty?
account.holdings.delete_all
else
account.holdings.delete_by("date < ? OR security_id NOT IN (?)", account_start_date, portfolio_security_ids)
end
end
end

5
app/models/plaid_item.rb
View file

@ -45,6 +45,11 @@ class PlaidItem < ApplicationRecord
plaid_data = fetch_and_load_plaid_data
update!(status: :good) if requires_update?
# Schedule account syncs
accounts.each do |account|
account.sync_later(start_date: start_date)
end
Rails.logger.info("Plaid data fetched and loaded")
plaid_data
rescue Plaid::ApiError => e