1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-05 05:25:24 +02:00

Clarify backend data pipeline naming concepts (importers, processors, materializers, calculators, and syncers) (#2255)
Some checks are pending
Publish Docker image / ci (push) Waiting to run
Publish Docker image / Build docker image (push) Blocked by required conditions

* Rename MarketDataSyncer to MarketDataImporter

* Materializers

* Importers

* More reference replacements
This commit is contained in:
Zach Gollwitzer 2025-05-17 16:37:16 -04:00 committed by GitHub
parent b8903d0980
commit 10f255a9a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 165 additions and 163 deletions

View file

@ -0,0 +1,82 @@
class Account::MarketDataImporter
attr_reader :account
def initialize(account)
@account = account
end
def import_all
import_exchange_rates
import_security_prices
end
def import_exchange_rates
return unless needs_exchange_rates?
return unless ExchangeRate.provider
pair_dates = {}
# 1. ENTRY-BASED PAIRS currencies that differ from the account currency
account.entries
.where.not(currency: account.currency)
.group(:currency)
.minimum(:date)
.each do |source_currency, date|
key = [ source_currency, account.currency ]
pair_dates[key] = [ pair_dates[key], date ].compact.min
end
# 2. ACCOUNT-BASED PAIR convert the account currency to the family currency (if different)
if foreign_account?
key = [ account.currency, account.family.currency ]
pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min
end
pair_dates.each do |(source, target), start_date|
ExchangeRate.import_provider_rates(
from: source,
to: target,
start_date: start_date,
end_date: Date.current
)
end
end
def import_security_prices
return unless Security.provider
account_securities = account.trades.map(&:security).uniq
return if account_securities.empty?
account_securities.each do |security|
security.import_provider_prices(
start_date: first_required_price_date(security),
end_date: Date.current
)
security.import_provider_details
end
end
private
# Calculates the first date we require a price for the given security scoped to this account
def first_required_price_date(security)
account.trades.with_entry
.where(security: security)
.where(entries: { account_id: account.id })
.minimum("entries.date")
end
def needs_exchange_rates?
has_multi_currency_entries? || foreign_account?
end
def has_multi_currency_entries?
account.entries.where.not(currency: account.currency).exists?
end
def foreign_account?
account.currency != account.family.currency
end
end

View file

@ -1,82 +0,0 @@
class Account::MarketDataSyncer
attr_reader :account
def initialize(account)
@account = account
end
def sync_market_data
sync_exchange_rates
sync_security_prices
end
private
def sync_exchange_rates
return unless needs_exchange_rates?
return unless ExchangeRate.provider
pair_dates = {}
# 1. ENTRY-BASED PAIRS currencies that differ from the account currency
account.entries
.where.not(currency: account.currency)
.group(:currency)
.minimum(:date)
.each do |source_currency, date|
key = [ source_currency, account.currency ]
pair_dates[key] = [ pair_dates[key], date ].compact.min
end
# 2. ACCOUNT-BASED PAIR convert the account currency to the family currency (if different)
if foreign_account?
key = [ account.currency, account.family.currency ]
pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min
end
pair_dates.each do |(source, target), start_date|
ExchangeRate.sync_provider_rates(
from: source,
to: target,
start_date: start_date,
end_date: Date.current
)
end
end
def sync_security_prices
return unless Security.provider
account_securities = account.trades.map(&:security).uniq
return if account_securities.empty?
account_securities.each do |security|
security.sync_provider_prices(
start_date: first_required_price_date(security),
end_date: Date.current
)
security.sync_provider_details
end
end
# Calculates the first date we require a price for the given security scoped to this account
def first_required_price_date(security)
account.trades.with_entry
.where(security: security)
.where(entries: { account_id: account.id })
.minimum("entries.date")
end
def needs_exchange_rates?
has_multi_currency_entries? || foreign_account?
end
def has_multi_currency_entries?
account.entries.where.not(currency: account.currency).exists?
end
def foreign_account?
account.currency != account.family.currency
end
end

View file

@ -7,8 +7,8 @@ class Account::Syncer
def perform_sync(sync)
Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})")
sync_market_data
sync_balances
import_market_data
materialize_balances
end
def perform_post_sync
@ -16,9 +16,9 @@ class Account::Syncer
end
private
def sync_balances
def materialize_balances
strategy = account.linked? ? :reverse : :forward
Balance::Syncer.new(account, strategy: strategy).sync_balances
Balance::Materializer.new(account, strategy: strategy).materialize_balances
end
# Syncs all the exchange rates + security prices this account needs to display historical chart data
@ -28,8 +28,8 @@ class Account::Syncer
#
# We rescue errors here because if this operation fails, we don't want to fail the entire sync since
# we have reasonable fallbacks for missing market data.
def sync_market_data
Account::MarketDataSyncer.new(account).sync_market_data
def import_market_data
Account::MarketDataImporter.new(account).import_all
rescue => e
Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}")
Sentry.capture_exception(e)