mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-07-19 13:19:39 +02:00
Clarify backend data pipeline naming concepts (importers, processors, materializers, calculators, and syncers) (#2255)
* Rename MarketDataSyncer to MarketDataImporter * Materializers * Importers * More reference replacements
This commit is contained in:
parent
b8903d0980
commit
10f255a9a9
18 changed files with 165 additions and 163 deletions
|
@ -7,7 +7,7 @@
|
||||||
# Each individual account sync will still fetch any missing market data that isn't yet synced, but by running
|
# Each individual account sync will still fetch any missing market data that isn't yet synced, but by running
|
||||||
# this job daily, we significantly reduce overlapping account syncs that both need the same market data (e.g. common security like `AAPL`)
|
# this job daily, we significantly reduce overlapping account syncs that both need the same market data (e.g. common security like `AAPL`)
|
||||||
#
|
#
|
||||||
class SyncMarketDataJob < ApplicationJob
|
class ImportMarketDataJob < ApplicationJob
|
||||||
queue_as :scheduled
|
queue_as :scheduled
|
||||||
|
|
||||||
def perform(opts)
|
def perform(opts)
|
||||||
|
@ -15,6 +15,6 @@ class SyncMarketDataJob < ApplicationJob
|
||||||
mode = opts.fetch(:mode, :full)
|
mode = opts.fetch(:mode, :full)
|
||||||
clear_cache = opts.fetch(:clear_cache, false)
|
clear_cache = opts.fetch(:clear_cache, false)
|
||||||
|
|
||||||
MarketDataSyncer.new(mode: mode, clear_cache: clear_cache).sync
|
MarketDataImporter.new(mode: mode, clear_cache: clear_cache).import_all
|
||||||
end
|
end
|
||||||
end
|
end
|
82
app/models/account/market_data_importer.rb
Normal file
82
app/models/account/market_data_importer.rb
Normal file
|
@ -0,0 +1,82 @@
|
||||||
|
class Account::MarketDataImporter
|
||||||
|
attr_reader :account
|
||||||
|
|
||||||
|
def initialize(account)
|
||||||
|
@account = account
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_all
|
||||||
|
import_exchange_rates
|
||||||
|
import_security_prices
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_exchange_rates
|
||||||
|
return unless needs_exchange_rates?
|
||||||
|
return unless ExchangeRate.provider
|
||||||
|
|
||||||
|
pair_dates = {}
|
||||||
|
|
||||||
|
# 1. ENTRY-BASED PAIRS – currencies that differ from the account currency
|
||||||
|
account.entries
|
||||||
|
.where.not(currency: account.currency)
|
||||||
|
.group(:currency)
|
||||||
|
.minimum(:date)
|
||||||
|
.each do |source_currency, date|
|
||||||
|
key = [ source_currency, account.currency ]
|
||||||
|
pair_dates[key] = [ pair_dates[key], date ].compact.min
|
||||||
|
end
|
||||||
|
|
||||||
|
# 2. ACCOUNT-BASED PAIR – convert the account currency to the family currency (if different)
|
||||||
|
if foreign_account?
|
||||||
|
key = [ account.currency, account.family.currency ]
|
||||||
|
pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min
|
||||||
|
end
|
||||||
|
|
||||||
|
pair_dates.each do |(source, target), start_date|
|
||||||
|
ExchangeRate.import_provider_rates(
|
||||||
|
from: source,
|
||||||
|
to: target,
|
||||||
|
start_date: start_date,
|
||||||
|
end_date: Date.current
|
||||||
|
)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def import_security_prices
|
||||||
|
return unless Security.provider
|
||||||
|
|
||||||
|
account_securities = account.trades.map(&:security).uniq
|
||||||
|
|
||||||
|
return if account_securities.empty?
|
||||||
|
|
||||||
|
account_securities.each do |security|
|
||||||
|
security.import_provider_prices(
|
||||||
|
start_date: first_required_price_date(security),
|
||||||
|
end_date: Date.current
|
||||||
|
)
|
||||||
|
|
||||||
|
security.import_provider_details
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
# Calculates the first date we require a price for the given security scoped to this account
|
||||||
|
def first_required_price_date(security)
|
||||||
|
account.trades.with_entry
|
||||||
|
.where(security: security)
|
||||||
|
.where(entries: { account_id: account.id })
|
||||||
|
.minimum("entries.date")
|
||||||
|
end
|
||||||
|
|
||||||
|
def needs_exchange_rates?
|
||||||
|
has_multi_currency_entries? || foreign_account?
|
||||||
|
end
|
||||||
|
|
||||||
|
def has_multi_currency_entries?
|
||||||
|
account.entries.where.not(currency: account.currency).exists?
|
||||||
|
end
|
||||||
|
|
||||||
|
def foreign_account?
|
||||||
|
account.currency != account.family.currency
|
||||||
|
end
|
||||||
|
end
|
|
@ -1,82 +0,0 @@
|
||||||
class Account::MarketDataSyncer
|
|
||||||
attr_reader :account
|
|
||||||
|
|
||||||
def initialize(account)
|
|
||||||
@account = account
|
|
||||||
end
|
|
||||||
|
|
||||||
def sync_market_data
|
|
||||||
sync_exchange_rates
|
|
||||||
sync_security_prices
|
|
||||||
end
|
|
||||||
|
|
||||||
private
|
|
||||||
def sync_exchange_rates
|
|
||||||
return unless needs_exchange_rates?
|
|
||||||
return unless ExchangeRate.provider
|
|
||||||
|
|
||||||
pair_dates = {}
|
|
||||||
|
|
||||||
# 1. ENTRY-BASED PAIRS – currencies that differ from the account currency
|
|
||||||
account.entries
|
|
||||||
.where.not(currency: account.currency)
|
|
||||||
.group(:currency)
|
|
||||||
.minimum(:date)
|
|
||||||
.each do |source_currency, date|
|
|
||||||
key = [ source_currency, account.currency ]
|
|
||||||
pair_dates[key] = [ pair_dates[key], date ].compact.min
|
|
||||||
end
|
|
||||||
|
|
||||||
# 2. ACCOUNT-BASED PAIR – convert the account currency to the family currency (if different)
|
|
||||||
if foreign_account?
|
|
||||||
key = [ account.currency, account.family.currency ]
|
|
||||||
pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min
|
|
||||||
end
|
|
||||||
|
|
||||||
pair_dates.each do |(source, target), start_date|
|
|
||||||
ExchangeRate.sync_provider_rates(
|
|
||||||
from: source,
|
|
||||||
to: target,
|
|
||||||
start_date: start_date,
|
|
||||||
end_date: Date.current
|
|
||||||
)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def sync_security_prices
|
|
||||||
return unless Security.provider
|
|
||||||
|
|
||||||
account_securities = account.trades.map(&:security).uniq
|
|
||||||
|
|
||||||
return if account_securities.empty?
|
|
||||||
|
|
||||||
account_securities.each do |security|
|
|
||||||
security.sync_provider_prices(
|
|
||||||
start_date: first_required_price_date(security),
|
|
||||||
end_date: Date.current
|
|
||||||
)
|
|
||||||
|
|
||||||
security.sync_provider_details
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# Calculates the first date we require a price for the given security scoped to this account
|
|
||||||
def first_required_price_date(security)
|
|
||||||
account.trades.with_entry
|
|
||||||
.where(security: security)
|
|
||||||
.where(entries: { account_id: account.id })
|
|
||||||
.minimum("entries.date")
|
|
||||||
end
|
|
||||||
|
|
||||||
def needs_exchange_rates?
|
|
||||||
has_multi_currency_entries? || foreign_account?
|
|
||||||
end
|
|
||||||
|
|
||||||
def has_multi_currency_entries?
|
|
||||||
account.entries.where.not(currency: account.currency).exists?
|
|
||||||
end
|
|
||||||
|
|
||||||
def foreign_account?
|
|
||||||
account.currency != account.family.currency
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -7,8 +7,8 @@ class Account::Syncer
|
||||||
|
|
||||||
def perform_sync(sync)
|
def perform_sync(sync)
|
||||||
Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})")
|
Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})")
|
||||||
sync_market_data
|
import_market_data
|
||||||
sync_balances
|
materialize_balances
|
||||||
end
|
end
|
||||||
|
|
||||||
def perform_post_sync
|
def perform_post_sync
|
||||||
|
@ -16,9 +16,9 @@ class Account::Syncer
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
def sync_balances
|
def materialize_balances
|
||||||
strategy = account.linked? ? :reverse : :forward
|
strategy = account.linked? ? :reverse : :forward
|
||||||
Balance::Syncer.new(account, strategy: strategy).sync_balances
|
Balance::Materializer.new(account, strategy: strategy).materialize_balances
|
||||||
end
|
end
|
||||||
|
|
||||||
# Syncs all the exchange rates + security prices this account needs to display historical chart data
|
# Syncs all the exchange rates + security prices this account needs to display historical chart data
|
||||||
|
@ -28,8 +28,8 @@ class Account::Syncer
|
||||||
#
|
#
|
||||||
# We rescue errors here because if this operation fails, we don't want to fail the entire sync since
|
# We rescue errors here because if this operation fails, we don't want to fail the entire sync since
|
||||||
# we have reasonable fallbacks for missing market data.
|
# we have reasonable fallbacks for missing market data.
|
||||||
def sync_market_data
|
def import_market_data
|
||||||
Account::MarketDataSyncer.new(account).sync_market_data
|
Account::MarketDataImporter.new(account).import_all
|
||||||
rescue => e
|
rescue => e
|
||||||
Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}")
|
Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}")
|
||||||
Sentry.capture_exception(e)
|
Sentry.capture_exception(e)
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
class Balance::Syncer
|
class Balance::Materializer
|
||||||
attr_reader :account, :strategy
|
attr_reader :account, :strategy
|
||||||
|
|
||||||
def initialize(account, strategy:)
|
def initialize(account, strategy:)
|
||||||
|
@ -6,9 +6,9 @@ class Balance::Syncer
|
||||||
@strategy = strategy
|
@strategy = strategy
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_balances
|
def materialize_balances
|
||||||
Balance.transaction do
|
Balance.transaction do
|
||||||
sync_holdings
|
materialize_holdings
|
||||||
calculate_balances
|
calculate_balances
|
||||||
|
|
||||||
Rails.logger.info("Persisting #{@balances.size} balances")
|
Rails.logger.info("Persisting #{@balances.size} balances")
|
||||||
|
@ -23,8 +23,8 @@ class Balance::Syncer
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
def sync_holdings
|
def materialize_holdings
|
||||||
@holdings = Holding::Syncer.new(account, strategy: strategy).sync_holdings
|
@holdings = Holding::Materializer.new(account, strategy: strategy).materialize_holdings
|
||||||
end
|
end
|
||||||
|
|
||||||
def update_account_info
|
def update_account_info
|
|
@ -1,4 +1,4 @@
|
||||||
class ExchangeRate::Syncer
|
class ExchangeRate::Importer
|
||||||
MissingExchangeRateError = Class.new(StandardError)
|
MissingExchangeRateError = Class.new(StandardError)
|
||||||
MissingStartRateError = Class.new(StandardError)
|
MissingStartRateError = Class.new(StandardError)
|
||||||
|
|
||||||
|
@ -12,7 +12,7 @@ class ExchangeRate::Syncer
|
||||||
end
|
end
|
||||||
|
|
||||||
# Constructs a daily series of rates for the given currency pair for date range
|
# Constructs a daily series of rates for the given currency pair for date range
|
||||||
def sync_provider_rates
|
def import_provider_rates
|
||||||
if !clear_cache && all_rates_exist?
|
if !clear_cache && all_rates_exist?
|
||||||
Rails.logger.info("No new rates to sync for #{from} to #{to} between #{start_date} and #{end_date}, skipping")
|
Rails.logger.info("No new rates to sync for #{from} to #{to} between #{start_date} and #{end_date}, skipping")
|
||||||
return
|
return
|
|
@ -28,20 +28,20 @@ module ExchangeRate::Provided
|
||||||
end
|
end
|
||||||
|
|
||||||
# @return [Integer] The number of exchange rates synced
|
# @return [Integer] The number of exchange rates synced
|
||||||
def sync_provider_rates(from:, to:, start_date:, end_date:, clear_cache: false)
|
def import_provider_rates(from:, to:, start_date:, end_date:, clear_cache: false)
|
||||||
unless provider.present?
|
unless provider.present?
|
||||||
Rails.logger.warn("No provider configured for ExchangeRate.sync_provider_rates")
|
Rails.logger.warn("No provider configured for ExchangeRate.import_provider_rates")
|
||||||
return 0
|
return 0
|
||||||
end
|
end
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: provider,
|
exchange_rate_provider: provider,
|
||||||
from: from,
|
from: from,
|
||||||
to: to,
|
to: to,
|
||||||
start_date: start_date,
|
start_date: start_date,
|
||||||
end_date: end_date,
|
end_date: end_date,
|
||||||
clear_cache: clear_cache
|
clear_cache: clear_cache
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
class Holding::Syncer
|
# "Materializes" holdings (similar to a DB materialized view, but done at the app level)
|
||||||
|
# into a series of records we can easily query and join with other data.
|
||||||
|
class Holding::Materializer
|
||||||
def initialize(account, strategy:)
|
def initialize(account, strategy:)
|
||||||
@account = account
|
@account = account
|
||||||
@strategy = strategy
|
@strategy = strategy
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_holdings
|
def materialize_holdings
|
||||||
calculate_holdings
|
calculate_holdings
|
||||||
|
|
||||||
Rails.logger.info("Persisting #{@holdings.size} holdings")
|
Rails.logger.info("Persisting #{@holdings.size} holdings")
|
|
@ -1,4 +1,4 @@
|
||||||
class MarketDataSyncer
|
class MarketDataImporter
|
||||||
# By default, our graphs show 1M as the view, so by fetching 31 days,
|
# By default, our graphs show 1M as the view, so by fetching 31 days,
|
||||||
# we ensure we can always show an accurate default graph
|
# we ensure we can always show an accurate default graph
|
||||||
SNAPSHOT_DAYS = 31
|
SNAPSHOT_DAYS = 31
|
||||||
|
@ -10,32 +10,32 @@ class MarketDataSyncer
|
||||||
@clear_cache = clear_cache
|
@clear_cache = clear_cache
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync
|
def import_all
|
||||||
sync_prices
|
import_security_prices
|
||||||
sync_exchange_rates
|
import_exchange_rates
|
||||||
end
|
end
|
||||||
|
|
||||||
# Syncs historical security prices (and details)
|
# Syncs historical security prices (and details)
|
||||||
def sync_prices
|
def import_security_prices
|
||||||
unless Security.provider
|
unless Security.provider
|
||||||
Rails.logger.warn("No provider configured for MarketDataSyncer.sync_prices, skipping sync")
|
Rails.logger.warn("No provider configured for MarketDataImporter.import_security_prices, skipping sync")
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
Security.where.not(exchange_operating_mic: nil).find_each do |security|
|
Security.where.not(exchange_operating_mic: nil).find_each do |security|
|
||||||
security.sync_provider_prices(
|
security.import_provider_prices(
|
||||||
start_date: get_first_required_price_date(security),
|
start_date: get_first_required_price_date(security),
|
||||||
end_date: end_date,
|
end_date: end_date,
|
||||||
clear_cache: clear_cache
|
clear_cache: clear_cache
|
||||||
)
|
)
|
||||||
|
|
||||||
security.sync_provider_details(clear_cache: clear_cache)
|
security.import_provider_details(clear_cache: clear_cache)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_exchange_rates
|
def import_exchange_rates
|
||||||
unless ExchangeRate.provider
|
unless ExchangeRate.provider
|
||||||
Rails.logger.warn("No provider configured for MarketDataSyncer.sync_exchange_rates, skipping sync")
|
Rails.logger.warn("No provider configured for MarketDataImporter.import_exchange_rates, skipping sync")
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -43,7 +43,7 @@ class MarketDataSyncer
|
||||||
# pair is a Hash with keys :source, :target, and :start_date
|
# pair is a Hash with keys :source, :target, and :start_date
|
||||||
start_date = snapshot? ? default_start_date : pair[:start_date]
|
start_date = snapshot? ? default_start_date : pair[:start_date]
|
||||||
|
|
||||||
ExchangeRate.sync_provider_rates(
|
ExchangeRate.import_provider_rates(
|
||||||
from: pair[:source],
|
from: pair[:source],
|
||||||
to: pair[:target],
|
to: pair[:target],
|
||||||
start_date: start_date,
|
start_date: start_date,
|
||||||
|
@ -124,7 +124,7 @@ class MarketDataSyncer
|
||||||
valid_modes = [ :full, :snapshot ]
|
valid_modes = [ :full, :snapshot ]
|
||||||
|
|
||||||
unless valid_modes.include?(mode.to_sym)
|
unless valid_modes.include?(mode.to_sym)
|
||||||
raise InvalidModeError, "Invalid mode for MarketDataSyncer, can only be :full or :snapshot, but was #{mode}"
|
raise InvalidModeError, "Invalid mode for MarketDataImporter, can only be :full or :snapshot, but was #{mode}"
|
||||||
end
|
end
|
||||||
|
|
||||||
mode.to_sym
|
mode.to_sym
|
|
@ -1,4 +1,4 @@
|
||||||
class Security::Price::Syncer
|
class Security::Price::Importer
|
||||||
MissingSecurityPriceError = Class.new(StandardError)
|
MissingSecurityPriceError = Class.new(StandardError)
|
||||||
MissingStartPriceError = Class.new(StandardError)
|
MissingStartPriceError = Class.new(StandardError)
|
||||||
|
|
||||||
|
@ -12,7 +12,7 @@ class Security::Price::Syncer
|
||||||
|
|
||||||
# Constructs a daily series of prices for a single security over the date range.
|
# Constructs a daily series of prices for a single security over the date range.
|
||||||
# Returns the number of rows upserted.
|
# Returns the number of rows upserted.
|
||||||
def sync_provider_prices
|
def import_provider_prices
|
||||||
if !clear_cache && all_prices_exist?
|
if !clear_cache && all_prices_exist?
|
||||||
Rails.logger.info("No new prices to sync for #{security.ticker} between #{start_date} and #{end_date}, skipping")
|
Rails.logger.info("No new prices to sync for #{security.ticker} between #{start_date} and #{end_date}, skipping")
|
||||||
return 0
|
return 0
|
|
@ -49,9 +49,9 @@ module Security::Provided
|
||||||
price
|
price
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_provider_details(clear_cache: false)
|
def import_provider_details(clear_cache: false)
|
||||||
unless provider.present?
|
unless provider.present?
|
||||||
Rails.logger.warn("No provider configured for Security.sync_provider_details")
|
Rails.logger.warn("No provider configured for Security.import_provider_details")
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -76,19 +76,19 @@ module Security::Provided
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def sync_provider_prices(start_date:, end_date:, clear_cache: false)
|
def import_provider_prices(start_date:, end_date:, clear_cache: false)
|
||||||
unless provider.present?
|
unless provider.present?
|
||||||
Rails.logger.warn("No provider configured for Security.sync_provider_prices")
|
Rails.logger.warn("No provider configured for Security.import_provider_prices")
|
||||||
return 0
|
return 0
|
||||||
end
|
end
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: self,
|
security: self,
|
||||||
security_provider: provider,
|
security_provider: provider,
|
||||||
start_date: start_date,
|
start_date: start_date,
|
||||||
end_date: end_date,
|
end_date: end_date,
|
||||||
clear_cache: clear_cache
|
clear_cache: clear_cache
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
|
@ -1,8 +1,8 @@
|
||||||
sync_market_data:
|
import_market_data:
|
||||||
cron: "0 22 * * 1-5" # 5:00 PM EST / 6:00 PM EDT (NY time)
|
cron: "0 22 * * 1-5" # 5:00 PM EST / 6:00 PM EDT (NY time)
|
||||||
class: "SyncMarketDataJob"
|
class: "ImportMarketDataJob"
|
||||||
queue: "scheduled"
|
queue: "scheduled"
|
||||||
description: "Syncs market data daily at 5:00 PM EST (1 hour after market close)"
|
description: "Imports market data daily at 5:00 PM EST (1 hour after market close)"
|
||||||
args:
|
args:
|
||||||
mode: "full"
|
mode: "full"
|
||||||
clear_cache: false
|
clear_cache: false
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
require "ostruct"
|
require "ostruct"
|
||||||
|
|
||||||
class Account::MarketDataSyncerTest < ActiveSupport::TestCase
|
class Account::MarketDataImporterTest < ActiveSupport::TestCase
|
||||||
include ProviderTestHelper
|
include ProviderTestHelper
|
||||||
|
|
||||||
PROVIDER_BUFFER = 5.days
|
PROVIDER_BUFFER = 5.days
|
||||||
|
@ -32,7 +32,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
|
||||||
accountable: Depository.new
|
accountable: Depository.new
|
||||||
)
|
)
|
||||||
|
|
||||||
# Seed a rate for the first required day so that the syncer only needs the next day forward
|
# Seed a rate for the first required day so that the importer only needs the next day forward
|
||||||
existing_date = account.start_date
|
existing_date = account.start_date
|
||||||
ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: existing_date, rate: 2.0)
|
ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: existing_date, rate: 2.0)
|
||||||
|
|
||||||
|
@ -49,7 +49,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
|
||||||
]))
|
]))
|
||||||
|
|
||||||
before = ExchangeRate.count
|
before = ExchangeRate.count
|
||||||
Account::MarketDataSyncer.new(account).sync_market_data
|
Account::MarketDataImporter.new(account).import_all
|
||||||
after = ExchangeRate.count
|
after = ExchangeRate.count
|
||||||
|
|
||||||
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
|
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
|
||||||
|
@ -100,7 +100,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
|
||||||
# Ignore exchange-rate calls for this test
|
# Ignore exchange-rate calls for this test
|
||||||
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
|
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
|
||||||
|
|
||||||
Account::MarketDataSyncer.new(account).sync_market_data
|
Account::MarketDataImporter.new(account).import_all
|
||||||
|
|
||||||
assert_equal 1, Security::Price.where(security: security, date: trade_date).count
|
assert_equal 1, Security::Price.where(security: security, date: trade_date).count
|
||||||
end
|
end
|
|
@ -1,6 +1,6 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
|
|
||||||
class Balance::SyncerTest < ActiveSupport::TestCase
|
class Balance::MaterializerTest < ActiveSupport::TestCase
|
||||||
include EntriesTestHelper
|
include EntriesTestHelper
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
|
@ -14,7 +14,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
|
||||||
end
|
end
|
||||||
|
|
||||||
test "syncs balances" do
|
test "syncs balances" do
|
||||||
Holding::Syncer.any_instance.expects(:sync_holdings).returns([]).once
|
Holding::Materializer.any_instance.expects(:materialize_holdings).returns([]).once
|
||||||
|
|
||||||
@account.expects(:start_date).returns(2.days.ago.to_date)
|
@account.expects(:start_date).returns(2.days.ago.to_date)
|
||||||
|
|
||||||
|
@ -26,7 +26,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
|
||||||
)
|
)
|
||||||
|
|
||||||
assert_difference "@account.balances.count", 2 do
|
assert_difference "@account.balances.count", 2 do
|
||||||
Balance::Syncer.new(@account, strategy: :forward).sync_balances
|
Balance::Materializer.new(@account, strategy: :forward).materialize_balances
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
|
||||||
)
|
)
|
||||||
|
|
||||||
assert_difference "@account.balances.count", 3 do
|
assert_difference "@account.balances.count", 3 do
|
||||||
Balance::Syncer.new(@account, strategy: :forward).sync_balances
|
Balance::Materializer.new(@account, strategy: :forward).materialize_balances
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -1,7 +1,7 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
require "ostruct"
|
require "ostruct"
|
||||||
|
|
||||||
class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
class ExchangeRate::ImporterTest < ActiveSupport::TestCase
|
||||||
include ProviderTestHelper
|
include ProviderTestHelper
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
|
@ -21,13 +21,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
||||||
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: @provider,
|
exchange_rate_provider: @provider,
|
||||||
from: "USD",
|
from: "USD",
|
||||||
to: "EUR",
|
to: "EUR",
|
||||||
start_date: 2.days.ago.to_date,
|
start_date: 2.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
|
|
||||||
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date..Date.current)
|
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date..Date.current)
|
||||||
.order(:date)
|
.order(:date)
|
||||||
|
@ -53,13 +53,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
||||||
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
|
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: @provider,
|
exchange_rate_provider: @provider,
|
||||||
from: "USD",
|
from: "USD",
|
||||||
to: "EUR",
|
to: "EUR",
|
||||||
start_date: 3.days.ago.to_date,
|
start_date: 3.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
|
|
||||||
db_rates = ExchangeRate.order(:date)
|
db_rates = ExchangeRate.order(:date)
|
||||||
assert_equal 4, db_rates.count
|
assert_equal 4, db_rates.count
|
||||||
|
@ -75,13 +75,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
||||||
|
|
||||||
@provider.expects(:fetch_exchange_rates).never
|
@provider.expects(:fetch_exchange_rates).never
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: @provider,
|
exchange_rate_provider: @provider,
|
||||||
from: "USD",
|
from: "USD",
|
||||||
to: "EUR",
|
to: "EUR",
|
||||||
start_date: 3.days.ago.to_date,
|
start_date: 3.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
end
|
end
|
||||||
|
|
||||||
# A helpful "reset" option for when we need to refresh provider data
|
# A helpful "reset" option for when we need to refresh provider data
|
||||||
|
@ -103,14 +103,14 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
||||||
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: @provider,
|
exchange_rate_provider: @provider,
|
||||||
from: "USD",
|
from: "USD",
|
||||||
to: "EUR",
|
to: "EUR",
|
||||||
start_date: 2.days.ago.to_date,
|
start_date: 2.days.ago.to_date,
|
||||||
end_date: Date.current,
|
end_date: Date.current,
|
||||||
clear_cache: true
|
clear_cache: true
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
|
|
||||||
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date)
|
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date)
|
||||||
assert_equal [ 1.3, 1.4, 1.5 ], db_rates.map(&:rate)
|
assert_equal [ 1.3, 1.4, 1.5 ], db_rates.map(&:rate)
|
||||||
|
@ -129,13 +129,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
|
||||||
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
|
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
ExchangeRate::Syncer.new(
|
ExchangeRate::Importer.new(
|
||||||
exchange_rate_provider: @provider,
|
exchange_rate_provider: @provider,
|
||||||
from: "USD",
|
from: "USD",
|
||||||
to: "EUR",
|
to: "EUR",
|
||||||
start_date: Date.current,
|
start_date: Date.current,
|
||||||
end_date: future_date
|
end_date: future_date
|
||||||
).sync_provider_rates
|
).import_provider_rates
|
||||||
|
|
||||||
assert_equal 1, ExchangeRate.count
|
assert_equal 1, ExchangeRate.count
|
||||||
end
|
end
|
|
@ -1,6 +1,6 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
|
|
||||||
class Holding::SyncerTest < ActiveSupport::TestCase
|
class Holding::MaterializerTest < ActiveSupport::TestCase
|
||||||
include EntriesTestHelper
|
include EntriesTestHelper
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
|
@ -14,7 +14,7 @@ class Holding::SyncerTest < ActiveSupport::TestCase
|
||||||
|
|
||||||
# Should have yesterday's and today's holdings
|
# Should have yesterday's and today's holdings
|
||||||
assert_difference "@account.holdings.count", 2 do
|
assert_difference "@account.holdings.count", 2 do
|
||||||
Holding::Syncer.new(@account, strategy: :forward).sync_holdings
|
Holding::Materializer.new(@account, strategy: :forward).materialize_holdings
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ class Holding::SyncerTest < ActiveSupport::TestCase
|
||||||
Holding.create!(account: @account, security: @aapl, qty: 1, price: 100, amount: 100, currency: "USD", date: Date.current)
|
Holding.create!(account: @account, security: @aapl, qty: 1, price: 100, amount: 100, currency: "USD", date: Date.current)
|
||||||
|
|
||||||
assert_difference "Holding.count", -1 do
|
assert_difference "Holding.count", -1 do
|
||||||
Holding::Syncer.new(@account, strategy: :forward).sync_holdings
|
Holding::Materializer.new(@account, strategy: :forward).materialize_holdings
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -1,10 +1,10 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
require "ostruct"
|
require "ostruct"
|
||||||
|
|
||||||
class MarketDataSyncerTest < ActiveSupport::TestCase
|
class MarketDataImporterTest < ActiveSupport::TestCase
|
||||||
include ProviderTestHelper
|
include ProviderTestHelper
|
||||||
|
|
||||||
SNAPSHOT_START_DATE = MarketDataSyncer::SNAPSHOT_DAYS.days.ago.to_date
|
SNAPSHOT_START_DATE = MarketDataImporter::SNAPSHOT_DAYS.days.ago.to_date
|
||||||
PROVIDER_BUFFER = 5.days
|
PROVIDER_BUFFER = 5.days
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
|
@ -47,7 +47,7 @@ class MarketDataSyncerTest < ActiveSupport::TestCase
|
||||||
]))
|
]))
|
||||||
|
|
||||||
before = ExchangeRate.count
|
before = ExchangeRate.count
|
||||||
MarketDataSyncer.new(mode: :snapshot).sync_exchange_rates
|
MarketDataImporter.new(mode: :snapshot).import_exchange_rates
|
||||||
after = ExchangeRate.count
|
after = ExchangeRate.count
|
||||||
|
|
||||||
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
|
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
|
||||||
|
@ -78,7 +78,7 @@ class MarketDataSyncerTest < ActiveSupport::TestCase
|
||||||
# Ignore exchange rate calls for this test
|
# Ignore exchange rate calls for this test
|
||||||
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
|
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
|
||||||
|
|
||||||
MarketDataSyncer.new(mode: :snapshot).sync_prices
|
MarketDataImporter.new(mode: :snapshot).import_security_prices
|
||||||
|
|
||||||
assert_equal 1, Security::Price.where(security: security, date: SNAPSHOT_START_DATE).count
|
assert_equal 1, Security::Price.where(security: security, date: SNAPSHOT_START_DATE).count
|
||||||
end
|
end
|
|
@ -1,7 +1,7 @@
|
||||||
require "test_helper"
|
require "test_helper"
|
||||||
require "ostruct"
|
require "ostruct"
|
||||||
|
|
||||||
class Security::Price::SyncerTest < ActiveSupport::TestCase
|
class Security::Price::ImporterTest < ActiveSupport::TestCase
|
||||||
include ProviderTestHelper
|
include ProviderTestHelper
|
||||||
|
|
||||||
setup do
|
setup do
|
||||||
|
@ -23,12 +23,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
|
||||||
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: @security,
|
security: @security,
|
||||||
security_provider: @provider,
|
security_provider: @provider,
|
||||||
start_date: 2.days.ago.to_date,
|
start_date: 2.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
|
|
||||||
db_prices = Security::Price.where(security: @security, date: 2.days.ago.to_date..Date.current).order(:date)
|
db_prices = Security::Price.where(security: @security, date: 2.days.ago.to_date..Date.current).order(:date)
|
||||||
|
|
||||||
|
@ -52,12 +52,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
|
||||||
start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
|
start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: @security,
|
security: @security,
|
||||||
security_provider: @provider,
|
security_provider: @provider,
|
||||||
start_date: 3.days.ago.to_date,
|
start_date: 3.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
|
|
||||||
db_prices = Security::Price.where(security: @security).order(:date)
|
db_prices = Security::Price.where(security: @security).order(:date)
|
||||||
assert_equal 4, db_prices.count
|
assert_equal 4, db_prices.count
|
||||||
|
@ -73,12 +73,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
|
||||||
|
|
||||||
@provider.expects(:fetch_security_prices).never
|
@provider.expects(:fetch_security_prices).never
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: @security,
|
security: @security,
|
||||||
security_provider: @provider,
|
security_provider: @provider,
|
||||||
start_date: 3.days.ago.to_date,
|
start_date: 3.days.ago.to_date,
|
||||||
end_date: Date.current
|
end_date: Date.current
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
end
|
end
|
||||||
|
|
||||||
test "full upsert if clear_cache is true" do
|
test "full upsert if clear_cache is true" do
|
||||||
|
@ -100,13 +100,13 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
|
||||||
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: @security,
|
security: @security,
|
||||||
security_provider: @provider,
|
security_provider: @provider,
|
||||||
start_date: 2.days.ago.to_date,
|
start_date: 2.days.ago.to_date,
|
||||||
end_date: Date.current,
|
end_date: Date.current,
|
||||||
clear_cache: true
|
clear_cache: true
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
|
|
||||||
db_prices = Security::Price.where(security: @security).order(:date)
|
db_prices = Security::Price.where(security: @security).order(:date)
|
||||||
assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
|
assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
|
||||||
|
@ -126,12 +126,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
|
||||||
start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
|
start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
|
||||||
.returns(provider_response)
|
.returns(provider_response)
|
||||||
|
|
||||||
Security::Price::Syncer.new(
|
Security::Price::Importer.new(
|
||||||
security: @security,
|
security: @security,
|
||||||
security_provider: @provider,
|
security_provider: @provider,
|
||||||
start_date: Date.current,
|
start_date: Date.current,
|
||||||
end_date: future_date
|
end_date: future_date
|
||||||
).sync_provider_prices
|
).import_provider_prices
|
||||||
|
|
||||||
assert_equal 1, Security::Price.count
|
assert_equal 1, Security::Price.count
|
||||||
end
|
end
|
Loading…
Add table
Add a link
Reference in a new issue