1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-04 21:15:19 +02:00

Market data sync refinements (#2252)

* Exchange rate syncer implementation

* Security price syncer

* Fix issues with provider API

* Add back prod schedule

* Add back price and exchange rate syncs to account syncs

* Remove unused stock_exchanges table
This commit is contained in:
Zach Gollwitzer 2025-05-16 14:17:56 -04:00 committed by GitHub
parent 6917cecf33
commit 6dc1d22672
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 1206 additions and 1615 deletions

View file

@ -1,52 +0,0 @@
require "test_helper"
require "ostruct"
class Account::ConvertibleTest < ActiveSupport::TestCase
include EntriesTestHelper, ProviderTestHelper
setup do
@family = families(:empty)
@family.update!(currency: "USD")
# Foreign account (currency is not in the family's primary currency, so it will require exchange rates for net worth rollups)
@account = @family.accounts.create!(name: "Test Account", currency: "EUR", balance: 10000, accountable: Depository.new)
@provider = mock
ExchangeRate.stubs(:provider).returns(@provider)
end
test "syncs required exchange rates for an account" do
create_valuation(account: @account, date: 1.day.ago.to_date, amount: 9500, currency: "EUR")
# Since we had a valuation 1 day ago, this account starts 2 days ago and needs daily exchange rates looking forward
assert_equal 2.days.ago.to_date, @account.start_date
ExchangeRate.delete_all
provider_response = provider_success_response(
[
OpenStruct.new(from: "EUR", to: "USD", date: 2.days.ago.to_date, rate: 1.1),
OpenStruct.new(from: "EUR", to: "USD", date: 1.day.ago.to_date, rate: 1.2),
OpenStruct.new(from: "EUR", to: "USD", date: Date.current, rate: 1.3)
]
)
@provider.expects(:fetch_exchange_rates)
.with(from: "EUR", to: "USD", start_date: 2.days.ago.to_date, end_date: Date.current)
.returns(provider_response)
assert_difference "ExchangeRate.count", 3 do
@account.sync_required_exchange_rates
end
end
test "does not sync rates for a domestic account" do
@account.update!(currency: "USD")
@provider.expects(:fetch_exchange_rates).never
assert_no_difference "ExchangeRate.count" do
@account.sync_required_exchange_rates
end
end
end

View file

@ -0,0 +1,107 @@
require "test_helper"
require "ostruct"
class Account::MarketDataSyncerTest < ActiveSupport::TestCase
include ProviderTestHelper
PROVIDER_BUFFER = 5.days
setup do
# Ensure a clean slate for deterministic assertions
Security::Price.delete_all
ExchangeRate.delete_all
Trade.delete_all
Holding.delete_all
Security.delete_all
Entry.delete_all
@provider = mock("provider")
Provider::Registry.any_instance
.stubs(:get_provider)
.with(:synth)
.returns(@provider)
end
test "syncs required exchange rates for a foreign-currency account" do
family = Family.create!(name: "Smith", currency: "USD")
account = family.accounts.create!(
name: "Chequing",
currency: "CAD",
balance: 100,
accountable: Depository.new
)
# Seed a rate for the first required day so that the syncer only needs the next day forward
existing_date = account.start_date
ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: existing_date, rate: 2.0)
expected_start_date = (existing_date + 1.day) - PROVIDER_BUFFER
end_date = Date.current.in_time_zone("America/New_York").to_date
@provider.expects(:fetch_exchange_rates)
.with(from: "CAD",
to: "USD",
start_date: expected_start_date,
end_date: end_date)
.returns(provider_success_response([
OpenStruct.new(from: "CAD", to: "USD", date: existing_date, rate: 1.5)
]))
before = ExchangeRate.count
Account::MarketDataSyncer.new(account).sync_market_data
after = ExchangeRate.count
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
end
test "syncs security prices for securities traded by the account" do
family = Family.create!(name: "Smith", currency: "USD")
account = family.accounts.create!(
name: "Brokerage",
currency: "USD",
balance: 0,
accountable: Investment.new
)
security = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
trade_date = 10.days.ago.to_date
trade = Trade.new(security: security, qty: 1, price: 100, currency: "USD")
account.entries.create!(
name: "Buy AAPL",
date: trade_date,
amount: 100,
currency: "USD",
entryable: trade
)
expected_start_date = trade_date - PROVIDER_BUFFER
end_date = Date.current.in_time_zone("America/New_York").to_date
@provider.expects(:fetch_security_prices)
.with(symbol: security.ticker,
exchange_operating_mic: security.exchange_operating_mic,
start_date: expected_start_date,
end_date: end_date)
.returns(provider_success_response([
OpenStruct.new(security: security,
date: trade_date,
price: 100,
currency: "USD")
]))
@provider.stubs(:fetch_security_info)
.with(symbol: security.ticker, exchange_operating_mic: security.exchange_operating_mic)
.returns(provider_success_response(OpenStruct.new(name: "Apple", logo_url: "logo")))
# Ignore exchange-rate calls for this test
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
Account::MarketDataSyncer.new(account).sync_market_data
assert_equal 1, Security::Price.where(security: security, date: trade_date).count
end
end

View file

@ -0,0 +1,148 @@
require "test_helper"
require "ostruct"
class ExchangeRate::SyncerTest < ActiveSupport::TestCase
include ProviderTestHelper
setup do
@provider = mock
end
test "syncs missing rates from provider" do
ExchangeRate.delete_all
provider_response = provider_success_response([
OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.3),
OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.5)
])
@provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
.returns(provider_response)
ExchangeRate::Syncer.new(
exchange_rate_provider: @provider,
from: "USD",
to: "EUR",
start_date: 2.days.ago.to_date,
end_date: Date.current
).sync_provider_rates
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date..Date.current)
.order(:date)
assert_equal 3, db_rates.count
assert_equal 1.3, db_rates[0].rate
assert_equal 1.4, db_rates[1].rate
assert_equal 1.5, db_rates[2].rate
end
test "syncs diff when some rates already exist" do
ExchangeRate.delete_all
# Pre-populate DB with the first two days
ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date: 3.days.ago.to_date, rate: 1.2)
ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date, rate: 1.25)
provider_response = provider_success_response([
OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.3)
])
@provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
.returns(provider_response)
ExchangeRate::Syncer.new(
exchange_rate_provider: @provider,
from: "USD",
to: "EUR",
start_date: 3.days.ago.to_date,
end_date: Date.current
).sync_provider_rates
db_rates = ExchangeRate.order(:date)
assert_equal 4, db_rates.count
assert_equal [ 1.2, 1.25, 1.3, 1.3 ], db_rates.map(&:rate)
end
test "no provider calls when all rates exist" do
ExchangeRate.delete_all
(3.days.ago.to_date..Date.current).each_with_index do |date, idx|
ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date:, rate: 1.2 + idx * 0.01)
end
@provider.expects(:fetch_exchange_rates).never
ExchangeRate::Syncer.new(
exchange_rate_provider: @provider,
from: "USD",
to: "EUR",
start_date: 3.days.ago.to_date,
end_date: Date.current
).sync_provider_rates
end
# A helpful "reset" option for when we need to refresh provider data
test "full upsert if clear_cache is true" do
ExchangeRate.delete_all
# Seed DB with stale data
(2.days.ago.to_date..Date.current).each do |date|
ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date:, rate: 1.0)
end
provider_response = provider_success_response([
OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.3),
OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.5)
])
@provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
.returns(provider_response)
ExchangeRate::Syncer.new(
exchange_rate_provider: @provider,
from: "USD",
to: "EUR",
start_date: 2.days.ago.to_date,
end_date: Date.current,
clear_cache: true
).sync_provider_rates
db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date)
assert_equal [ 1.3, 1.4, 1.5 ], db_rates.map(&:rate)
end
test "clamps end_date to today when future date is provided" do
ExchangeRate.delete_all
future_date = Date.current + 3.days
provider_response = provider_success_response([
OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.6)
])
@provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
.returns(provider_response)
ExchangeRate::Syncer.new(
exchange_rate_provider: @provider,
from: "USD",
to: "EUR",
start_date: Date.current,
end_date: future_date
).sync_provider_rates
assert_equal 1, ExchangeRate.count
end
private
def get_provider_fetch_start_date(start_date)
# We fetch with a 5 day buffer to account for weekends and holidays
start_date - 5.days
end
end

View file

@ -67,26 +67,4 @@ class ExchangeRateTest < ActiveSupport::TestCase
assert_nil ExchangeRate.find_or_fetch_rate(from: "USD", to: "EUR", date: Date.current, cache: true)
end
test "upserts rates for currency pair and date range" do
ExchangeRate.delete_all
ExchangeRate.create!(date: 1.day.ago.to_date, from_currency: "USD", to_currency: "EUR", rate: 0.9)
provider_response = provider_success_response([
OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.3),
OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.5)
])
@provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: 2.days.ago.to_date, end_date: Date.current)
.returns(provider_response)
ExchangeRate.sync_provider_rates(from: "USD", to: "EUR", start_date: 2.days.ago.to_date)
assert_equal 1.3, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: Date.current).rate
assert_equal 1.4, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: 1.day.ago.to_date).rate
assert_equal 1.5, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date).rate
end
end

View file

@ -2,70 +2,84 @@ require "test_helper"
require "ostruct"
class MarketDataSyncerTest < ActiveSupport::TestCase
include EntriesTestHelper, ProviderTestHelper
include ProviderTestHelper
test "syncs exchange rates with upsert" do
empty_db
SNAPSHOT_START_DATE = MarketDataSyncer::SNAPSHOT_DAYS.days.ago.to_date
PROVIDER_BUFFER = 5.days
family1 = Family.create!(name: "Family 1", currency: "USD")
account1 = family1.accounts.create!(name: "Account 1", currency: "USD", balance: 100, accountable: Depository.new)
account2 = family1.accounts.create!(name: "Account 2", currency: "CAD", balance: 100, accountable: Depository.new)
setup do
Security::Price.delete_all
ExchangeRate.delete_all
Trade.delete_all
Holding.delete_all
Security.delete_all
family2 = Family.create!(name: "Family 2", currency: "EUR")
account3 = family2.accounts.create!(name: "Account 3", currency: "EUR", balance: 100, accountable: Depository.new)
account4 = family2.accounts.create!(name: "Account 4", currency: "USD", balance: 100, accountable: Depository.new)
mock_provider = mock
Provider::Registry.any_instance.expects(:get_provider).with(:synth).returns(mock_provider).at_least_once
start_date = 1.month.ago.to_date
end_date = Date.current.in_time_zone("America/New_York").to_date
# Put an existing rate in DB to test upsert
ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: start_date, rate: 2.0)
mock_provider.expects(:fetch_exchange_rates)
.with(from: "CAD", to: "USD", start_date: start_date, end_date: end_date)
.returns(provider_success_response([ OpenStruct.new(from: "CAD", to: "USD", date: start_date, rate: 1.0) ]))
mock_provider.expects(:fetch_exchange_rates)
.with(from: "USD", to: "EUR", start_date: start_date, end_date: end_date)
.returns(provider_success_response([ OpenStruct.new(from: "USD", to: "EUR", date: start_date, rate: 1.0) ]))
assert_difference "ExchangeRate.count", 1 do
MarketDataSyncer.new.sync_exchange_rates
end
assert_equal 1.0, ExchangeRate.where(from_currency: "CAD", to_currency: "USD", date: start_date).first.rate
@provider = mock("provider")
Provider::Registry.any_instance
.stubs(:get_provider)
.with(:synth)
.returns(@provider)
end
test "syncs security prices with upsert" do
empty_db
test "syncs required exchange rates" do
family = Family.create!(name: "Smith", currency: "USD")
family.accounts.create!(name: "Chequing",
currency: "CAD",
balance: 100,
accountable: Depository.new)
aapl = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
# Seed stale rate so only the next missing day is fetched
ExchangeRate.create!(from_currency: "CAD",
to_currency: "USD",
date: SNAPSHOT_START_DATE,
rate: 2.0)
family = Family.create!(name: "Family 1", currency: "USD")
account = family.accounts.create!(name: "Account 1", currency: "USD", balance: 100, accountable: Investment.new)
expected_start_date = (SNAPSHOT_START_DATE + 1.day) - PROVIDER_BUFFER
end_date = Date.current.in_time_zone("America/New_York").to_date
mock_provider = mock
Provider::Registry.any_instance.expects(:get_provider).with(:synth).returns(mock_provider).at_least_once
@provider.expects(:fetch_exchange_rates)
.with(from: "CAD",
to: "USD",
start_date: expected_start_date,
end_date: end_date)
.returns(provider_success_response([
OpenStruct.new(from: "CAD", to: "USD", date: SNAPSHOT_START_DATE, rate: 1.5)
]))
start_date = 1.month.ago.to_date
end_date = Date.current.in_time_zone("America/New_York").to_date
before = ExchangeRate.count
MarketDataSyncer.new(mode: :snapshot).sync_exchange_rates
after = ExchangeRate.count
mock_provider.expects(:fetch_security_prices)
.with(aapl, start_date: start_date, end_date: end_date)
.returns(provider_success_response([ OpenStruct.new(security: aapl, date: start_date, price: 100, currency: "USD") ]))
assert_difference "Security::Price.count", 1 do
MarketDataSyncer.new.sync_prices
end
assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
end
private
def empty_db
Invitation.destroy_all
Family.destroy_all
Security.destroy_all
end
test "syncs security prices" do
security = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
expected_start_date = SNAPSHOT_START_DATE - PROVIDER_BUFFER
end_date = Date.current.in_time_zone("America/New_York").to_date
@provider.expects(:fetch_security_prices)
.with(symbol: security.ticker,
exchange_operating_mic: security.exchange_operating_mic,
start_date: expected_start_date,
end_date: end_date)
.returns(provider_success_response([
OpenStruct.new(security: security,
date: SNAPSHOT_START_DATE,
price: 100,
currency: "USD")
]))
@provider.stubs(:fetch_security_info)
.with(symbol: "AAPL", exchange_operating_mic: "XNAS")
.returns(provider_success_response(OpenStruct.new(name: "Apple", logo_url: "logo")))
# Ignore exchange rate calls for this test
@provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
MarketDataSyncer.new(mode: :snapshot).sync_prices
assert_equal 1, Security::Price.where(security: security, date: SNAPSHOT_START_DATE).count
end
end

View file

@ -0,0 +1,143 @@
require "test_helper"
require "ostruct"
class Security::Price::SyncerTest < ActiveSupport::TestCase
include ProviderTestHelper
setup do
@provider = mock
@security = Security.create!(ticker: "AAPL")
end
test "syncs missing prices from provider" do
Security::Price.delete_all
provider_response = provider_success_response([
OpenStruct.new(security: @security, date: 2.days.ago.to_date, price: 150, currency: "USD"),
OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 155, currency: "USD"),
OpenStruct.new(security: @security, date: Date.current, price: 160, currency: "USD")
])
@provider.expects(:fetch_security_prices)
.with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
.returns(provider_response)
Security::Price::Syncer.new(
security: @security,
security_provider: @provider,
start_date: 2.days.ago.to_date,
end_date: Date.current
).sync_provider_prices
db_prices = Security::Price.where(security: @security, date: 2.days.ago.to_date..Date.current).order(:date)
assert_equal 3, db_prices.count
assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
end
test "syncs diff when some prices already exist" do
Security::Price.delete_all
# Pre-populate DB with first two days
Security::Price.create!(security: @security, date: 3.days.ago.to_date, price: 140, currency: "USD")
Security::Price.create!(security: @security, date: 2.days.ago.to_date, price: 145, currency: "USD")
provider_response = provider_success_response([
OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 150, currency: "USD")
])
@provider.expects(:fetch_security_prices)
.with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
.returns(provider_response)
Security::Price::Syncer.new(
security: @security,
security_provider: @provider,
start_date: 3.days.ago.to_date,
end_date: Date.current
).sync_provider_prices
db_prices = Security::Price.where(security: @security).order(:date)
assert_equal 4, db_prices.count
assert_equal [ 140, 145, 150, 150 ], db_prices.map(&:price)
end
test "no provider calls when all prices exist" do
Security::Price.delete_all
(3.days.ago.to_date..Date.current).each_with_index do |date, idx|
Security::Price.create!(security: @security, date:, price: 100 + idx, currency: "USD")
end
@provider.expects(:fetch_security_prices).never
Security::Price::Syncer.new(
security: @security,
security_provider: @provider,
start_date: 3.days.ago.to_date,
end_date: Date.current
).sync_provider_prices
end
test "full upsert if clear_cache is true" do
Security::Price.delete_all
# Seed DB with stale prices
(2.days.ago.to_date..Date.current).each do |date|
Security::Price.create!(security: @security, date:, price: 100, currency: "USD")
end
provider_response = provider_success_response([
OpenStruct.new(security: @security, date: 2.days.ago.to_date, price: 150, currency: "USD"),
OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 155, currency: "USD"),
OpenStruct.new(security: @security, date: Date.current, price: 160, currency: "USD")
])
@provider.expects(:fetch_security_prices)
.with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
.returns(provider_response)
Security::Price::Syncer.new(
security: @security,
security_provider: @provider,
start_date: 2.days.ago.to_date,
end_date: Date.current,
clear_cache: true
).sync_provider_prices
db_prices = Security::Price.where(security: @security).order(:date)
assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
end
test "clamps end_date to today when future date is provided" do
Security::Price.delete_all
future_date = Date.current + 3.days
provider_response = provider_success_response([
OpenStruct.new(security: @security, date: Date.current, price: 165, currency: "USD")
])
@provider.expects(:fetch_security_prices)
.with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
.returns(provider_response)
Security::Price::Syncer.new(
security: @security,
security_provider: @provider,
start_date: Date.current,
end_date: future_date
).sync_provider_prices
assert_equal 1, Security::Price.count
end
private
def get_provider_fetch_start_date(start_date)
start_date - 5.days
end
end