diff --git a/app/data_migrations/balance_component_migrator.rb b/app/data_migrations/balance_component_migrator.rb new file mode 100644 index 00000000..806c2abc --- /dev/null +++ b/app/data_migrations/balance_component_migrator.rb @@ -0,0 +1,59 @@ +class BalanceComponentMigrator + def self.run + ActiveRecord::Base.transaction do + # Step 1: Update flows factor + ActiveRecord::Base.connection.execute <<~SQL + UPDATE balances SET + flows_factor = CASE WHEN a.classification = 'asset' THEN 1 ELSE -1 END + FROM accounts a + WHERE a.id = balances.account_id + SQL + + # Step 2: Set start values using LOCF (Last Observation Carried Forward) + ActiveRecord::Base.connection.execute <<~SQL + UPDATE balances b1 + SET + start_cash_balance = COALESCE(prev.cash_balance, 0), + start_non_cash_balance = COALESCE(prev.balance - prev.cash_balance, 0) + FROM balances b1_inner + LEFT JOIN LATERAL ( + SELECT + b2.cash_balance, + b2.balance + FROM balances b2 + WHERE b2.account_id = b1_inner.account_id + AND b2.currency = b1_inner.currency + AND b2.date < b1_inner.date + ORDER BY b2.date DESC + LIMIT 1 + ) prev ON true + WHERE b1.id = b1_inner.id + SQL + + # Step 3: Calculate net inflows + # A slight workaround to the fact that we can't easily derive inflows/outflows from our current data model, and + # the tradeoff not worth it since each new sync will fix it. So instead, we sum up *net* flows, and throw the signed + # amount in the "inflows" column, and zero-out the "outflows" column so our math works correctly with incomplete data. + ActiveRecord::Base.connection.execute <<~SQL + UPDATE balances SET + cash_inflows = (cash_balance - start_cash_balance) * flows_factor, + cash_outflows = 0, + non_cash_inflows = ((balance - cash_balance) - start_non_cash_balance) * flows_factor, + non_cash_outflows = 0, + net_market_flows = 0 + SQL + + # Verify data integrity + # All end_balance values should match the original balance + invalid_count = ActiveRecord::Base.connection.select_value(<<~SQL) + SELECT COUNT(*) + FROM balances b + WHERE ABS(b.balance - b.end_balance) > 0.0001 + SQL + + if invalid_count > 0 + raise "Data migration failed validation: #{invalid_count} balances have incorrect end_balance values" + end + end + end +end diff --git a/lib/tasks/data_migration.rake b/lib/tasks/data_migration.rake index 6d3a14fd..febdcb3b 100644 --- a/lib/tasks/data_migration.rake +++ b/lib/tasks/data_migration.rake @@ -154,4 +154,19 @@ namespace :data_migration do puts " Processed: #{accounts_processed} accounts" puts " Opening anchors set: #{opening_anchors_set}" end + + desc "Migrate balance components" + # 2025-07-20: Migrate balance components to support event-sourced ledger model. + # This task: + # 1. Sets the flows_factor for each account based on the account's classification + # 2. Sets the start_cash_balance, start_non_cash_balance, and start_balance for each balance + # 3. Sets the cash_inflows, cash_outflows, non_cash_inflows, non_cash_outflows, net_market_flows, cash_adjustments, and non_cash_adjustments for each balance + # 4. Sets the end_cash_balance, end_non_cash_balance, and end_balance for each balance + task migrate_balance_components: :environment do + puts "==> Migrating balance components..." + + BalanceComponentMigrator.run + + puts "✅ Balance component migration complete." + end end diff --git a/test/data_migrations/balance_component_migrator_test.rb b/test/data_migrations/balance_component_migrator_test.rb new file mode 100644 index 00000000..add8384c --- /dev/null +++ b/test/data_migrations/balance_component_migrator_test.rb @@ -0,0 +1,160 @@ +require "test_helper" + +class BalanceComponentMigratorTest < ActiveSupport::TestCase + include EntriesTestHelper + + setup do + @depository = accounts(:depository) + @investment = accounts(:investment) + @loan = accounts(:loan) + + # Start fresh + Balance.delete_all + end + + test "depository account with no gaps" do + create_balance_history(@depository, [ + { date: 5.days.ago, cash_balance: 1000, balance: 1000 }, + { date: 4.days.ago, cash_balance: 1100, balance: 1100 }, + { date: 3.days.ago, cash_balance: 1050, balance: 1050 }, + { date: 2.days.ago, cash_balance: 1200, balance: 1200 }, + { date: 1.day.ago, cash_balance: 1150, balance: 1150 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @depository, [ + { date: 5.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 1000, non_cash_inflows: 0, end_cash: 1000, end_non_cash: 0, end: 1000 }, + { date: 4.days.ago, start_cash: 1000, start_non_cash: 0, start: 1000, cash_inflows: 100, non_cash_inflows: 0, end_cash: 1100, end_non_cash: 0, end: 1100 }, + { date: 3.days.ago, start_cash: 1100, start_non_cash: 0, start: 1100, cash_inflows: -50, non_cash_inflows: 0, end_cash: 1050, end_non_cash: 0, end: 1050 }, + { date: 2.days.ago, start_cash: 1050, start_non_cash: 0, start: 1050, cash_inflows: 150, non_cash_inflows: 0, end_cash: 1200, end_non_cash: 0, end: 1200 }, + { date: 1.day.ago, start_cash: 1200, start_non_cash: 0, start: 1200, cash_inflows: -50, non_cash_inflows: 0, end_cash: 1150, end_non_cash: 0, end: 1150 } + ] + end + + test "depository account with gaps" do + create_balance_history(@depository, [ + { date: 5.days.ago, cash_balance: 1000, balance: 1000 }, + { date: 1.day.ago, cash_balance: 1150, balance: 1150 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @depository, [ + { date: 5.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 1000, non_cash_inflows: 0, end_cash: 1000, end_non_cash: 0, end: 1000 }, + { date: 1.day.ago, start_cash: 1000, start_non_cash: 0, start: 1000, cash_inflows: 150, non_cash_inflows: 0, end_cash: 1150, end_non_cash: 0, end: 1150 } + ] + end + + test "investment account with no gaps" do + create_balance_history(@investment, [ + { date: 3.days.ago, cash_balance: 100, balance: 200 }, + { date: 2.days.ago, cash_balance: 200, balance: 300 }, + { date: 1.day.ago, cash_balance: 0, balance: 300 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @investment, [ + { date: 3.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 100, non_cash_inflows: 100, end_cash: 100, end_non_cash: 100, end: 200 }, + { date: 2.days.ago, start_cash: 100, start_non_cash: 100, start: 200, cash_inflows: 100, non_cash_inflows: 0, end_cash: 200, end_non_cash: 100, end: 300 }, + { date: 1.day.ago, start_cash: 200, start_non_cash: 100, start: 300, cash_inflows: -200, non_cash_inflows: 200, end_cash: 0, end_non_cash: 300, end: 300 } + ] + end + + test "investment account with gaps" do + create_balance_history(@investment, [ + { date: 5.days.ago, cash_balance: 1000, balance: 1000 }, + { date: 1.day.ago, cash_balance: 1150, balance: 1150 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @investment, [ + { date: 5.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 1000, non_cash_inflows: 0, end_cash: 1000, end_non_cash: 0, end: 1000 }, + { date: 1.day.ago, start_cash: 1000, start_non_cash: 0, start: 1000, cash_inflows: 150, non_cash_inflows: 0, end_cash: 1150, end_non_cash: 0, end: 1150 } + ] + end + + # Negative flows factor test + test "loan account with no gaps" do + create_balance_history(@loan, [ + { date: 3.days.ago, cash_balance: 0, balance: 200 }, + { date: 2.days.ago, cash_balance: 0, balance: 300 }, + { date: 1.day.ago, cash_balance: 0, balance: 500 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @loan, [ + { date: 3.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 0, non_cash_inflows: -200, end_cash: 0, end_non_cash: 200, end: 200 }, + { date: 2.days.ago, start_cash: 0, start_non_cash: 200, start: 200, cash_inflows: 0, non_cash_inflows: -100, end_cash: 0, end_non_cash: 300, end: 300 }, + { date: 1.day.ago, start_cash: 0, start_non_cash: 300, start: 300, cash_inflows: 0, non_cash_inflows: -200, end_cash: 0, end_non_cash: 500, end: 500 } + ] + end + + test "loan account with gaps" do + create_balance_history(@loan, [ + { date: 5.days.ago, cash_balance: 0, balance: 1000 }, + { date: 1.day.ago, cash_balance: 0, balance: 2000 } + ]) + + BalanceComponentMigrator.run + + assert_migrated_balances @loan, [ + { date: 5.days.ago, start_cash: 0, start_non_cash: 0, start: 0, cash_inflows: 0, non_cash_inflows: -1000, end_cash: 0, end_non_cash: 1000, end: 1000 }, + { date: 1.day.ago, start_cash: 0, start_non_cash: 1000, start: 1000, cash_inflows: 0, non_cash_inflows: -1000, end_cash: 0, end_non_cash: 2000, end: 2000 } + ] + end + + private + def create_balance_history(account, balances) + balances.each do |balance| + account.balances.create!( + date: balance[:date].to_date, + balance: balance[:balance], + cash_balance: balance[:cash_balance], + currency: account.currency + ) + end + end + + def assert_migrated_balances(account, expected) + balances = account.balances.order(:date) + + expected.each_with_index do |expected_values, index| + balance = balances.find { |b| b.date == expected_values[:date].to_date } + assert balance, "Expected balance for #{expected_values[:date].to_date} but none found" + + # Assert expected values + assert_equal expected_values[:start_cash], balance.start_cash_balance, + "start_cash_balance mismatch for #{balance.date}" + assert_equal expected_values[:start_non_cash], balance.start_non_cash_balance, + "start_non_cash_balance mismatch for #{balance.date}" + assert_equal expected_values[:start], balance.start_balance, + "start_balance mismatch for #{balance.date}" + assert_equal expected_values[:cash_inflows], balance.cash_inflows, + "cash_inflows mismatch for #{balance.date}" + assert_equal expected_values[:non_cash_inflows], balance.non_cash_inflows, + "non_cash_inflows mismatch for #{balance.date}" + assert_equal expected_values[:end_cash], balance.end_cash_balance, + "end_cash_balance mismatch for #{balance.date}" + assert_equal expected_values[:end_non_cash], balance.end_non_cash_balance, + "end_non_cash_balance mismatch for #{balance.date}" + assert_equal expected_values[:end], balance.end_balance, + "end_balance mismatch for #{balance.date}" + + # Assert zeros for other fields + assert_equal 0, balance.cash_outflows, + "cash_outflows should be zero for #{balance.date}" + assert_equal 0, balance.non_cash_outflows, + "non_cash_outflows should be zero for #{balance.date}" + assert_equal 0, balance.cash_adjustments, + "cash_adjustments should be zero for #{balance.date}" + assert_equal 0, balance.non_cash_adjustments, + "non_cash_adjustments should be zero for #{balance.date}" + assert_equal 0, balance.net_market_flows, + "net_market_flows should be zero for #{balance.date}" + end + end +end