Multi-Currency Part 2 (#543)

* Support all currencies, handle outside DB

* Remove currencies from seed

* Fix account balance namespace

* Set default currency on authentication

* Cache currency instances

* Implement multi-currency syncs with tests

* Series fallback, passing tests

* Fix conflicts

* Make value group concrete class that works with currency values

* Fix migration conflict

* Update tests to expect multi-currency results

* Update account list to use group method

* Namespace updates

* Fetch unknown exchange rates from API

* Fix date range bug

* Ensure demo data works without external API

* Enforce cascades only at DB level

app/models/account/balance.rb

@@ -0,0 +1,8 @@
+class Account::Balance < ApplicationRecord
+    include Monetizable
+
+    belongs_to :account
+    validates :account, :date, :balance, presence: true
+    monetize :balance
+    scope :in_period, ->(period) { period.date_range.nil? ? all : where(date: period.date_range) }
+end

app/models/account/balance/calculator.rb

@@ -0,0 +1,108 @@
+class Account::Balance::Calculator
+    attr_reader :daily_balances, :errors, :warnings
+
+    @daily_balances = []
+    @errors = []
+    @warnings = []
+
+    def initialize(account, options = {})
+      @account = account
+      @calc_start_date = [ options[:calc_start_date], @account.effective_start_date ].compact.max
+    end
+
+    def calculate
+      prior_balance = implied_start_balance
+
+      calculated_balances = ((@calc_start_date + 1.day)...Date.current).map do |date|
+        valuation = normalized_valuations.find { |v| v["date"] == date }
+
+        if valuation
+          current_balance = valuation["value"]
+        else
+          txn_flows = transaction_flows(date)
+          current_balance = prior_balance - txn_flows
+        end
+
+        prior_balance = current_balance
+
+        { date: date, balance: current_balance, currency: @account.currency, updated_at: Time.current }
+      end
+
+      @daily_balances = [
+        { date: @calc_start_date, balance: implied_start_balance, currency: @account.currency, updated_at: Time.current },
+        *calculated_balances,
+        { date: Date.current, balance: @account.balance, currency: @account.currency, updated_at: Time.current } # Last balance must always match "source of truth"
+      ]
+
+      if @account.foreign_currency?
+        converted_balances = convert_balances_to_family_currency
+        @daily_balances.concat(converted_balances)
+      end
+
+      self
+    end
+
+    private
+      def convert_balances_to_family_currency
+        rates = ExchangeRate.get_rate_series(
+          @account.currency,
+          @account.family.currency,
+          @calc_start_date..Date.current
+        ).to_a
+
+        @daily_balances.map do |balance|
+          rate = rates.find { |rate| rate.date == balance[:date] }
+          raise "Rate for #{@account.currency} to #{@account.family.currency} on #{balance[:date]} not found" if rate.nil?
+          converted_balance = balance[:balance] * rate.rate
+          { date: balance[:date], balance: converted_balance, currency: @account.family.currency, updated_at: Time.current }
+        end
+      end
+
+      # For calculation, all transactions and valuations need to be normalized to the same currency (the account's primary currency)
+      def normalize_entries_to_account_currency(entries, value_key)
+        entries.map do |entry|
+          currency = entry.currency
+          date = entry.date
+          value = entry.send(value_key)
+
+          if currency != @account.currency
+            rate = ExchangeRate.find_by(base_currency: currency, converted_currency: @account.currency, date: date)
+            raise "Rate for #{currency} to #{@account.currency} not found" unless rate
+
+            value *= rate.rate
+            currency = @account.currency
+          end
+
+          entry.attributes.merge(value_key.to_s => value, "currency" => currency)
+        end
+      end
+
+      def normalized_valuations
+        @normalized_valuations ||= normalize_entries_to_account_currency(@account.valuations.where("date >= ?", @calc_start_date).order(:date).select(:date, :value, :currency), :value)
+      end
+
+      def normalized_transactions
+        @normalized_transactions ||= normalize_entries_to_account_currency(@account.transactions.where("date >= ?", @calc_start_date).order(:date).select(:date, :amount, :currency), :amount)
+      end
+
+      def transaction_flows(date)
+        flows = normalized_transactions.select { |t| t["date"] == date }.sum { |t| t["amount"] }
+        flows *= -1 if @account.classification == "liability"
+        flows
+      end
+
+      def implied_start_balance
+        oldest_valuation_date = normalized_valuations.first&.dig("date")
+        oldest_transaction_date = normalized_transactions.first&.dig("date")
+        oldest_entry_date = [ oldest_valuation_date, oldest_transaction_date ].compact.min
+
+        if oldest_entry_date == oldest_valuation_date
+          oldest_valuation = normalized_valuations.find { |v| v["date"] == oldest_valuation_date }
+          oldest_valuation["value"].to_d
+        else
+          net_transaction_flows = normalized_transactions.sum { |t| t["amount"].to_d }
+          net_transaction_flows *= -1 if @account.classification == "liability"
+          @account.balance.to_d + net_transaction_flows
+        end
+      end
+end

app/models/account/balance_calculator.rb

@@ -1,40 +0,0 @@
-class Account::BalanceCalculator
-    def initialize(account)
-      @account = account
-    end
-
-    def daily_balances(start_date = nil)
-      calc_start_date = [ start_date, @account.effective_start_date ].compact.max
-
-      valuations = @account.valuations.where("date >= ?", calc_start_date).order(:date).select(:date, :value, :currency)
-      transactions = @account.transactions.where("date > ?", calc_start_date).order(:date).select(:date, :amount, :currency)
-      oldest_entry = [ valuations.first, transactions.first ].compact.min_by(&:date)
-
-      net_transaction_flows = transactions.sum(&:amount)
-      net_transaction_flows *= -1 if @account.classification == "liability"
-      implied_start_balance = oldest_entry.is_a?(Valuation) ? oldest_entry.value : @account.balance + net_transaction_flows
-
-      prior_balance = implied_start_balance
-      calculated_balances = ((calc_start_date + 1.day)...Date.current).map do |date|
-        valuation = valuations.find { |v| v.date == date }
-
-        if valuation
-            current_balance = valuation.value
-        else
-            current_day_net_transaction_flows = transactions.select { |t| t.date == date }.sum(&:amount)
-            current_day_net_transaction_flows *= -1 if @account.classification == "liability"
-            current_balance = prior_balance - current_day_net_transaction_flows
-        end
-
-        prior_balance = current_balance
-
-        { date: date, balance: current_balance, updated_at: Time.current }
-      end
-
-      [
-        { date: calc_start_date, balance: implied_start_balance, updated_at: Time.current },
-        *calculated_balances,
-        { date: Date.current, balance: @account.balance, updated_at: Time.current } # Last balance must always match "source of truth"
-      ]
-    end
-end

app/models/account/syncable.rb

@@ -7,8 +7,12 @@ module Account::Syncable
 
     def sync
         update!(status: "syncing")
-        synced_daily_balances = Account::BalanceCalculator.new(self).daily_balances
-        self.balances.upsert_all(synced_daily_balances, unique_by: :index_account_balances_on_account_id_and_date)
+
+        sync_exchange_rates
+
+        calculator = Account::Balance::Calculator.new(self)
+        calculator.calculate
+        self.balances.upsert_all(calculator.daily_balances, unique_by: :index_account_balances_on_account_id_date_currency_unique)
         self.balances.where("date < ?", effective_start_date).delete_all
         update!(status: "ok")
     rescue => e
@@ -23,4 +27,45 @@ module Account::Syncable
 
         [ first_valuation_date, first_transaction_date&.prev_day ].compact.min || Date.current
     end
+
+    # Finds all the rate pairs that are required to calculate balances for an account and syncs them
+    def sync_exchange_rates
+        rate_candidates = []
+
+        if multi_currency?
+            transactions_in_foreign_currency = self.transactions.where.not(currency: self.currency).pluck(:currency, :date).uniq
+            transactions_in_foreign_currency.each do |currency, date|
+                rate_candidates << { date: date, from_currency: currency, to_currency: self.currency }
+            end
+        end
+
+        if foreign_currency?
+            (effective_start_date..Date.current).each do |date|
+                rate_candidates << { date: date, from_currency: self.currency, to_currency: self.family.currency }
+            end
+        end
+
+        existing_rates = ExchangeRate.where(
+            base_currency: rate_candidates.map { |rc| rc[:from_currency] },
+            converted_currency: rate_candidates.map { |rc| rc[:to_currency] },
+            date: rate_candidates.map { |rc| rc[:date] }
+        ).pluck(:base_currency, :converted_currency, :date)
+
+        # Convert to a set for faster lookup
+        existing_rates_set = existing_rates.map { |er| [ er[0], er[1], er[2].to_s ] }.to_set
+
+        rate_candidates.each do |rate_candidate|
+            rc_from = rate_candidate[:from_currency]
+            rc_to = rate_candidate[:to_currency]
+            rc_date = rate_candidate[:date]
+
+            next if existing_rates_set.include?([ rc_from, rc_to, rc_date.to_s ])
+
+            logger.info "Fetching exchange rate from provider for account #{self.name}: #{self.id} (#{rc_from} to #{rc_to} on #{rc_date})"
+            rate = ExchangeRate.fetch_rate_from_provider(rc_from, rc_to, rc_date)
+            ExchangeRate.create! base_currency: rc_from, converted_currency: rc_to, date: rc_date, rate: rate if rate
+        end
+
+        nil
+    end
 end