Plaid sync tests and multi-currency investment support (#1531)

* Plaid sync tests and multi-currency investment support

* Fix system test

* Cleanup

* Remove data migration

app/controllers/account/holdings_controller.rb

@@ -23,6 +23,6 @@ class Account::HoldingsController < ApplicationController
 
   private
     def set_holding
-      @holding = Current.family.holdings.current.find(params[:id])
+      @holding = Current.family.holdings.find(params[:id])
     end
 end

app/models/account.rb

@@ -118,11 +118,8 @@ class Account < ApplicationRecord
     Money.new(balance_amount, currency)
   end
 
-  def owns_ticker?(ticker)
-    security_id = Security.find_by(ticker: ticker)&.id
-    entries.account_trades
-           .joins("JOIN account_trades ON account_entries.entryable_id = account_trades.id")
-           .where(account_trades: { security_id: security_id }).any?
+  def current_holdings
+    holdings.where(currency: currency, date: holdings.maximum(:date)).order(amount: :desc)
   end
 
   def favorable_direction
@@ -151,12 +148,4 @@ class Account < ApplicationRecord
         entryable: Account::Valuation.new
     end
   end
-
-  def holding_qty(security, date: Date.current)
-    entries.account_trades
-           .joins("JOIN account_trades ON account_entries.entryable_id = account_trades.id")
-           .where(account_trades: { security_id: security.id })
-           .where("account_entries.date <= ?", date)
-           .sum("account_trades.qty")
-  end
 end

app/models/account/holding.rb

@@ -9,9 +9,6 @@ class Account::Holding < ApplicationRecord
   validates :qty, :currency, presence: true
 
   scope :chronological, -> { order(:date) }
-  scope :current, -> { where(date: Date.current).order(amount: :desc) }
-  scope :in_period, ->(period) { period.date_range.nil? ? all : where(date: period.date_range) }
-  scope :known_value, -> { where.not(amount: nil) }
   scope :for, ->(security) { where(security_id: security).order(:date) }
 
   delegate :ticker, to: :security
@@ -29,7 +26,7 @@ class Account::Holding < ApplicationRecord
 
   # Basic approximation of cost-basis
   def avg_cost
-    avg_cost = account.holdings.for(security).where("date <= ?", date).average(:price)
+    avg_cost = account.holdings.for(security).where(currency: currency).where("date <= ?", date).average(:price)
     Money.new(avg_cost, currency)
   end
 

app/models/account/holding_calculator.rb

@@ -52,13 +52,15 @@ class Account::HoldingCalculator
 
         next if price.blank?
 
+        converted_price = Money.new(price.price, price.currency).exchange_to(account.currency, fallback_rate: 1).amount
+
         account.holdings.build(
           security: security.dig(:security),
           date: date,
           qty: qty,
-          price: price.price,
-          currency: price.currency,
-          amount: qty * price.price
+          price: converted_price,
+          currency: account.currency,
+          amount: qty * converted_price
         )
       end.compact
     end
@@ -145,7 +147,7 @@ class Account::HoldingCalculator
     def load_current_holding_quantities
       holding_quantities = load_empty_holding_quantities
 
-      account.holdings.where(date: Date.current).map do |holding|
+      account.holdings.where(date: Date.current, currency: account.currency).map do |holding|
         holding_quantities[holding.security_id] = holding.qty
       end
 

app/models/account/syncer.rb

@@ -9,7 +9,7 @@ class Account::Syncer
     balances = sync_balances(holdings)
     account.reload
     update_account_info(balances, holdings) unless account.plaid_account_id.present?
-    convert_foreign_records(balances)
+    convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency
   end
 
   private
@@ -37,12 +37,7 @@ class Account::Syncer
       current_time = Time.now
 
       Account.transaction do
-        account.holdings.upsert_all(
-          calculated_holdings.map { |h| h.attributes
-                 .slice("date", "currency", "qty", "price", "amount", "security_id")
-                 .merge("updated_at" => current_time) },
-          unique_by: %i[account_id security_id date currency]
-        ) if calculated_holdings.any?
+        load_holdings(calculated_holdings)
 
         # Purge outdated holdings
         account.holdings.delete_by("date < ? OR security_id NOT IN (?)", account_start_date, calculated_holdings.map(&:security_id))
@@ -65,24 +60,7 @@ class Account::Syncer
       calculated_balances
     end
 
-    def convert_foreign_records(balances)
-      converted_balances = convert_balances(balances)
-      load_balances(converted_balances)
-    end
-
-    def load_balances(balances)
-      current_time = Time.now
-      account.balances.upsert_all(
-        balances.map { |b| b.attributes
-               .slice("date", "balance", "cash_balance", "currency")
-               .merge("updated_at" => current_time) },
-        unique_by: %i[account_id date currency]
-      ) if balances.any?
-    end
-
-    def convert_balances(balances)
-      return [] if account.currency == account.family.currency
-
+    def convert_records_to_family_currency(balances, holdings)
       from_currency = account.currency
       to_currency = account.family.currency
 
@@ -92,7 +70,7 @@ class Account::Syncer
         start_date: balances.first.date
       )
 
-      balances.map do |balance|
+      converted_balances = balances.map do |balance|
         exchange_rate = exchange_rates.find { |er| er.date == balance.date }
 
         account.balances.build(
@@ -101,5 +79,41 @@ class Account::Syncer
           currency: to_currency
         ) if exchange_rate.present?
       end
+
+      converted_holdings = holdings.map do |holding|
+        exchange_rate = exchange_rates.find { |er| er.date == holding.date }
+
+        account.holdings.build(
+          security: holding.security,
+          date: holding.date,
+          amount: exchange_rate.rate * holding.amount,
+          currency: to_currency
+        ) if exchange_rate.present?
+      end
+
+      Account.transaction do
+        load_balances(converted_balances)
+        load_holdings(converted_holdings)
+      end
+    end
+
+    def load_balances(balances = [])
+      current_time = Time.now
+      account.balances.upsert_all(
+        balances.map { |b| b.attributes
+               .slice("date", "balance", "cash_balance", "currency")
+               .merge("updated_at" => current_time) },
+        unique_by: %i[account_id date currency]
+      )
+    end
+
+    def load_holdings(holdings = [])
+      current_time = Time.now
+      account.holdings.upsert_all(
+        holdings.map { |h| h.attributes
+               .slice("date", "currency", "qty", "price", "amount", "security_id")
+               .merge("updated_at" => current_time) },
+        unique_by: %i[account_id security_id date currency]
+      )
     end
 end

app/models/investment.rb

@@ -23,8 +23,4 @@ class Investment < ApplicationRecord
   def icon
     "line-chart"
   end
-
-  def post_sync
-    broadcast_refresh_to account.family
-  end
 end

app/models/plaid_account.rb

@@ -45,50 +45,7 @@ class PlaidAccount < ApplicationRecord
   end
 
   def sync_investments!(transactions:, holdings:, securities:)
-    transactions.each do |transaction|
-      if transaction.type == "cash"
-        new_transaction = account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
-          t.name = transaction.name
-          t.amount = transaction.amount
-          t.currency = transaction.iso_currency_code
-          t.date = transaction.date
-          t.marked_as_transfer = transaction.subtype.in?(%w[deposit withdrawal])
-          t.entryable = Account::Transaction.new
-        end
-      else
-        security = get_security(transaction.security, securities)
-        next if security.nil?
-        new_transaction = account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
-          t.name = transaction.name
-          t.amount = transaction.quantity * transaction.price
-          t.currency = transaction.iso_currency_code
-          t.date = transaction.date
-          t.entryable = Account::Trade.new(
-            security: security,
-            qty: transaction.quantity,
-            price: transaction.price,
-            currency: transaction.iso_currency_code
-          )
-        end
-      end
-    end
-
-    # Update only the current day holdings.  The account sync will populate historical values based on trades.
-    holdings.each do |holding|
-      internal_security = get_security(holding.security, securities)
-      next if internal_security.nil?
-
-      existing_holding = account.holdings.find_or_initialize_by(
-        security: internal_security,
-        date: Date.current,
-        currency: holding.iso_currency_code
-      )
-
-      existing_holding.qty = holding.quantity
-      existing_holding.price = holding.institution_price
-      existing_holding.amount = holding.quantity * holding.institution_price
-      existing_holding.save!
-    end
+    PlaidInvestmentSync.new(self).sync!(transactions:, holdings:, securities:)
   end
 
   def sync_credit_data!(plaid_credit_data)
@@ -159,25 +116,6 @@ class PlaidAccount < ApplicationRecord
       plaid_item.family
     end
 
-    def get_security(plaid_security, securities)
-      return nil if plaid_security.nil?
-
-      security = if plaid_security.ticker_symbol.present?
-        plaid_security
-      else
-        securities.find { |s| s.security_id == plaid_security.proxy_security_id }
-      end
-
-      return nil if security.nil? || security.ticker_symbol.blank?
-      return nil if security.ticker_symbol == "CUR:USD" # Internally, we do not consider cash a "holding" and track it separately
-
-      Security.find_or_create_by!(
-        ticker: security.ticker_symbol,
-        exchange_mic: security.market_identifier_code || "XNAS",
-        country_code: "US"
-      )
-    end
-
     def transfer?(plaid_txn)
       transfer_categories = [ "TRANSFER_IN", "TRANSFER_OUT", "LOAN_PAYMENTS" ]
 

app/models/plaid_investment_sync.rb

@@ -0,0 +1,95 @@
+class PlaidInvestmentSync
+  attr_reader :plaid_account
+
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  def sync!(transactions: [], holdings: [], securities: [])
+    @transactions = transactions
+    @holdings = holdings
+    @securities = securities
+
+    PlaidAccount.transaction do
+      sync_transactions!
+      sync_holdings!
+    end
+  end
+
+  private
+    attr_reader :transactions, :holdings, :securities
+
+    def sync_transactions!
+      transactions.each do |transaction|
+        security, plaid_security = get_security(transaction.security_id, securities)
+
+        next if security.nil? && plaid_security.nil?
+
+        if transaction.type == "cash" || plaid_security.ticker_symbol == "CUR:USD"
+          new_transaction = plaid_account.account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
+            t.name = transaction.name
+            t.amount = transaction.amount
+            t.currency = transaction.iso_currency_code
+            t.date = transaction.date
+            t.marked_as_transfer = transaction.subtype.in?(%w[deposit withdrawal])
+            t.entryable = Account::Transaction.new
+          end
+        else
+          new_transaction = plaid_account.account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
+            t.name = transaction.name
+            t.amount = transaction.quantity * transaction.price
+            t.currency = transaction.iso_currency_code
+            t.date = transaction.date
+            t.entryable = Account::Trade.new(
+              security: security,
+              qty: transaction.quantity,
+              price: transaction.price,
+              currency: transaction.iso_currency_code
+            )
+          end
+        end
+      end
+    end
+
+    def sync_holdings!
+      # Update only the current day holdings.  The account sync will populate historical values based on trades.
+      holdings.each do |holding|
+        internal_security, _plaid_security = get_security(holding.security_id, securities)
+
+        next if internal_security.nil?
+
+        existing_holding = plaid_account.account.holdings.find_or_initialize_by(
+          security: internal_security,
+          date: Date.current,
+          currency: holding.iso_currency_code
+        )
+
+        existing_holding.qty = holding.quantity
+        existing_holding.price = holding.institution_price
+        existing_holding.amount = holding.quantity * holding.institution_price
+        existing_holding.save!
+      end
+    end
+
+    def get_security(plaid_security_id, securities)
+      plaid_security = securities.find { |s| s.security_id == plaid_security_id }
+
+      return [ nil, nil ] if plaid_security.nil?
+
+      plaid_security = if plaid_security.ticker_symbol.present?
+        plaid_security
+      else
+        securities.find { |s| s.security_id == plaid_security.proxy_security_id }
+      end
+
+      return [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank?
+
+      security = Security.find_or_create_by!(
+        ticker: plaid_security.ticker_symbol,
+        exchange_mic: plaid_security.market_identifier_code || "XNAS",
+        country_code: "US"
+      ) unless plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
+
+      [ security, plaid_security ]
+    end
+end

app/models/provider/plaid.rb

@@ -134,10 +134,12 @@ class Provider::Plaid
 
   def get_item_investments(item, start_date: nil, end_date: Date.current)
     start_date = start_date || MAX_HISTORY_DAYS.days.ago.to_date
-    holdings = get_item_holdings(item)
-    transactions, securities = get_item_investment_transactions(item, start_date:, end_date:)
+    holdings, holding_securities = get_item_holdings(item)
+    transactions, transaction_securities = get_item_investment_transactions(item, start_date:, end_date:)
 
-    InvestmentsResponse.new(holdings:, transactions:, securities:)
+    merged_securities = ((holding_securities || []) + (transaction_securities || [])).uniq { |s| s.security_id }
+
+    InvestmentsResponse.new(holdings:, transactions:, securities: merged_securities)
   end
 
   def get_item_liabilities(item)
@@ -154,15 +156,7 @@ class Provider::Plaid
       request = Plaid::InvestmentsHoldingsGetRequest.new({ access_token: item.access_token })
       response = client.investments_holdings_get(request)
 
-      securities_by_id = response.securities.index_by(&:security_id)
-      accounts_by_id = response.accounts.index_by(&:account_id)
-
-      response.holdings.each do |holding|
-        holding.define_singleton_method(:security) { securities_by_id[holding.security_id] }
-        holding.define_singleton_method(:account) { accounts_by_id[holding.account_id] }
-      end
-
-      response.holdings
+      [ response.holdings, response.securities ]
     end
 
     def get_item_investment_transactions(item, start_date:, end_date:)
@@ -179,15 +173,8 @@ class Provider::Plaid
         )
 
         response = client.investments_transactions_get(request)
-        securities_by_id = response.securities.index_by(&:security_id)
-        accounts_by_id = response.accounts.index_by(&:account_id)
-
-        response.investment_transactions.each do |t|
-          t.define_singleton_method(:security) { securities_by_id[t.security_id] }
-          t.define_singleton_method(:account) { accounts_by_id[t.account_id] }
-          transactions << t
-        end
 
+        transactions += response.investment_transactions
         securities += response.securities
 
         break if transactions.length >= response.total_investment_transactions

app/models/provider/synth.rb

@@ -43,18 +43,23 @@ class Provider::Synth
     )
   end
 
-  def fetch_security_prices(ticker:, mic_code:, start_date:, end_date:)
-    prices = paginate(
-      "#{base_url}/tickers/#{ticker}/open-close",
-      mic_code: mic_code,
+  def fetch_security_prices(ticker:, start_date:, end_date:, mic_code: nil)
+    params = {
       start_date: start_date,
       end_date: end_date
+    }
+
+    params[:mic_code] = mic_code if mic_code.present?
+
+    prices = paginate(
+      "#{base_url}/tickers/#{ticker}/open-close",
+      params
     ) do |body|
       body.dig("prices").map do |price|
         {
           date: price.dig("date"),
           price: price.dig("close")&.to_f || price.dig("open")&.to_f,
-          currency: "USD"
+          currency: price.dig("currency") || "USD"
         }
       end
     end

app/views/account/holdings/_cash.html.erb

@@ -23,7 +23,7 @@
   </div>
 
   <div class="col-span-2 text-right">
-    <%= tag.p format_money account.cash_balance %>
+    <%= tag.p format_money account.cash_balance_money %>
   </div>
 
   <div class="col-span-2 text-right">

app/views/account/holdings/index.html.erb

@@ -21,10 +21,10 @@
       </div>
 
       <div class="rounded-lg bg-white border-alpha-black-25 shadow-xs">
-        <% if @account.holdings.current.any? %>
+        <% if @account.current_holdings.any? %>
           <%= render "account/holdings/cash", account: @account %>
           <%= render "account/holdings/ruler" %>
-          <%= render partial: "account/holdings/holding", collection: @account.holdings.current, spacer_template: "ruler" %>
+          <%= render partial: "account/holdings/holding", collection: @account.current_holdings, spacer_template: "ruler" %>
         <% else %>
           <p class="text-gray-500 text-sm p-4"><%= t(".no_holdings") %></p>
         <% end %>

app/views/accounts/show/_header.html.erb

@@ -20,7 +20,13 @@
     <% end %>
 
     <div class="flex items-center gap-3 ml-auto">
-      <% unless account.plaid_account_id.present? %>
+      <% if account.plaid_account_id.present? %>
+        <% if Rails.env.development? %>
+          <%= button_to sync_plaid_item_path(account.plaid_account.plaid_item), disabled: account.syncing?, data: { turbo: false }, class: "flex items-center gap-2", title: "Sync Account" do %>
+            <%= lucide_icon "refresh-cw", class: "w-4 h-4 text-gray-500 hover:text-gray-400" %>
+          <% end %>
+        <% end %>
+      <% else %>
         <%= button_to sync_account_path(account), disabled: account.syncing?, data: { turbo: false }, class: "flex items-center gap-2", title: "Sync Account" do %>
           <%= lucide_icon "refresh-cw", class: "w-4 h-4 text-gray-500 hover:text-gray-400" %>
         <% end %>

app/views/shared/_progress_circle.html.erb

@@ -1,7 +1,7 @@
 <%# locals: (progress:, radius: 7, stroke: 2, text_class: "text-green-500") %>
 
-<% 
-  circumference = Math::PI * 2 * radius 
+<%
+  circumference = Math::PI * 2 * radius
   progress_percent = progress.clamp(0, 100)
   stroke_dashoffset = ((100 - progress_percent) * circumference) / 100
   center = radius + stroke / 2
@@ -9,16 +9,15 @@
 
 <svg width="<%= radius * 2 + stroke %>" height="<%= radius * 2 + stroke %>">
   <!-- Background Circle -->
-  <circle 
+  <circle
     class="fill-transparent stroke-current text-gray-300"
     r="<%= radius %>"
     cx="<%= center %>"
     cy="<%= center %>"
-    stroke-width="<%= stroke %>"
-  ></circle>
+    stroke-width="<%= stroke %>"></circle>
 
   <!-- Foreground Circle -->
-  <circle 
+  <circle
     class="fill-transparent stroke-current <%= text_class %>"
     r="<%= radius %>"
     cx="<%= center %>"
@@ -26,6 +25,5 @@
     stroke-width="<%= stroke %>"
     stroke-dasharray="<%= circumference %>"
     stroke-dashoffset="<%= stroke_dashoffset %>"
-    transform="rotate(-90, <%= center %>, <%= center %>)"
-    ></circle>
+    transform="rotate(-90, <%= center %>, <%= center %>)"></circle>
 </svg>