Plaid sync domain improvements (#2267)

Breaks our Plaid sync process out into more manageable classes. Notably, this moves the sync process to a distinct, 2-step flow:

1. Import stage - we first make API calls and import Plaid data to "mirror" tables
2. Processing stage - read the raw data, apply business rules, build internal domain models and sync balances

This provides several benefits:

- Plaid syncs can now be "replayed" without fetching API data again
- Mirror tables provide better audit and debugging capabilities
- Eliminates the "all or nothing" sync behavior that is currently in place, which is brittle

app/models/concerns/enrichable.rb

@@ -27,14 +27,23 @@ module Enrichable
     enrich_attributes({ attr => value }, source:, metadata:)
   end
 
-  # Enriches all attributes that haven't been locked yet
+  # Enriches and logs all attributes that:
+  # - Are not locked
+  # - Are not ignored
+  # - Have changed value from the last saved value
   def enrich_attributes(attrs, source:, metadata: {})
-    enrichable_attrs = Array(attrs).reject { |k, _v| locked?(k) }
+    enrichable_attrs = Array(attrs).reject do |attr_key, attr_value|
+      locked?(attr_key) || ignored_enrichable_attributes.include?(attr_key) || self[attr_key.to_s] == attr_value
+    end
 
     ActiveRecord::Base.transaction do
       enrichable_attrs.each do |attr, value|
         self.send("#{attr}=", value)
-        log_enrichment(attribute_name: attr, attribute_value: value, source: source, metadata: metadata)
+
+        # If it's a new record, this isn't technically an "enrichment".  No logging necessary.
+        unless self.new_record?
+          log_enrichment(attribute_name: attr, attribute_value: value, source: source, metadata: metadata)
+        end
       end
 
       save

app/models/credit_card.rb

@@ -1,6 +1,10 @@
 class CreditCard < ApplicationRecord
   include Accountable
 
+  SUBTYPES = {
+    "credit_card" => { short: "Credit Card", long: "Credit Card" }
+  }.freeze
+
   class << self
     def color
       "#F13636"

app/models/depository.rb

@@ -3,7 +3,10 @@ class Depository < ApplicationRecord
 
   SUBTYPES = {
     "checking" => { short: "Checking", long: "Checking" },
-    "savings" => { short: "Savings", long: "Savings" }
+    "savings" => { short: "Savings", long: "Savings" },
+    "hsa" => { short: "HSA", long: "Health Savings Account" },
+    "cd" => { short: "CD", long: "Certificate of Deposit" },
+    "money_market" => { short: "MM", long: "Money Market" }
   }.freeze
 
   class << self

app/models/entry.rb

@@ -56,6 +56,10 @@ class Entry < ApplicationRecord
     Balance::TrendCalculator.new(self, entries, balances).trend
   end
 
+  def linked?
+    plaid_id.present?
+  end
+
   class << self
     def search(params)
       EntrySearch.new(params).build_query(all)

app/models/family/auto_categorizer.rb

@@ -70,8 +70,7 @@ class Family::AutoCategorizer
           amount: transaction.entry.amount.abs,
           classification: transaction.entry.classification,
           description: transaction.entry.name,
-          merchant: transaction.merchant&.name,
-          hint: transaction.plaid_category_detailed
+          merchant: transaction.merchant&.name
         }
       end
     end

app/models/family/plaid_connectable.rb

@@ -6,9 +6,7 @@ module Family::PlaidConnectable
   end
 
   def create_plaid_item!(public_token:, item_name:, region:)
-    provider = plaid_provider_for_region(region)
-
-    public_token_response = provider.exchange_public_token(public_token)
+    public_token_response = plaid(region).exchange_public_token(public_token)
 
     plaid_item = plaid_items.create!(
       name: item_name,
@@ -23,11 +21,9 @@ module Family::PlaidConnectable
   end
 
   def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil)
-    return nil unless plaid_us || plaid_eu
+    return nil unless plaid(region)
 
-    provider = plaid_provider_for_region(region)
-
-    provider.get_link_token(
+    plaid(region).get_link_token(
       user_id: self.id,
       webhooks_url: webhooks_url,
       redirect_url: redirect_url,
@@ -37,15 +33,7 @@ module Family::PlaidConnectable
   end
 
   private
-    def plaid_us
-      @plaid ||= Provider::Registry.get_provider(:plaid_us)
-    end
-
-    def plaid_eu
-      @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu)
-    end
-
-    def plaid_provider_for_region(region)
-      region.to_sym == :eu ? plaid_eu : plaid_us
+    def plaid(region)
+      @plaid ||= Provider::Registry.plaid_provider_for_region(region)
     end
 end

app/models/investment.rb

@@ -6,12 +6,11 @@ class Investment < ApplicationRecord
     "pension" => { short: "Pension", long: "Pension" },
     "retirement" => { short: "Retirement", long: "Retirement" },
     "401k" => { short: "401(k)", long: "401(k)" },
-    "traditional_401k" => { short: "Traditional 401(k)", long: "Traditional 401(k)" },
     "roth_401k" => { short: "Roth 401(k)", long: "Roth 401(k)" },
     "529_plan" => { short: "529 Plan", long: "529 Plan" },
     "hsa" => { short: "HSA", long: "Health Savings Account" },
     "mutual_fund" => { short: "Mutual Fund", long: "Mutual Fund" },
-    "traditional_ira" => { short: "Traditional IRA", long: "Traditional IRA" },
+    "ira" => { short: "IRA", long: "Traditional IRA" },
     "roth_ira" => { short: "Roth IRA", long: "Roth IRA" },
     "angel" => { short: "Angel", long: "Angel" }
   }.freeze

app/models/loan.rb

@@ -1,6 +1,13 @@
 class Loan < ApplicationRecord
   include Accountable
 
+  SUBTYPES = {
+    "mortgage" => { short: "Mortgage", long: "Mortgage" },
+    "student" => { short: "Student", long: "Student Loan" },
+    "auto" => { short: "Auto", long: "Auto Loan" },
+    "other" => { short: "Other", long: "Other Loan" }
+  }.freeze
+
   def monthly_payment
     return nil if term_months.nil? || interest_rate.nil? || rate_type.nil? || rate_type != "fixed"
     return Money.new(0, account.currency) if account.loan.original_balance.amount.zero? || term_months.zero?

app/models/plaid_account.rb

@@ -1,167 +1,54 @@
 class PlaidAccount < ApplicationRecord
-  TYPE_MAPPING = {
-    "depository" => Depository,
-    "credit" => CreditCard,
-    "loan" => Loan,
-    "investment" => Investment,
-    "other" => OtherAsset
-  }
-
   belongs_to :plaid_item
 
   has_one :account, dependent: :destroy
 
-  accepts_nested_attributes_for :account
+  validates :name, :plaid_type, :currency, presence: true
+  validate :has_balance
 
-  class << self
-    def find_or_create_from_plaid_data!(plaid_data, family)
-      PlaidAccount.transaction do
-        plaid_account = find_or_create_by!(plaid_id: plaid_data.account_id)
-
-        internal_account = family.accounts.find_or_initialize_by(plaid_account_id: plaid_account.id)
-
-        # Only set the name for new records or if the name is not locked
-        if internal_account.new_record? || internal_account.enrichable?(:name)
-          internal_account.name = plaid_data.name
-        end
-        internal_account.balance = plaid_data.balances.current || plaid_data.balances.available
-        internal_account.currency = plaid_data.balances.iso_currency_code
-        internal_account.accountable = TYPE_MAPPING[plaid_data.type].new
-
-        internal_account.save!
-        plaid_account.save!
-
-        plaid_account
-      end
-    end
-  end
-
-  def sync_account_data!(plaid_account_data)
-    update!(
-      current_balance: plaid_account_data.balances.current,
-      available_balance: plaid_account_data.balances.available,
-      currency: plaid_account_data.balances.iso_currency_code,
-      plaid_type: plaid_account_data.type,
-      plaid_subtype: plaid_account_data.subtype,
-      account_attributes: {
-        id: account.id,
-        # Plaid guarantees at least 1 of these
-        balance: plaid_account_data.balances.current || plaid_account_data.balances.available,
-        cash_balance: derive_plaid_cash_balance(plaid_account_data.balances)
-      }
+  def upsert_plaid_snapshot!(account_snapshot)
+    assign_attributes(
+      current_balance: account_snapshot.balances.current,
+      available_balance: account_snapshot.balances.available,
+      currency: account_snapshot.balances.iso_currency_code,
+      plaid_type: account_snapshot.type,
+      plaid_subtype: account_snapshot.subtype,
+      name: account_snapshot.name,
+      mask: account_snapshot.mask,
+      raw_payload: account_snapshot
     )
+
+    save!
   end
 
-  def sync_investments!(transactions:, holdings:, securities:)
-    PlaidInvestmentSync.new(self).sync!(transactions:, holdings:, securities:)
-  end
-
-  def sync_credit_data!(plaid_credit_data)
-    account.update!(
-      accountable_attributes: {
-        id: account.accountable_id,
-        minimum_payment: plaid_credit_data.minimum_payment_amount,
-        apr: plaid_credit_data.aprs.first&.apr_percentage
-      }
+  def upsert_plaid_transactions_snapshot!(transactions_snapshot)
+    assign_attributes(
+      raw_transactions_payload: transactions_snapshot
     )
+
+    save!
   end
 
-  def sync_mortgage_data!(plaid_mortgage_data)
-    create_initial_loan_balance(plaid_mortgage_data)
-
-    account.update!(
-      accountable_attributes: {
-        id: account.accountable_id,
-        rate_type: plaid_mortgage_data.interest_rate&.type,
-        interest_rate: plaid_mortgage_data.interest_rate&.percentage
-      }
+  def upsert_plaid_investments_snapshot!(investments_snapshot)
+    assign_attributes(
+      raw_investments_payload: investments_snapshot
     )
+
+    save!
   end
 
-  def sync_student_loan_data!(plaid_student_loan_data)
-    create_initial_loan_balance(plaid_student_loan_data)
-
-    account.update!(
-      accountable_attributes: {
-        id: account.accountable_id,
-        rate_type: "fixed",
-        interest_rate: plaid_student_loan_data.interest_rate_percentage
-      }
+  def upsert_plaid_liabilities_snapshot!(liabilities_snapshot)
+    assign_attributes(
+      raw_liabilities_payload: liabilities_snapshot
     )
-  end
 
-  def sync_transactions!(added:, modified:, removed:)
-    added.each do |plaid_txn|
-      account.entries.find_or_create_by!(plaid_id: plaid_txn.transaction_id) do |t|
-        t.name = plaid_txn.merchant_name || plaid_txn.original_description
-        t.amount = plaid_txn.amount
-        t.currency = plaid_txn.iso_currency_code
-        t.date = plaid_txn.date
-        t.entryable = Transaction.new(
-          plaid_category: plaid_txn.personal_finance_category.primary,
-          plaid_category_detailed: plaid_txn.personal_finance_category.detailed,
-          merchant: find_or_create_merchant(plaid_txn)
-        )
-      end
-    end
-
-    modified.each do |plaid_txn|
-      existing_txn = account.entries.find_by(plaid_id: plaid_txn.transaction_id)
-
-      existing_txn.update!(
-        amount: plaid_txn.amount,
-        date: plaid_txn.date,
-        entryable_attributes: {
-          plaid_category: plaid_txn.personal_finance_category.primary,
-          plaid_category_detailed: plaid_txn.personal_finance_category.detailed,
-          merchant: find_or_create_merchant(plaid_txn)
-        }
-      )
-    end
-
-    removed.each do |plaid_txn|
-      account.entries.find_by(plaid_id: plaid_txn.transaction_id)&.destroy
-    end
+    save!
   end
 
   private
-    def family
-      plaid_item.family
-    end
-
-    def create_initial_loan_balance(loan_data)
-      if loan_data.origination_principal_amount.present? && loan_data.origination_date.present?
-        account.entries.find_or_create_by!(plaid_id: loan_data.account_id) do |e|
-          e.name = "Initial Principal"
-          e.amount = loan_data.origination_principal_amount
-          e.currency = account.currency
-          e.date = loan_data.origination_date
-          e.entryable = Valuation.new
-        end
-      end
-    end
-
-    def find_or_create_merchant(plaid_txn)
-      unless plaid_txn.merchant_entity_id.present? && plaid_txn.merchant_name.present?
-        return nil
-      end
-
-      ProviderMerchant.find_or_create_by!(
-        source: "plaid",
-        name: plaid_txn.merchant_name,
-      ) do |m|
-        m.provider_merchant_id = plaid_txn.merchant_entity_id
-        m.website_url = plaid_txn.website
-        m.logo_url = plaid_txn.logo_url
-      end
-    end
-
-    def derive_plaid_cash_balance(plaid_balances)
-      if account.investment?
-        plaid_balances.available || 0
-      else
-        # For now, we will not distinguish between "cash" and "overall" balance for non-investment accounts
-        plaid_balances.current || plaid_balances.available
-      end
+    # Plaid guarantees at least one of these.  This validation is a sanity check for that guarantee.
+    def has_balance
+      return if current_balance.present? || available_balance.present?
+      errors.add(:base, "Plaid account must have either current or available balance")
     end
 end

app/models/plaid_account/importer.rb

@@ -0,0 +1,34 @@
+class PlaidAccount::Importer
+  def initialize(plaid_account, account_snapshot:)
+    @plaid_account = plaid_account
+    @account_snapshot = account_snapshot
+  end
+
+  def import
+    PlaidAccount.transaction do
+      import_account_info
+      import_transactions if account_snapshot.transactions_data.present?
+      import_investments if account_snapshot.investments_data.present?
+      import_liabilities if account_snapshot.liabilities_data.present?
+    end
+  end
+
+  private
+    attr_reader :plaid_account, :account_snapshot
+
+    def import_account_info
+      plaid_account.upsert_plaid_snapshot!(account_snapshot.account_data)
+    end
+
+    def import_transactions
+      plaid_account.upsert_plaid_transactions_snapshot!(account_snapshot.transactions_data)
+    end
+
+    def import_investments
+      plaid_account.upsert_plaid_investments_snapshot!(account_snapshot.investments_data)
+    end
+
+    def import_liabilities
+      plaid_account.upsert_plaid_liabilities_snapshot!(account_snapshot.liabilities_data)
+    end
+end

app/models/plaid_account/investments/balance_calculator.rb

@@ -0,0 +1,71 @@
+# Plaid Investment balances have a ton of edge cases.  This processor is responsible
+# for deriving "brokerage cash" vs. "total value" based on Plaid's reported balances and holdings.
+class PlaidAccount::Investments::BalanceCalculator
+  NegativeCashBalanceError = Class.new(StandardError)
+  NegativeTotalValueError = Class.new(StandardError)
+
+  def initialize(plaid_account, security_resolver:)
+    @plaid_account = plaid_account
+    @security_resolver = security_resolver
+  end
+
+  def balance
+    total_value = total_investment_account_value
+
+    if total_value.negative?
+      Sentry.capture_exception(
+        NegativeTotalValueError.new("Total value is negative for plaid investment account"),
+        level: :warning
+      )
+    end
+
+    total_value
+  end
+
+  # Plaid considers "brokerage cash" and "cash equivalent holdings" to all be part of "cash balance"
+  #
+  # Internally, we DO NOT.  Maybe clearly distinguishes between "brokerage cash" vs. "holdings (i.e. invested cash)"
+  # For this reason, we must manually calculate the cash balance based on "total value" and "holdings value"
+  # See PlaidAccount::Investments::SecurityResolver for more details.
+  def cash_balance
+    cash_balance = calculate_investment_brokerage_cash
+
+    if cash_balance.negative?
+      Sentry.capture_exception(
+        NegativeCashBalanceError.new("Cash balance is negative for plaid investment account"),
+        level: :warning
+      )
+    end
+
+    cash_balance
+  end
+
+  private
+    attr_reader :plaid_account, :security_resolver
+
+    def holdings
+      plaid_account.raw_investments_payload["holdings"] || []
+    end
+
+    def calculate_investment_brokerage_cash
+      total_investment_account_value - true_holdings_value
+    end
+
+    # This is our source of truth.  We assume Plaid's `current_balance` reporting is 100% accurate
+    # Plaid guarantees `current_balance` AND/OR `available_balance` is always present, and based on the docs,
+    # `current_balance` should represent "total account value".
+    def total_investment_account_value
+      plaid_account.current_balance || plaid_account.available_balance
+    end
+
+    # Plaid holdings summed up, LESS "brokerage cash" holdings (that we've manually identified)
+    def true_holdings_value
+      # True holdings are holdings *less* Plaid's "pseudo-securities" (e.g. `CUR:USD` brokerage cash "holding")
+      true_holdings = holdings.reject do |h|
+        security = security_resolver.resolve(plaid_security_id: h["security_id"])
+        security.brokerage_cash?
+      end
+
+      true_holdings.sum { |h| h["quantity"] * h["institution_price"] }
+    end
+end

app/models/plaid_account/investments/holdings_processor.rb

@@ -0,0 +1,39 @@
+class PlaidAccount::Investments::HoldingsProcessor
+  def initialize(plaid_account, security_resolver:)
+    @plaid_account = plaid_account
+    @security_resolver = security_resolver
+  end
+
+  def process
+    holdings.each do |plaid_holding|
+      resolved_security_result = security_resolver.resolve(plaid_security_id: plaid_holding["security_id"])
+
+      return unless resolved_security_result.security.present?
+
+      holding = account.holdings.find_or_initialize_by(
+        security: resolved_security_result.security,
+        date: Date.current,
+        currency: plaid_holding["iso_currency_code"]
+      )
+
+      holding.assign_attributes(
+        qty: plaid_holding["quantity"],
+        price: plaid_holding["institution_price"],
+        amount: plaid_holding["quantity"] * plaid_holding["institution_price"]
+      )
+
+      holding.save!
+    end
+  end
+
+  private
+    attr_reader :plaid_account, :security_resolver
+
+    def account
+      plaid_account.account
+    end
+
+    def holdings
+      plaid_account.raw_investments_payload["holdings"] || []
+    end
+end

app/models/plaid_account/investments/security_resolver.rb

@@ -0,0 +1,93 @@
+# Resolves a Plaid security to an internal Security record, or nil
+class PlaidAccount::Investments::SecurityResolver
+  UnresolvablePlaidSecurityError = Class.new(StandardError)
+
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+    @security_cache = {}
+  end
+
+  # Resolves an internal Security record for a given Plaid security ID
+  def resolve(plaid_security_id:)
+    response = @security_cache[plaid_security_id]
+    return response if response.present?
+
+    plaid_security = get_plaid_security(plaid_security_id)
+
+    if plaid_security.nil?
+      report_unresolvable_security(plaid_security_id)
+      response = Response.new(security: nil, cash_equivalent?: false, brokerage_cash?: false)
+    elsif brokerage_cash?(plaid_security)
+      response = Response.new(security: nil, cash_equivalent?: true, brokerage_cash?: true)
+    else
+      security = Security::Resolver.new(
+        plaid_security["ticker_symbol"],
+        exchange_operating_mic: plaid_security["market_identifier_code"]
+      ).resolve
+
+      response = Response.new(
+        security: security,
+        cash_equivalent?: cash_equivalent?(plaid_security),
+        brokerage_cash?: false
+      )
+    end
+
+    @security_cache[plaid_security_id] = response
+
+    response
+  end
+
+  private
+    attr_reader :plaid_account, :security_cache
+
+    Response = Struct.new(:security, :cash_equivalent?, :brokerage_cash?, keyword_init: true)
+
+    def securities
+      plaid_account.raw_investments_payload["securities"] || []
+    end
+
+    # Tries to find security, or returns the "proxy security" (common with options contracts that have underlying securities)
+    def get_plaid_security(plaid_security_id)
+      security = securities.find { |s| s["security_id"] == plaid_security_id && s["ticker_symbol"].present? }
+
+      return security if security.present?
+
+      securities.find { |s| s["proxy_security_id"] == plaid_security_id }
+    end
+
+    def report_unresolvable_security(plaid_security_id)
+      Sentry.capture_exception(UnresolvablePlaidSecurityError.new("Could not resolve Plaid security from provided data")) do |scope|
+        scope.set_context("plaid_security", {
+          plaid_security_id: plaid_security_id
+        })
+      end
+    end
+
+    # Plaid treats "brokerage cash" differently than us.  Internally, Maybe treats "brokerage cash"
+    # as "uninvested cash" (i.e. cash that doesn't have a corresponding Security and can be withdrawn).
+    #
+    # Plaid treats everything as a "holding" with a corresponding Security.  For example, "brokerage cash" (USD)
+    # in Plaids data model would be represented as:
+    #
+    # - A Security with ticker `CUR:USD`
+    # - A holding, linked to the `CUR:USD` Security, with an institution price of $1
+    #
+    # Internally, we store brokerage cash balance as `account.cash_balance`, NOT as a holding + security.
+    # This allows us to properly build historical cash balances and holdings values separately and accurately.
+    #
+    # These help identify these "special case" securities for various calculations.
+    #
+    def known_plaid_brokerage_cash_tickers
+      [ "CUR:USD" ]
+    end
+
+    def brokerage_cash?(plaid_security)
+      return false unless plaid_security["ticker_symbol"].present?
+      known_plaid_brokerage_cash_tickers.include?(plaid_security["ticker_symbol"])
+    end
+
+    def cash_equivalent?(plaid_security)
+      return false unless plaid_security["type"].present?
+      plaid_security["type"] == "cash" || plaid_security["is_cash_equivalent"] == true
+    end
+end

app/models/plaid_account/investments/transactions_processor.rb

@@ -0,0 +1,90 @@
+class PlaidAccount::Investments::TransactionsProcessor
+  SecurityNotFoundError = Class.new(StandardError)
+
+  def initialize(plaid_account, security_resolver:)
+    @plaid_account = plaid_account
+    @security_resolver = security_resolver
+  end
+
+  def process
+    transactions.each do |transaction|
+      if cash_transaction?(transaction)
+        find_or_create_cash_entry(transaction)
+      else
+        find_or_create_trade_entry(transaction)
+      end
+    end
+  end
+
+  private
+    attr_reader :plaid_account, :security_resolver
+
+    def account
+      plaid_account.account
+    end
+
+    def cash_transaction?(transaction)
+      transaction["type"] == "cash" || transaction["type"] == "fee"
+    end
+
+    def find_or_create_trade_entry(transaction)
+      resolved_security_result = security_resolver.resolve(plaid_security_id: transaction["security_id"])
+
+      unless resolved_security_result.security.present?
+        Sentry.capture_exception(SecurityNotFoundError.new("Could not find security for plaid trade")) do |scope|
+          scope.set_tags(plaid_account_id: plaid_account.id)
+        end
+
+        return # We can't process a non-cash transaction without a security
+      end
+
+      entry = account.entries.find_or_initialize_by(plaid_id: transaction["investment_transaction_id"]) do |e|
+        e.entryable = Trade.new
+      end
+
+      entry.assign_attributes(
+        amount: transaction["quantity"] * transaction["price"],
+        currency: transaction["iso_currency_code"],
+        date: transaction["date"]
+      )
+
+      entry.trade.assign_attributes(
+        security: resolved_security_result.security,
+        qty: transaction["quantity"],
+        price: transaction["price"],
+        currency: transaction["iso_currency_code"]
+      )
+
+      entry.enrich_attribute(
+        :name,
+        transaction["name"],
+        source: "plaid"
+      )
+
+      entry.save!
+    end
+
+    def find_or_create_cash_entry(transaction)
+      entry = account.entries.find_or_initialize_by(plaid_id: transaction["investment_transaction_id"]) do |e|
+        e.entryable = Transaction.new
+      end
+
+      entry.assign_attributes(
+        amount: transaction["amount"],
+        currency: transaction["iso_currency_code"],
+        date: transaction["date"]
+      )
+
+      entry.enrich_attribute(
+        :name,
+        transaction["name"],
+        source: "plaid"
+      )
+
+      entry.save!
+    end
+
+    def transactions
+      plaid_account.raw_investments_payload["transactions"] || []
+    end
+end

app/models/plaid_account/liabilities/credit_processor.rb

@@ -0,0 +1,25 @@
+class PlaidAccount::Liabilities::CreditProcessor
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  def process
+    return unless credit_data.present?
+
+    account.credit_card.update!(
+      minimum_payment: credit_data.dig("minimum_payment_amount"),
+      apr: credit_data.dig("aprs", 0, "apr_percentage")
+    )
+  end
+
+  private
+    attr_reader :plaid_account
+
+    def account
+      plaid_account.account
+    end
+
+    def credit_data
+      plaid_account.raw_liabilities_payload["credit"]
+    end
+end

app/models/plaid_account/liabilities/mortgage_processor.rb

@@ -0,0 +1,25 @@
+class PlaidAccount::Liabilities::MortgageProcessor
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  def process
+    return unless mortgage_data.present?
+
+    account.loan.update!(
+      rate_type: mortgage_data.dig("interest_rate", "type"),
+      interest_rate: mortgage_data.dig("interest_rate", "percentage")
+    )
+  end
+
+  private
+    attr_reader :plaid_account
+
+    def account
+      plaid_account.account
+    end
+
+    def mortgage_data
+      plaid_account.raw_liabilities_payload["mortgage"]
+    end
+end

app/models/plaid_account/liabilities/student_loan_processor.rb

@@ -0,0 +1,50 @@
+class PlaidAccount::Liabilities::StudentLoanProcessor
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  def process
+    return unless student_loan_data.present?
+
+    account.loan.update!(
+      rate_type: "fixed",
+      interest_rate: student_loan_data["interest_rate_percentage"],
+      initial_balance: student_loan_data["origination_principal_amount"],
+      term_months: term_months
+    )
+  end
+
+  private
+    attr_reader :plaid_account
+
+    def account
+      plaid_account.account
+    end
+
+    def term_months
+      return nil unless origination_date && expected_payoff_date
+
+      ((expected_payoff_date - origination_date).to_i / 30).to_i
+    end
+
+    def origination_date
+      parse_date(student_loan_data["origination_date"])
+    end
+
+    def expected_payoff_date
+      parse_date(student_loan_data["expected_payoff_date"])
+    end
+
+    def parse_date(value)
+      return value if value.is_a?(Date)
+      return nil unless value.present?
+
+      Date.parse(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+
+    def student_loan_data
+      plaid_account.raw_liabilities_payload["student"]
+    end
+end

app/models/plaid_account/processor.rb

@@ -0,0 +1,99 @@
+class PlaidAccount::Processor
+  include PlaidAccount::TypeMappable
+
+  attr_reader :plaid_account
+
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  # Each step represents a different Plaid API endpoint / "product"
+  #
+  # Processing the account is the first step and if it fails, we halt the entire processor
+  # Each subsequent step can fail independently, but we continue processing the rest of the steps
+  def process
+    process_account!
+    process_transactions
+    process_investments
+    process_liabilities
+  end
+
+  private
+    def family
+      plaid_account.plaid_item.family
+    end
+
+    # Shared securities reader and resolver
+    def security_resolver
+      @security_resolver ||= PlaidAccount::Investments::SecurityResolver.new(plaid_account)
+    end
+
+    def process_account!
+      PlaidAccount.transaction do
+        account = family.accounts.find_or_initialize_by(
+          plaid_account_id: plaid_account.id
+        )
+
+        # Name and subtype are the only attributes a user can override for Plaid accounts
+        account.enrich_attributes(
+          {
+            name: plaid_account.name,
+            subtype: map_subtype(plaid_account.plaid_type, plaid_account.plaid_subtype)
+          },
+          source: "plaid"
+        )
+
+        account.assign_attributes(
+          accountable: map_accountable(plaid_account.plaid_type),
+          balance: balance_calculator.balance,
+          currency: plaid_account.currency,
+          cash_balance: balance_calculator.cash_balance
+        )
+
+        account.save!
+      end
+    end
+
+    def process_transactions
+      PlaidAccount::Transactions::Processor.new(plaid_account).process
+    rescue => e
+      report_exception(e)
+    end
+
+    def process_investments
+      PlaidAccount::Investments::TransactionsProcessor.new(plaid_account, security_resolver: security_resolver).process
+      PlaidAccount::Investments::HoldingsProcessor.new(plaid_account, security_resolver: security_resolver).process
+    rescue => e
+      report_exception(e)
+    end
+
+    def process_liabilities
+      case [ plaid_account.plaid_type, plaid_account.plaid_subtype ]
+      when [ "credit", "credit card" ]
+        PlaidAccount::Liabilities::CreditProcessor.new(plaid_account).process
+      when [ "loan", "mortgage" ]
+        PlaidAccount::Liabilities::MortgageProcessor.new(plaid_account).process
+      when [ "loan", "student" ]
+        PlaidAccount::Liabilities::StudentLoanProcessor.new(plaid_account).process
+      end
+    rescue => e
+      report_exception(e)
+    end
+
+    def balance_calculator
+      if plaid_account.plaid_type == "investment"
+        @balance_calculator ||= PlaidAccount::Investments::BalanceCalculator.new(plaid_account, security_resolver: security_resolver)
+      else
+        OpenStruct.new(
+          balance: plaid_account.current_balance || plaid_account.available_balance,
+          cash_balance: plaid_account.available_balance || 0
+        )
+      end
+    end
+
+    def report_exception(error)
+      Sentry.capture_exception(error) do |scope|
+        scope.set_tags(plaid_account_id: plaid_account.id)
+      end
+    end
+end

app/models/plaid_account/transactions/category_matcher.rb

@@ -10,10 +10,10 @@
 #
 # This class is simply a FAST and CHEAP way to match categories that are high confidence.
 # Edge cases will be handled by user-defined rules.
-class Provider::Plaid::CategoryAliasMatcher
-  include Provider::Plaid::CategoryTaxonomy
+class PlaidAccount::Transactions::CategoryMatcher
+  include PlaidAccount::Transactions::CategoryTaxonomy
 
-  def initialize(user_categories)
+  def initialize(user_categories = [])
     @user_categories = user_categories
   end
 

app/models/plaid_account/transactions/category_taxonomy.rb

@@ -1,5 +1,5 @@
 # https://plaid.com/documents/transactions-personal-finance-category-taxonomy.csv
-module Provider::Plaid::CategoryTaxonomy
+module PlaidAccount::Transactions::CategoryTaxonomy
   CATEGORIES_MAP = {
     income: {
       classification: :income,

app/models/plaid_account/transactions/processor.rb

@@ -0,0 +1,60 @@
+class PlaidAccount::Transactions::Processor
+  def initialize(plaid_account)
+    @plaid_account = plaid_account
+  end
+
+  def process
+    # Each entry is processed inside a transaction, but to avoid locking up the DB when
+    # there are hundreds or thousands of transactions, we process them individually.
+    modified_transactions.each do |transaction|
+      PlaidEntry::Processor.new(
+        transaction,
+        plaid_account: plaid_account,
+        category_matcher: category_matcher
+      ).process
+    end
+
+    PlaidAccount.transaction do
+      removed_transactions.each do |transaction|
+        remove_plaid_transaction(transaction)
+      end
+    end
+  end
+
+  private
+    attr_reader :plaid_account
+
+    def category_matcher
+      @category_matcher ||= PlaidAccount::Transactions::CategoryMatcher.new(family_categories)
+    end
+
+    def family_categories
+      @family_categories ||= begin
+        if account.family.categories.none?
+          account.family.categories.bootstrap!
+        end
+
+        account.family.categories
+      end
+    end
+
+    def account
+      plaid_account.account
+    end
+
+    def remove_plaid_transaction(raw_transaction)
+      account.entries.find_by(plaid_id: raw_transaction["transaction_id"])&.destroy
+    end
+
+    # Since we find_or_create_by transactions, we don't need a distinction between added/modified
+    def modified_transactions
+      modified = plaid_account.raw_transactions_payload["modified"] || []
+      added = plaid_account.raw_transactions_payload["added"] || []
+
+      modified + added
+    end
+
+    def removed_transactions
+      plaid_account.raw_transactions_payload["removed"] || []
+    end
+end

app/models/plaid_account/type_mappable.rb

@@ -0,0 +1,77 @@
+module PlaidAccount::TypeMappable
+  extend ActiveSupport::Concern
+
+  UnknownAccountTypeError = Class.new(StandardError)
+
+  def map_accountable(plaid_type)
+    accountable_class = TYPE_MAPPING.dig(
+      plaid_type.to_sym,
+      :accountable
+    )
+
+    unless accountable_class
+      raise UnknownAccountTypeError, "Unknown account type: #{plaid_type}"
+    end
+
+    accountable_class.new
+  end
+
+  def map_subtype(plaid_type, plaid_subtype)
+    TYPE_MAPPING.dig(
+      plaid_type.to_sym,
+      :subtype_mapping,
+      plaid_subtype
+    ) || "other"
+  end
+
+  # Plaid Account Types -> Accountable Types
+  # https://plaid.com/docs/api/accounts/#account-type-schema
+  TYPE_MAPPING = {
+    depository: {
+      accountable: Depository,
+      subtype_mapping: {
+        "checking" => "checking",
+        "savings" => "savings",
+        "hsa" => "hsa",
+        "cd" => "cd",
+        "money market" => "money_market"
+      }
+    },
+    credit: {
+      accountable: CreditCard,
+      subtype_mapping: {
+        "credit card" => "credit_card"
+      }
+    },
+    loan: {
+      accountable: Loan,
+      subtype_mapping: {
+        "mortgage" => "mortgage",
+        "student" => "student",
+        "auto" => "auto",
+        "business" => "business",
+        "home equity" => "home_equity",
+        "line of credit" => "line_of_credit"
+      }
+    },
+    investment: {
+      accountable: Investment,
+      subtype_mapping: {
+        "brokerage" => "brokerage",
+        "pension" => "pension",
+        "retirement" => "retirement",
+        "401k" => "401k",
+        "roth 401k" => "roth_401k",
+        "529" => "529_plan",
+        "hsa" => "hsa",
+        "mutual fund" => "mutual_fund",
+        "roth" => "roth_ira",
+        "ira" => "ira"
+      }
+    },
+    other: {
+      accountable: OtherAsset,
+      subtype_mapping: {}
+    }
+  }
+end

app/models/plaid_entry/processor.rb

@@ -0,0 +1,95 @@
+class PlaidEntry::Processor
+  # plaid_transaction is the raw hash fetched from Plaid API and converted to JSONB
+  def initialize(plaid_transaction, plaid_account:, category_matcher:)
+    @plaid_transaction = plaid_transaction
+    @plaid_account = plaid_account
+    @category_matcher = category_matcher
+  end
+
+  def process
+    PlaidAccount.transaction do
+      entry = account.entries.find_or_initialize_by(plaid_id: plaid_id) do |e|
+        e.entryable = Transaction.new
+      end
+
+      entry.assign_attributes(
+        amount: amount,
+        currency: currency,
+        date: date
+      )
+
+      entry.enrich_attribute(
+        :name,
+        name,
+        source: "plaid"
+      )
+
+      if detailed_category
+        matched_category = category_matcher.match(detailed_category)
+
+        if matched_category
+          entry.transaction.enrich_attribute(
+            :category_id,
+            matched_category.id,
+            source: "plaid"
+          )
+        end
+      end
+
+      if merchant
+        entry.transaction.enrich_attribute(
+          :merchant_id,
+          merchant.id,
+          source: "plaid"
+        )
+      end
+    end
+  end
+
+  private
+    attr_reader :plaid_transaction, :plaid_account, :category_matcher
+
+    def account
+      plaid_account.account
+    end
+
+    def plaid_id
+      plaid_transaction["transaction_id"]
+    end
+
+    def name
+      plaid_transaction["merchant_name"] || plaid_transaction["original_description"]
+    end
+
+    def amount
+      plaid_transaction["amount"]
+    end
+
+    def currency
+      plaid_transaction["iso_currency_code"]
+    end
+
+    def date
+      plaid_transaction["date"]
+    end
+
+    def detailed_category
+      plaid_transaction.dig("personal_finance_category", "detailed")
+    end
+
+    def merchant
+      merchant_id = plaid_transaction["merchant_entity_id"]
+      merchant_name = plaid_transaction["merchant_name"]
+
+      return nil unless merchant_id.present? && merchant_name.present?
+
+      ProviderMerchant.find_or_create_by!(
+        source: "plaid",
+        name: merchant_name,
+      ) do |m|
+        m.provider_merchant_id = merchant_id
+        m.website_url = plaid_transaction["website"]
+        m.logo_url = plaid_transaction["logo_url"]
+      end
+    end
+end

app/models/plaid_investment_sync.rb

@@ -1,115 +0,0 @@
-class PlaidInvestmentSync
-  attr_reader :plaid_account
-
-  def initialize(plaid_account)
-    @plaid_account = plaid_account
-  end
-
-  def sync!(transactions: [], holdings: [], securities: [])
-    @transactions = transactions
-    @holdings = holdings
-    @securities = securities
-
-    PlaidAccount.transaction do
-      normalize_cash_balance!
-      sync_transactions!
-      sync_holdings!
-    end
-  end
-
-  private
-    attr_reader :transactions, :holdings, :securities
-
-    # Plaid considers "brokerage cash" and "cash equivalent holdings" to all be part of "cash balance"
-    # Internally, we DO NOT.
-    # Maybe clearly distinguishes between "brokerage cash" vs. "holdings (i.e. invested cash)"
-    # For this reason, we must back out cash + cash equivalent holdings from the reported cash balance to avoid double counting
-    def normalize_cash_balance!
-      excludable_cash_holdings = holdings.select do |h|
-        internal_security, plaid_security = get_security(h.security_id, securities)
-        internal_security.present? && (plaid_security&.is_cash_equivalent || plaid_security&.type == "cash")
-      end
-
-      excludable_cash_holdings_value = excludable_cash_holdings.sum { |h| h.quantity * h.institution_price }
-
-      plaid_account.account.update!(
-        cash_balance: plaid_account.account.cash_balance - excludable_cash_holdings_value
-      )
-    end
-
-    def sync_transactions!
-      transactions.each do |transaction|
-        security, plaid_security = get_security(transaction.security_id, securities)
-
-        next if security.nil? && plaid_security.nil?
-
-        if transaction.type == "cash" || plaid_security.ticker_symbol == "CUR:USD"
-          new_transaction = plaid_account.account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
-            t.name = transaction.name
-            t.amount = transaction.amount
-            t.currency = transaction.iso_currency_code
-            t.date = transaction.date
-            t.entryable = Transaction.new
-          end
-        else
-          new_transaction = plaid_account.account.entries.find_or_create_by!(plaid_id: transaction.investment_transaction_id) do |t|
-            t.name = transaction.name
-            t.amount = transaction.quantity * transaction.price
-            t.currency = transaction.iso_currency_code
-            t.date = transaction.date
-            t.entryable = Trade.new(
-              security: security,
-              qty: transaction.quantity,
-              price: transaction.price,
-              currency: transaction.iso_currency_code
-            )
-          end
-        end
-      end
-    end
-
-    def sync_holdings!
-      # Update only the current day holdings.  The account sync will populate historical values based on trades.
-      holdings.each do |holding|
-        internal_security, _plaid_security = get_security(holding.security_id, securities)
-
-        next if internal_security.nil?
-
-        existing_holding = plaid_account.account.holdings.find_or_initialize_by(
-          security: internal_security,
-          date: Date.current,
-          currency: holding.iso_currency_code
-        )
-
-        existing_holding.qty = holding.quantity
-        existing_holding.price = holding.institution_price
-        existing_holding.amount = holding.quantity * holding.institution_price
-        existing_holding.save!
-      end
-    end
-
-    def get_security(plaid_security_id, securities)
-      plaid_security = securities.find { |s| s.security_id == plaid_security_id }
-
-      return [ nil, nil ] if plaid_security.nil?
-
-      plaid_security = if plaid_security.ticker_symbol.present?
-        plaid_security
-      else
-        securities.find { |s| s.security_id == plaid_security.proxy_security_id }
-      end
-
-      return [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank?
-      return [ nil, plaid_security ] if plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
-
-      operating_mic = plaid_security.market_identifier_code
-
-      # Find any matching security
-      security = Security.find_or_create_by!(
-        ticker: plaid_security.ticker_symbol&.upcase,
-        exchange_operating_mic: operating_mic&.upcase
-      )
-
-      [ security, plaid_security ]
-    end
-end

app/models/plaid_item.rb

@@ -1,5 +1,5 @@
 class PlaidItem < ApplicationRecord
-  include Syncable
+  include Syncable, Provided
 
   enum :plaid_region, { us: "us", eu: "eu" }
   enum :status, { good: "good", requires_update: "requires_update" }, default: :good
@@ -43,10 +43,6 @@ class PlaidItem < ApplicationRecord
     end
   end
 
-  def build_category_alias_matcher(user_categories)
-    Provider::Plaid::CategoryAliasMatcher.new(user_categories)
-  end
-
   def destroy_later
     update!(scheduled_for_deletion: true)
     DestroyJob.perform_later(self)
@@ -60,41 +56,70 @@ class PlaidItem < ApplicationRecord
         .exists?
   end
 
-  def auto_match_categories!
-    if family.categories.none?
-      family.categories.bootstrap!
-    end
+  def import_latest_plaid_data
+    PlaidItem::Importer.new(self, plaid_provider: plaid_provider).import
+  end
 
-    alias_matcher = build_category_alias_matcher(family.categories)
-
-    accounts.each do |account|
-      matchable_transactions = account.transactions
-                                      .where(category_id: nil)
-                                      .where.not(plaid_category: nil)
-                                      .enrichable(:category_id)
-
-      matchable_transactions.each do |transaction|
-        category = alias_matcher.match(transaction.plaid_category_detailed)
-
-        if category.present?
-          # Matcher could either return a string or a Category object
-          user_category = if category.is_a?(String)
-            family.categories.find_or_create_by!(name: category)
-          else
-            category
-          end
-
-          transaction.enrich_attribute(:category_id, user_category.id, source: "plaid")
-        end
-      end
+  # Reads the fetched data and updates internal domain objects
+  # Generally, this should only be called within a "sync", but can be called
+  # manually to "re-sync" the already fetched data
+  def process_accounts
+    plaid_accounts.each do |plaid_account|
+      PlaidAccount::Processor.new(plaid_account).process
     end
   end
 
+  # Once all the data is fetched, we can schedule account syncs to calculate historical balances
+  def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil)
+    accounts.each do |account|
+      account.sync_later(
+        parent_sync: parent_sync,
+        window_start_date: window_start_date,
+        window_end_date: window_end_date
+      )
+    end
+  end
+
+  # Saves the raw data fetched from Plaid API for this item
+  def upsert_plaid_snapshot!(item_snapshot)
+    assign_attributes(
+      available_products: item_snapshot.available_products,
+      billed_products: item_snapshot.billed_products,
+      raw_payload: item_snapshot,
+    )
+
+    save!
+  end
+
+  # Saves the raw data fetched from Plaid API for this item's institution
+  def upsert_plaid_institution_snapshot!(institution_snapshot)
+    assign_attributes(
+      institution_id: institution_snapshot.institution_id,
+      institution_url: institution_snapshot.url,
+      institution_color: institution_snapshot.primary_color,
+      raw_institution_payload: institution_snapshot
+    )
+
+    save!
+  end
+
+  def supports_product?(product)
+    supported_products.include?(product)
+  end
+
   private
+    # Silently swallow and report error so that we don't block the user from deleting the item
     def remove_plaid_item
       plaid_provider.remove_item(access_token)
     rescue StandardError => e
-      Rails.logger.warn("Failed to remove Plaid item #{id}: #{e.message}")
+      Sentry.capture_exception(e)
+    end
+
+    # Plaid returns mutually exclusive arrays here.  If the item has made a request for a product,
+    # it is put in the billed_products array.  If it is supported, but not yet used, it goes in the
+    # available_products array.
+    def supported_products
+      available_products + billed_products
     end
 
     class PlaidConnectionLostError < StandardError; end

app/models/plaid_item/accounts_snapshot.rb

@@ -0,0 +1,79 @@
+# All Plaid data is fetched at the item-level.  This class is a simple wrapper that
+# providers a convenience method, get_account_data which scopes the item-level payload
+# to each Plaid Account
+class PlaidItem::AccountsSnapshot
+  def initialize(plaid_item, plaid_provider:)
+    @plaid_item = plaid_item
+    @plaid_provider = plaid_provider
+  end
+
+  def accounts
+    @accounts ||= plaid_provider.get_item_accounts(plaid_item.access_token).accounts
+  end
+
+  def get_account_data(account_id)
+    AccountData.new(
+      account_data: accounts.find { |a| a.account_id == account_id },
+      transactions_data: account_scoped_transactions_data(account_id),
+      investments_data: account_scoped_investments_data(account_id),
+      liabilities_data: account_scoped_liabilities_data(account_id)
+    )
+  end
+
+  private
+    attr_reader :plaid_item, :plaid_provider
+
+    TransactionsData = Data.define(:added, :modified, :removed)
+    LiabilitiesData = Data.define(:credit, :mortgage, :student)
+    InvestmentsData = Data.define(:transactions, :holdings, :securities)
+    AccountData = Data.define(:account_data, :transactions_data, :investments_data, :liabilities_data)
+
+    def account_scoped_transactions_data(account_id)
+      return nil unless transactions_data
+
+      TransactionsData.new(
+        added: transactions_data.added.select { |t| t.account_id == account_id },
+        modified: transactions_data.modified.select { |t| t.account_id == account_id },
+        removed: transactions_data.removed.select { |t| t.account_id == account_id }
+      )
+    end
+
+    def account_scoped_investments_data(account_id)
+      return nil unless investments_data
+
+      transactions = investments_data.transactions.select { |t| t.account_id == account_id }
+      holdings = investments_data.holdings.select { |h| h.account_id == account_id }
+      securities = transactions.count > 0 && holdings.count > 0 ? investments_data.securities : []
+
+      InvestmentsData.new(
+        transactions: transactions,
+        holdings: holdings,
+        securities: securities
+      )
+    end
+
+    def account_scoped_liabilities_data(account_id)
+      return nil unless liabilities_data
+
+      LiabilitiesData.new(
+        credit: liabilities_data.credit&.find { |c| c.account_id == account_id },
+        mortgage: liabilities_data.mortgage&.find { |m| m.account_id == account_id },
+        student: liabilities_data.student&.find { |s| s.account_id == account_id }
+      )
+    end
+
+    def transactions_data
+      return nil unless plaid_item.supports_product?("transactions")
+      @transactions_data ||= plaid_provider.get_transactions(plaid_item.access_token)
+    end
+
+    def investments_data
+      return nil unless plaid_item.supports_product?("investments")
+      @investments_data ||= plaid_provider.get_item_investments(plaid_item.access_token)
+    end
+
+    def liabilities_data
+      return nil unless plaid_item.supports_product?("liabilities")
+      @liabilities_data ||= plaid_provider.get_item_liabilities(plaid_item.access_token)
+    end
+end

app/models/plaid_item/importer.rb

@@ -0,0 +1,53 @@
+class PlaidItem::Importer
+  def initialize(plaid_item, plaid_provider:)
+    @plaid_item = plaid_item
+    @plaid_provider = plaid_provider
+  end
+
+  def import
+    fetch_and_import_item_data
+    fetch_and_import_accounts_data
+  rescue Plaid::ApiError => e
+    handle_plaid_error(e)
+  end
+
+  private
+    attr_reader :plaid_item, :plaid_provider
+
+    # All errors that should halt the import should be re-raised after handling
+    # These errors will propagate up to the Sync record and mark it as failed.
+    def handle_plaid_error(error)
+      error_body = JSON.parse(error.response_body)
+
+      case error_body["error_code"]
+      when "ITEM_LOGIN_REQUIRED"
+        plaid_item.update!(status: :requires_update)
+        raise error
+      else
+        raise error
+      end
+    end
+
+    def fetch_and_import_item_data
+      item_data = plaid_provider.get_item(plaid_item.access_token).item
+      institution_data = plaid_provider.get_institution(item_data.institution_id).institution
+
+      plaid_item.upsert_plaid_snapshot!(item_data)
+      plaid_item.upsert_plaid_institution_snapshot!(institution_data)
+    end
+
+    def fetch_and_import_accounts_data
+      snapshot = PlaidItem::AccountsSnapshot.new(plaid_item, plaid_provider: plaid_provider)
+
+      snapshot.accounts.each do |raw_account|
+        plaid_account = plaid_item.plaid_accounts.find_or_initialize_by(
+          plaid_id: raw_account.account_id
+        )
+
+        PlaidAccount::Importer.new(
+          plaid_account,
+          account_snapshot: snapshot.get_account_data(raw_account.account_id)
+        ).import
+      end
+    end
+end

app/models/plaid_item/provided.rb

@@ -0,0 +1,7 @@
+module PlaidItem::Provided
+  extend ActiveSupport::Concern
+
+  def plaid_provider
+    @plaid_provider ||= Provider::Registry.plaid_provider_for_region(self.plaid_region)
+  end
+end

app/models/plaid_item/syncer.rb

@@ -6,144 +6,21 @@ class PlaidItem::Syncer
   end
 
   def perform_sync(sync)
-    begin
-      Rails.logger.info("Fetching and loading Plaid data")
-      fetch_and_load_plaid_data
-      plaid_item.update!(status: :good) if plaid_item.requires_update?
+    # Loads item metadata, accounts, transactions, and other data to our DB
+    plaid_item.import_latest_plaid_data
 
-      plaid_item.accounts.each do |account|
-        account.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date)
-      end
+    # Processes the raw Plaid data and updates internal domain objects
+    plaid_item.process_accounts
 
-      Rails.logger.info("Plaid data fetched and loaded")
-    rescue Plaid::ApiError => e
-      handle_plaid_error(e)
-      raise e
-    end
+    # All data is synced, so we can now run an account sync to calculate historical balances and more
+    plaid_item.schedule_account_syncs(
+      parent_sync: sync,
+      window_start_date: sync.window_start_date,
+      window_end_date: sync.window_end_date
+    )
   end
 
   def perform_post_sync
-    plaid_item.auto_match_categories!
+    # no-op
   end
-
-  private
-    def plaid
-      plaid_item.plaid_region == "eu" ? plaid_eu : plaid_us
-    end
-
-    def plaid_eu
-      @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu)
-    end
-
-    def plaid_us
-      @plaid_us ||= Provider::Registry.get_provider(:plaid_us)
-    end
-
-    def safe_fetch_plaid_data(method)
-      begin
-        plaid.send(method, plaid_item)
-      rescue Plaid::ApiError => e
-        Rails.logger.warn("Error fetching #{method} for item #{plaid_item.id}: #{e.message}")
-        nil
-      end
-    end
-
-    def handle_plaid_error(error)
-      error_body = JSON.parse(error.response_body)
-
-      if error_body["error_code"] == "ITEM_LOGIN_REQUIRED"
-        plaid_item.update!(status: :requires_update)
-      end
-    end
-
-    def fetch_and_load_plaid_data
-      data = {}
-
-      # Log what we're about to fetch
-      Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)"
-
-      item = plaid.get_item(plaid_item.access_token).item
-      plaid_item.update!(available_products: item.available_products, billed_products: item.billed_products)
-
-      # Institution details
-      if item.institution_id.present?
-        begin
-          Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}"
-          institution = plaid.get_institution(item.institution_id)
-          plaid_item.update!(
-            institution_id: item.institution_id,
-            institution_url: institution.institution.url,
-            institution_color: institution.institution.primary_color
-          )
-        rescue Plaid::ApiError => e
-          Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}"
-        end
-      end
-
-      # Accounts
-      fetched_accounts = plaid.get_item_accounts(plaid_item).accounts
-      data[:accounts] = fetched_accounts || []
-      Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})"
-
-      internal_plaid_accounts = fetched_accounts.map do |account|
-        internal_plaid_account = plaid_item.plaid_accounts.find_or_create_from_plaid_data!(account, plaid_item.family)
-        internal_plaid_account.sync_account_data!(account)
-        internal_plaid_account
-      end
-
-      # Transactions
-      fetched_transactions = safe_fetch_plaid_data(:get_item_transactions)
-      data[:transactions] = fetched_transactions || []
-
-      if fetched_transactions
-        Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})"
-        PlaidItem.transaction do
-          internal_plaid_accounts.each do |internal_plaid_account|
-            added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id }
-            modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id }
-            removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id }
-
-            internal_plaid_account.sync_transactions!(added:, modified:, removed:)
-          end
-
-          plaid_item.update!(next_cursor: fetched_transactions.cursor)
-        end
-      end
-
-      # Investments
-      fetched_investments = safe_fetch_plaid_data(:get_item_investments)
-      data[:investments] = fetched_investments || []
-
-      if fetched_investments
-        Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})"
-        PlaidItem.transaction do
-          internal_plaid_accounts.each do |internal_plaid_account|
-            transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id }
-            holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id }
-            securities = fetched_investments.securities
-
-            internal_plaid_account.sync_investments!(transactions:, holdings:, securities:)
-          end
-        end
-      end
-
-      # Liabilities
-      fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities)
-      data[:liabilities] = fetched_liabilities || []
-
-      if fetched_liabilities
-        Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})"
-        PlaidItem.transaction do
-          internal_plaid_accounts.each do |internal_plaid_account|
-            credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id }
-            mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id }
-            student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id }
-
-            internal_plaid_account.sync_credit_data!(credit) if credit
-            internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage
-            internal_plaid_account.sync_student_loan_data!(student) if student
-          end
-        end
-      end
-    end
 end

app/models/provider/plaid.rb

@@ -106,13 +106,13 @@ class Provider::Plaid
     client.item_remove(request)
   end
 
-  def get_item_accounts(item)
-    request = Plaid::AccountsGetRequest.new(access_token: item.access_token)
+  def get_item_accounts(access_token)
+    request = Plaid::AccountsGetRequest.new(access_token: access_token)
     client.accounts_get(request)
   end
 
-  def get_item_transactions(item)
-    cursor = item.next_cursor
+  def get_transactions(access_token, next_cursor: nil)
+    cursor = next_cursor
     added = []
     modified = []
     removed = []
@@ -120,7 +120,7 @@ class Provider::Plaid
 
     while has_more
       request = Plaid::TransactionsSyncRequest.new(
-        access_token: item.access_token,
+        access_token: access_token,
         cursor: cursor,
         options: {
           include_original_description: true
@@ -139,18 +139,18 @@ class Provider::Plaid
     TransactionSyncResponse.new(added:, modified:, removed:, cursor:)
   end
 
-  def get_item_investments(item, start_date: nil, end_date: Date.current)
+  def get_item_investments(access_token, start_date: nil, end_date: Date.current)
     start_date = start_date || MAX_HISTORY_DAYS.days.ago.to_date
-    holdings, holding_securities = get_item_holdings(item)
-    transactions, transaction_securities = get_item_investment_transactions(item, start_date:, end_date:)
+    holdings, holding_securities = get_item_holdings(access_token: access_token)
+    transactions, transaction_securities = get_item_investment_transactions(access_token: access_token, start_date:, end_date:)
 
     merged_securities = ((holding_securities || []) + (transaction_securities || [])).uniq { |s| s.security_id }
 
     InvestmentsResponse.new(holdings:, transactions:, securities: merged_securities)
   end
 
-  def get_item_liabilities(item)
-    request = Plaid::LiabilitiesGetRequest.new({ access_token: item.access_token })
+  def get_item_liabilities(access_token)
+    request = Plaid::LiabilitiesGetRequest.new({ access_token: access_token })
     response = client.liabilities_get(request)
     response.liabilities
   end
@@ -170,21 +170,21 @@ class Provider::Plaid
     TransactionSyncResponse = Struct.new :added, :modified, :removed, :cursor, keyword_init: true
     InvestmentsResponse = Struct.new :holdings, :transactions, :securities, keyword_init: true
 
-    def get_item_holdings(item)
-      request = Plaid::InvestmentsHoldingsGetRequest.new({ access_token: item.access_token })
+    def get_item_holdings(access_token:)
+      request = Plaid::InvestmentsHoldingsGetRequest.new({ access_token: access_token })
       response = client.investments_holdings_get(request)
 
       [ response.holdings, response.securities ]
     end
 
-    def get_item_investment_transactions(item, start_date:, end_date:)
+    def get_item_investment_transactions(access_token:, start_date:, end_date:)
       transactions = []
       securities = []
       offset = 0
 
       loop do
         request = Plaid::InvestmentsTransactionsGetRequest.new(
-          access_token: item.access_token,
+          access_token: access_token,
           start_date: start_date.to_s,
           end_date: end_date.to_s,
           options: { offset: offset }

app/models/provider/plaid_sandbox.rb

@@ -3,6 +3,19 @@ class Provider::PlaidSandbox < Provider::Plaid
 
   def initialize
     @client = create_client
+    @region = :us
+  end
+
+  def create_public_token(username: nil)
+    client.sandbox_public_token_create(
+      Plaid::SandboxPublicTokenCreateRequest.new(
+        institution_id: "ins_109508", # "First Platypus Bank" (Plaid's sandbox institution that works with all products)
+        initial_products: [ "transactions", "investments", "liabilities" ],
+        options: {
+          override_username: username || "custom_test"
+        }
+      )
+    ).public_token
   end
 
   def fire_webhook(item, type: "TRANSACTIONS", code: "SYNC_UPDATES_AVAILABLE")

app/models/provider/registry.rb

@@ -18,6 +18,10 @@ class Provider::Registry
       raise Error.new("Provider '#{name}' not found in registry")
     end
 
+    def plaid_provider_for_region(region)
+      region.to_sym == :us ? plaid_us : plaid_eu
+    end
+
     private
       def stripe
         secret_key = ENV["STRIPE_SECRET_KEY"]