Transaction processor test cases

app/models/plaid_account/transactions/category_matcher.rb

@@ -1,19 +1,7 @@
-# The purpose of this matcher is to auto-match Plaid categories to
-# known internal user categories.  Since we allow users to define their own
-# categories we cannot directly assign Plaid categories as this would overwrite
-# user data and create a confusing experience.
-#
-# Automated category matching in the Maybe app has a hierarchy:
-# 1. Naive string matching via CategoryAliasMatcher
-# 2. Rules-based matching set by user
-# 3. AI-powered matching (also enabled by user via rules)
-#
-# This class is simply a FAST and CHEAP way to match categories that are high confidence.
-# Edge cases will be handled by user-defined rules.
-class Provider::Plaid::CategoryAliasMatcher
-  include Provider::Plaid::CategoryTaxonomy
+class PlaidAccount::Transactions::CategoryMatcher
+  include PlaidAccount::Transactions::CategoryTaxonomy
 
-  def initialize(user_categories)
+  def initialize(user_categories = [])
     @user_categories = user_categories
   end
 

app/models/plaid_account/transactions/category_taxonomy.rb

@@ -1,5 +1,5 @@
 # https://plaid.com/documents/transactions-personal-finance-category-taxonomy.csv
-module Provider::Plaid::CategoryTaxonomy
+module PlaidAccount::Transactions::CategoryTaxonomy
   CATEGORIES_MAP = {
     income: {
       classification: :income,

app/models/plaid_account/transactions/processor.rb

@@ -4,12 +4,18 @@ class PlaidAccount::Transactions::Processor
   end
 
   def process
-    PlaidAccount.transaction do
-      modified_transactions.each do |transaction|
-        PlaidEntry::TransactionProcessor.new(transaction, plaid_account: plaid_account).process
-      end
+    # Each entry is processed inside a transaction, but to avoid locking up the DB when
+    # there are hundreds or thousands of transactions, we process them individually.
+    modified_transactions.find_each do |transaction|
+      PlaidEntry::Processor.new(
+        transaction,
+        plaid_account: plaid_account,
+        category_matcher: category_matcher
+      ).process
+    end
 
-      removed_transactions.each do |transaction|
+    PlaidAccount.transaction do
+      removed_transactions.find_each do |transaction|
         remove_plaid_transaction(transaction)
       end
     end
@@ -18,6 +24,20 @@ class PlaidAccount::Transactions::Processor
   private
     attr_reader :plaid_account
 
+    def category_matcher
+      @category_matcher ||= PlaidAccount::Transactions::CategoryMatcher.new(family_categories)
+    end
+
+    def family_categories
+      @family_categories ||= begin
+        if account.family.categories.none?
+          account.family.categories.bootstrap!
+        end
+
+        account.family.categories
+      end
+    end
+
     def account
       plaid_account.account
     end

app/models/plaid_entry/processor.rb

@@ -1,45 +1,53 @@
-class PlaidEntry::TransactionProcessor
+class PlaidEntry::Processor
   # plaid_transaction is the raw hash fetched from Plaid API and converted to JSONB
-  def initialize(plaid_transaction, plaid_account:)
+  def initialize(plaid_transaction, plaid_account:, category_matcher:)
     @plaid_transaction = plaid_transaction
     @plaid_account = plaid_account
+    @category_matcher = category_matcher
   end
 
   def process
-    entry = account.entries.find_or_initialize_by(plaid_id: plaid_id) do |e|
-      e.entryable = Transaction.new
-    end
+    PlaidAccount.transaction do
+      entry = account.entries.find_or_initialize_by(plaid_id: plaid_id) do |e|
+        e.entryable = Transaction.new
+      end
 
-    entry.enrich_attribute(
-      :name,
-      name,
-      source: "plaid"
-    )
+      entry.assign_attributes(
+        amount: amount,
+        currency: currency,
+        date: date
+      )
 
-    entry.assign_attributes(
-      amount: amount,
-      currency: currency,
-      date: date
-    )
-
-    if merchant
-      entry.transaction.enrich_attribute(
-        :merchant_id,
-        merchant.id,
+      entry.enrich_attribute(
+        :name,
+        name,
         source: "plaid"
       )
+
+      if detailed_category
+        matched_category = category_matcher.match(detailed_category)
+
+        if matched_category
+          entry.transaction.enrich_attribute(
+            :category_id,
+            matched_category.id,
+            source: "plaid"
+          )
+        end
+      end
+
+      if merchant
+        entry.transaction.enrich_attribute(
+          :merchant_id,
+          merchant.id,
+          source: "plaid"
+        )
+      end
     end
-
-    entry.transaction.assign_attributes(
-      plaid_category: primary_category,
-      plaid_category_detailed: detailed_category,
-    )
-
-    entry.save!
   end
 
   private
-    attr_reader :plaid_transaction, :plaid_account
+    attr_reader :plaid_transaction, :plaid_account, :category_matcher
 
     def account
       plaid_account.account
@@ -65,12 +73,8 @@ class PlaidEntry::TransactionProcessor
       plaid_transaction["date"]
     end
 
-    def primary_category
-      plaid_transaction["personal_finance_category"]["primary"]
-    end
-
     def detailed_category
-      plaid_transaction["personal_finance_category"]["detailed"]
+      plaid_transaction.dig("personal_finance_category", "detailed")
     end
 
     def merchant

app/models/plaid_item.rb

@@ -43,10 +43,6 @@ class PlaidItem < ApplicationRecord
     end
   end
 
-  def build_category_alias_matcher(user_categories)
-    Provider::Plaid::CategoryAliasMatcher.new(user_categories)
-  end
-
   def destroy_later
     update!(scheduled_for_deletion: true)
     DestroyJob.perform_later(self)
@@ -107,35 +103,35 @@ class PlaidItem < ApplicationRecord
     save!
   end
 
-  def auto_match_categories!
-    if family.categories.none?
-      family.categories.bootstrap!
-    end
+  # def auto_match_categories!
+  #   if family.categories.none?
+  #     family.categories.bootstrap!
+  #   end
 
-    alias_matcher = build_category_alias_matcher(family.categories)
+  #   alias_matcher = build_category_alias_matcher(family.categories)
 
-    accounts.each do |account|
-      matchable_transactions = account.transactions
-                                      .where(category_id: nil)
-                                      .where.not(plaid_category: nil)
-                                      .enrichable(:category_id)
+  #   accounts.each do |account|
+  #     matchable_transactions = account.transactions
+  #                                     .where(category_id: nil)
+  #                                     .where.not(plaid_category: nil)
+  #                                     .enrichable(:category_id)
 
-      matchable_transactions.each do |transaction|
-        category = alias_matcher.match(transaction.plaid_category_detailed)
+  #     matchable_transactions.each do |transaction|
+  #       category = alias_matcher.match(transaction.plaid_category_detailed)
 
-        if category.present?
-          # Matcher could either return a string or a Category object
-          user_category = if category.is_a?(String)
-            family.categories.find_or_create_by!(name: category)
-          else
-            category
-          end
+  #       if category.present?
+  #         # Matcher could either return a string or a Category object
+  #         user_category = if category.is_a?(String)
+  #           family.categories.find_or_create_by!(name: category)
+  #         else
+  #           category
+  #         end
 
-          transaction.enrich_attribute(:category_id, user_category.id, source: "plaid")
-        end
-      end
-    end
-  end
+  #         transaction.enrich_attribute(:category_id, user_category.id, source: "plaid")
+  #       end
+  #     end
+  #   end
+  # end
 
   def supports_product?(product)
     supported_products.include?(product)

db/migrate/20250523131455_add_raw_payloads_to_plaid_accounts.rb

@@ -16,5 +16,9 @@ class AddRawPayloadsToPlaidAccounts < ActiveRecord::Migration[7.2]
     change_column_null :plaid_accounts, :currency, false
     change_column_null :plaid_accounts, :name, false
     add_index :plaid_accounts, :plaid_id, unique: true
+
+    # No longer need to store on transaction model because it is stored in raw_transactions_payload
+    remove_column :transactions, :plaid_category
+    remove_column :transactions, :plaid_category_detailed
   end
 end

db/schema.rb

@@ -645,8 +645,6 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do
     t.uuid "category_id"
     t.uuid "merchant_id"
     t.jsonb "locked_attributes", default: {}
-    t.string "plaid_category"
-    t.string "plaid_category_detailed"
     t.index ["category_id"], name: "index_transactions_on_category_id"
     t.index ["merchant_id"], name: "index_transactions_on_merchant_id"
   end

test/models/plaid_account/transactions/category_matcher_test.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-class Provider::Plaid::CategoryAliasMatcherTest < ActiveSupport::TestCase
+class PlaidAccount::Transactions::CategoryMatcherTest < ActiveSupport::TestCase
   setup do
     @family = families(:empty)
 
@@ -32,7 +32,7 @@ class Provider::Plaid::CategoryAliasMatcherTest < ActiveSupport::TestCase
 
     @giving = @family.categories.create!(name: "Giving")
 
-    @matcher = Provider::Plaid::CategoryAliasMatcher.new(@family.categories)
+    @matcher = PlaidAccount::Transactions::CategoryMatcher.new(@family.categories)
   end
 
   test "matches expense categories" do

test/models/plaid_entry/processor_test.rb

@@ -0,0 +1,91 @@
+require "test_helper"
+
+class PlaidEntry::ProcessorTest < ActiveSupport::TestCase
+  setup do
+    @plaid_account = plaid_accounts(:one)
+    @category_matcher = mock("PlaidAccount::Transactions::CategoryMatcher")
+  end
+
+  test "creates new entry transaction" do
+    plaid_transaction = {
+      "transaction_id" => "123",
+      "merchant_name" => "Amazon", # this is used for merchant and entry name
+      "amount" => 100,
+      "date" => Date.current,
+      "iso_currency_code" => "USD",
+      "personal_finance_category" => {
+        "detailed" => "Food"
+      },
+      "merchant_entity_id" => "123"
+    }
+
+    @category_matcher.expects(:match).with("Food").returns(categories(:food_and_drink))
+
+    processor = PlaidEntry::Processor.new(
+      plaid_transaction,
+      plaid_account: @plaid_account,
+      category_matcher: @category_matcher
+    )
+
+    assert_difference [ "Entry.count", "Transaction.count", "ProviderMerchant.count" ], 1 do
+      processor.process
+    end
+
+    entry = Entry.order(created_at: :desc).first
+
+    assert_equal 100, entry.amount
+    assert_equal "USD", entry.currency
+    assert_equal Date.current, entry.date
+    assert_equal "Amazon", entry.name
+    assert_equal categories(:food_and_drink).id, entry.transaction.category_id
+
+    provider_merchant = ProviderMerchant.order(created_at: :desc).first
+
+    assert_equal "Amazon", provider_merchant.name
+  end
+
+  test "updates existing entry transaction" do
+    existing_plaid_id = "existing_plaid_id"
+
+    plaid_transaction = {
+      "transaction_id" => existing_plaid_id,
+      "merchant_name" => "Amazon", # this is used for merchant and entry name
+      "amount" => 200, # Changed amount will be updated
+      "date" => 1.day.ago.to_date, # Changed date will be updated
+      "iso_currency_code" => "USD",
+      "personal_finance_category" => {
+        "detailed" => "Food"
+      }
+    }
+
+    @category_matcher.expects(:match).with("Food").returns(categories(:food_and_drink))
+
+    # Create an existing entry
+    @plaid_account.account.entries.create!(
+      plaid_id: existing_plaid_id,
+      amount: 100,
+      currency: "USD",
+      date: Date.current,
+      name: "Amazon",
+      entryable: Transaction.new
+    )
+
+    processor = PlaidEntry::Processor.new(
+      plaid_transaction,
+      plaid_account: @plaid_account,
+      category_matcher: @category_matcher
+    )
+
+    assert_no_difference [ "Entry.count", "Transaction.count", "ProviderMerchant.count" ] do
+      processor.process
+    end
+
+    entry = Entry.order(created_at: :desc).first
+
+    assert_equal 200, entry.amount
+    assert_equal "USD", entry.currency
+    assert_equal 1.day.ago.to_date, entry.date
+    assert_equal "Amazon", entry.name
+    assert_equal categories(:food_and_drink).id, entry.transaction.category_id
+  end
+end