diff --git a/app/models/provider/plaid/category_alias_matcher.rb b/app/models/plaid_account/transactions/category_matcher.rb similarity index 80% rename from app/models/provider/plaid/category_alias_matcher.rb rename to app/models/plaid_account/transactions/category_matcher.rb index 41ada3a6..b3583b54 100644 --- a/app/models/provider/plaid/category_alias_matcher.rb +++ b/app/models/plaid_account/transactions/category_matcher.rb @@ -1,19 +1,7 @@ -# The purpose of this matcher is to auto-match Plaid categories to -# known internal user categories. Since we allow users to define their own -# categories we cannot directly assign Plaid categories as this would overwrite -# user data and create a confusing experience. -# -# Automated category matching in the Maybe app has a hierarchy: -# 1. Naive string matching via CategoryAliasMatcher -# 2. Rules-based matching set by user -# 3. AI-powered matching (also enabled by user via rules) -# -# This class is simply a FAST and CHEAP way to match categories that are high confidence. -# Edge cases will be handled by user-defined rules. -class Provider::Plaid::CategoryAliasMatcher - include Provider::Plaid::CategoryTaxonomy +class PlaidAccount::Transactions::CategoryMatcher + include PlaidAccount::Transactions::CategoryTaxonomy - def initialize(user_categories) + def initialize(user_categories = []) @user_categories = user_categories end diff --git a/app/models/provider/plaid/category_taxonomy.rb b/app/models/plaid_account/transactions/category_taxonomy.rb similarity index 99% rename from app/models/provider/plaid/category_taxonomy.rb rename to app/models/plaid_account/transactions/category_taxonomy.rb index 9766c724..84b4b8c1 100644 --- a/app/models/provider/plaid/category_taxonomy.rb +++ b/app/models/plaid_account/transactions/category_taxonomy.rb @@ -1,5 +1,5 @@ # https://plaid.com/documents/transactions-personal-finance-category-taxonomy.csv -module Provider::Plaid::CategoryTaxonomy +module PlaidAccount::Transactions::CategoryTaxonomy CATEGORIES_MAP = { income: { classification: :income, diff --git a/app/models/plaid_account/transactions/processor.rb b/app/models/plaid_account/transactions/processor.rb index 92b5214f..155b12fe 100644 --- a/app/models/plaid_account/transactions/processor.rb +++ b/app/models/plaid_account/transactions/processor.rb @@ -4,12 +4,18 @@ class PlaidAccount::Transactions::Processor end def process - PlaidAccount.transaction do - modified_transactions.each do |transaction| - PlaidEntry::TransactionProcessor.new(transaction, plaid_account: plaid_account).process - end + # Each entry is processed inside a transaction, but to avoid locking up the DB when + # there are hundreds or thousands of transactions, we process them individually. + modified_transactions.find_each do |transaction| + PlaidEntry::Processor.new( + transaction, + plaid_account: plaid_account, + category_matcher: category_matcher + ).process + end - removed_transactions.each do |transaction| + PlaidAccount.transaction do + removed_transactions.find_each do |transaction| remove_plaid_transaction(transaction) end end @@ -18,6 +24,20 @@ class PlaidAccount::Transactions::Processor private attr_reader :plaid_account + def category_matcher + @category_matcher ||= PlaidAccount::Transactions::CategoryMatcher.new(family_categories) + end + + def family_categories + @family_categories ||= begin + if account.family.categories.none? + account.family.categories.bootstrap! + end + + account.family.categories + end + end + def account plaid_account.account end diff --git a/app/models/plaid_entry/transaction_processor.rb b/app/models/plaid_entry/processor.rb similarity index 52% rename from app/models/plaid_entry/transaction_processor.rb rename to app/models/plaid_entry/processor.rb index 059b9640..182e382d 100644 --- a/app/models/plaid_entry/transaction_processor.rb +++ b/app/models/plaid_entry/processor.rb @@ -1,45 +1,53 @@ -class PlaidEntry::TransactionProcessor +class PlaidEntry::Processor # plaid_transaction is the raw hash fetched from Plaid API and converted to JSONB - def initialize(plaid_transaction, plaid_account:) + def initialize(plaid_transaction, plaid_account:, category_matcher:) @plaid_transaction = plaid_transaction @plaid_account = plaid_account + @category_matcher = category_matcher end def process - entry = account.entries.find_or_initialize_by(plaid_id: plaid_id) do |e| - e.entryable = Transaction.new - end + PlaidAccount.transaction do + entry = account.entries.find_or_initialize_by(plaid_id: plaid_id) do |e| + e.entryable = Transaction.new + end - entry.enrich_attribute( - :name, - name, - source: "plaid" - ) + entry.assign_attributes( + amount: amount, + currency: currency, + date: date + ) - entry.assign_attributes( - amount: amount, - currency: currency, - date: date - ) - - if merchant - entry.transaction.enrich_attribute( - :merchant_id, - merchant.id, + entry.enrich_attribute( + :name, + name, source: "plaid" ) + + if detailed_category + matched_category = category_matcher.match(detailed_category) + + if matched_category + entry.transaction.enrich_attribute( + :category_id, + matched_category.id, + source: "plaid" + ) + end + end + + if merchant + entry.transaction.enrich_attribute( + :merchant_id, + merchant.id, + source: "plaid" + ) + end end - - entry.transaction.assign_attributes( - plaid_category: primary_category, - plaid_category_detailed: detailed_category, - ) - - entry.save! end private - attr_reader :plaid_transaction, :plaid_account + attr_reader :plaid_transaction, :plaid_account, :category_matcher def account plaid_account.account @@ -65,12 +73,8 @@ class PlaidEntry::TransactionProcessor plaid_transaction["date"] end - def primary_category - plaid_transaction["personal_finance_category"]["primary"] - end - def detailed_category - plaid_transaction["personal_finance_category"]["detailed"] + plaid_transaction.dig("personal_finance_category", "detailed") end def merchant diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index bf4844ba..cb524fd1 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -43,10 +43,6 @@ class PlaidItem < ApplicationRecord end end - def build_category_alias_matcher(user_categories) - Provider::Plaid::CategoryAliasMatcher.new(user_categories) - end - def destroy_later update!(scheduled_for_deletion: true) DestroyJob.perform_later(self) @@ -107,35 +103,35 @@ class PlaidItem < ApplicationRecord save! end - def auto_match_categories! - if family.categories.none? - family.categories.bootstrap! - end + # def auto_match_categories! + # if family.categories.none? + # family.categories.bootstrap! + # end - alias_matcher = build_category_alias_matcher(family.categories) + # alias_matcher = build_category_alias_matcher(family.categories) - accounts.each do |account| - matchable_transactions = account.transactions - .where(category_id: nil) - .where.not(plaid_category: nil) - .enrichable(:category_id) + # accounts.each do |account| + # matchable_transactions = account.transactions + # .where(category_id: nil) + # .where.not(plaid_category: nil) + # .enrichable(:category_id) - matchable_transactions.each do |transaction| - category = alias_matcher.match(transaction.plaid_category_detailed) + # matchable_transactions.each do |transaction| + # category = alias_matcher.match(transaction.plaid_category_detailed) - if category.present? - # Matcher could either return a string or a Category object - user_category = if category.is_a?(String) - family.categories.find_or_create_by!(name: category) - else - category - end + # if category.present? + # # Matcher could either return a string or a Category object + # user_category = if category.is_a?(String) + # family.categories.find_or_create_by!(name: category) + # else + # category + # end - transaction.enrich_attribute(:category_id, user_category.id, source: "plaid") - end - end - end - end + # transaction.enrich_attribute(:category_id, user_category.id, source: "plaid") + # end + # end + # end + # end def supports_product?(product) supported_products.include?(product) diff --git a/db/migrate/20250523131455_add_raw_payloads_to_plaid_accounts.rb b/db/migrate/20250523131455_add_raw_payloads_to_plaid_accounts.rb index b9bcf884..fbff245f 100644 --- a/db/migrate/20250523131455_add_raw_payloads_to_plaid_accounts.rb +++ b/db/migrate/20250523131455_add_raw_payloads_to_plaid_accounts.rb @@ -16,5 +16,9 @@ class AddRawPayloadsToPlaidAccounts < ActiveRecord::Migration[7.2] change_column_null :plaid_accounts, :currency, false change_column_null :plaid_accounts, :name, false add_index :plaid_accounts, :plaid_id, unique: true + + # No longer need to store on transaction model because it is stored in raw_transactions_payload + remove_column :transactions, :plaid_category + remove_column :transactions, :plaid_category_detailed end end diff --git a/db/schema.rb b/db/schema.rb index 9cb2c7f4..b5f41eab 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -645,8 +645,6 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_23_131455) do t.uuid "category_id" t.uuid "merchant_id" t.jsonb "locked_attributes", default: {} - t.string "plaid_category" - t.string "plaid_category_detailed" t.index ["category_id"], name: "index_transactions_on_category_id" t.index ["merchant_id"], name: "index_transactions_on_merchant_id" end diff --git a/test/models/provider/plaid/category_alias_matcher_test.rb b/test/models/plaid_account/transactions/category_matcher_test.rb similarity index 98% rename from test/models/provider/plaid/category_alias_matcher_test.rb rename to test/models/plaid_account/transactions/category_matcher_test.rb index 11881dea..35bcf8fe 100644 --- a/test/models/provider/plaid/category_alias_matcher_test.rb +++ b/test/models/plaid_account/transactions/category_matcher_test.rb @@ -1,6 +1,6 @@ require "test_helper" -class Provider::Plaid::CategoryAliasMatcherTest < ActiveSupport::TestCase +class PlaidAccount::Transactions::CategoryMatcherTest < ActiveSupport::TestCase setup do @family = families(:empty) @@ -32,7 +32,7 @@ class Provider::Plaid::CategoryAliasMatcherTest < ActiveSupport::TestCase @giving = @family.categories.create!(name: "Giving") - @matcher = Provider::Plaid::CategoryAliasMatcher.new(@family.categories) + @matcher = PlaidAccount::Transactions::CategoryMatcher.new(@family.categories) end test "matches expense categories" do diff --git a/test/models/plaid_entry/processor_test.rb b/test/models/plaid_entry/processor_test.rb new file mode 100644 index 00000000..bce448b0 --- /dev/null +++ b/test/models/plaid_entry/processor_test.rb @@ -0,0 +1,91 @@ +require "test_helper" + +class PlaidEntry::ProcessorTest < ActiveSupport::TestCase + setup do + @plaid_account = plaid_accounts(:one) + @category_matcher = mock("PlaidAccount::Transactions::CategoryMatcher") + end + + test "creates new entry transaction" do + plaid_transaction = { + "transaction_id" => "123", + "merchant_name" => "Amazon", # this is used for merchant and entry name + "amount" => 100, + "date" => Date.current, + "iso_currency_code" => "USD", + "personal_finance_category" => { + "detailed" => "Food" + }, + "merchant_entity_id" => "123" + } + + @category_matcher.expects(:match).with("Food").returns(categories(:food_and_drink)) + + processor = PlaidEntry::Processor.new( + plaid_transaction, + plaid_account: @plaid_account, + category_matcher: @category_matcher + ) + + assert_difference [ "Entry.count", "Transaction.count", "ProviderMerchant.count" ], 1 do + processor.process + end + + entry = Entry.order(created_at: :desc).first + + assert_equal 100, entry.amount + assert_equal "USD", entry.currency + assert_equal Date.current, entry.date + assert_equal "Amazon", entry.name + assert_equal categories(:food_and_drink).id, entry.transaction.category_id + + provider_merchant = ProviderMerchant.order(created_at: :desc).first + + assert_equal "Amazon", provider_merchant.name + end + + test "updates existing entry transaction" do + existing_plaid_id = "existing_plaid_id" + + plaid_transaction = { + "transaction_id" => existing_plaid_id, + "merchant_name" => "Amazon", # this is used for merchant and entry name + "amount" => 200, # Changed amount will be updated + "date" => 1.day.ago.to_date, # Changed date will be updated + "iso_currency_code" => "USD", + "personal_finance_category" => { + "detailed" => "Food" + } + } + + @category_matcher.expects(:match).with("Food").returns(categories(:food_and_drink)) + + # Create an existing entry + @plaid_account.account.entries.create!( + plaid_id: existing_plaid_id, + amount: 100, + currency: "USD", + date: Date.current, + name: "Amazon", + entryable: Transaction.new + ) + + processor = PlaidEntry::Processor.new( + plaid_transaction, + plaid_account: @plaid_account, + category_matcher: @category_matcher + ) + + assert_no_difference [ "Entry.count", "Transaction.count", "ProviderMerchant.count" ] do + processor.process + end + + entry = Entry.order(created_at: :desc).first + + assert_equal 200, entry.amount + assert_equal "USD", entry.currency + assert_equal 1.day.ago.to_date, entry.date + assert_equal "Amazon", entry.name + assert_equal categories(:food_and_drink).id, entry.transaction.category_id + end +end