Docker/Maybe
Docker/Maybe
1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-05 13:35:21 +02:00
Code Issues Releases Activity Actions

Transaction rules engine V1 (#1900)

Browse source 
* Domain model sketch

* Scaffold out rules domain

* Migrations

* Remove existing data enrichment for clean slate

* Sketch out business logic and basic tests

* Simplify rule scope building and action executions

* Get generator working again

* Basic implementation + tests

* Remove manual merchant management (rules will replace)

* Revert "Remove manual merchant management (rules will replace)"

This reverts commit 83dcbd9ff0.

* Family and Provider merchants model

* Fix brakeman warnings

* Fix notification loader

* Update notification position

* Add Rule action and condition registries

* Rule form with compound conditions and tests

* Split out notification types, add CTA type

* Rules form builder and Stimulus controller

* Clean up rule registry domain

* Clean up rules stimulus controller

* CTA message for rule when user changes transaction category

* Fix tests

* Lint updates

* Centralize notifications in Notifiable concern

* Implement category rule prompts with auto backoff and option to disable

* Fix layout bug caused by merge conflict

* Initialize rule with correct action for category CTA

* Add rule deletions, get rules working

* Complete dynamic rule form, split Stimulus controllers by resource

* Fix failing tests

* Change test password to avoid chromium conflicts

* Update integration tests

* Centralize all test password references

* Add re-apply rule action

* Rule confirm modal

* Run migrations

* Trigger rule notification after inline category updates

* Clean up rule styles

* Basic attribute locking for rules

* Apply attribute locks on user edits

* Log data enrichments, only apply rules to unlocked attributes

* Fix merge errors

* Additional merge conflict fixes

* Form UI improvements, ignore attribute locks on manual rule application

* Batch AI auto-categorization of transactions

* Auto merchant detection, ai enrichment in batches

* Fix Plaid merchant assignments

* Plaid category matching

* Cleanup 1

* Test cleanup

* Remove stale route

* Fix desktop chat UI issues

* Fix mobile nav styling issues
This commit is contained in:
Zach Gollwitzer 2025-04-18 11:39:58 -04:00 • committed by GitHub
parent 8edd7ecef0
commit 297a695d0f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
152 changed files with 4502 additions and 612 deletions
Download patch file Download diff file Expand all files Collapse all files

12
app/models/account.rb
View file

@ -1,5 +1,5 @@
class Account < ApplicationRecord
include Syncable, Monetizable, Chartable, Enrichable, Linkable, Convertible
include Syncable, Monetizable, Chartable, Linkable, Convertible, Enrichable
validates :name, :balance, :currency, presence: true
@ -83,11 +83,6 @@ class Account < ApplicationRecord
accountable.post_sync(sync)
if enrichable?
Rails.logger.info("Enriching transaction data")
enrich_data
end
unless sync.child?
family.auto_match_transfers!
end
@ -147,6 +142,11 @@ class Account < ApplicationRecord
first_entry_date - 1.day
end
def lock_saved_attributes!
super
accountable.lock_saved_attributes!
end
def first_valuation
entries.valuations.order(:date).first
end

71
app/models/account/enrichable.rb
View file

@ -1,71 +0,0 @@
module Account::Enrichable
extend ActiveSupport::Concern
def enrich_data
total_unenriched = entries.transactions
.joins("JOIN transactions at ON at.id = entries.entryable_id AND entries.entryable_type = 'Transaction'")
.where("entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
.count
if total_unenriched > 0
batch_size = 50
batches = (total_unenriched.to_f / batch_size).ceil
batches.times do |batch|
EnrichTransactionBatchJob.perform_now(self, batch_size, batch * batch_size)
# EnrichTransactionBatchJob.perform_later(self, batch_size, batch * batch_size)
end
end
end
def enrich_transaction_batch(batch_size = 50, offset = 0)
transactions_batch = enrichable_transactions.offset(offset).limit(batch_size)
Rails.logger.info("Enriching batch of #{transactions_batch.count} transactions for account #{id} (offset: #{offset})")
merchants = {}
transactions_batch.each do |transaction|
begin
info = transaction.fetch_enrichment_info
next unless info.present?
if info.name.present?
merchant = merchants[info.name] ||= family.merchants.find_or_create_by(name: info.name)
if info.icon_url.present?
merchant.icon_url = info.icon_url
end
end
Account.transaction do
merchant.save! if merchant.present?
transaction.update!(merchant: merchant) if merchant.present? && transaction.merchant_id.nil?
transaction.entry.update!(
enriched_at: Time.current,
enriched_name: info.name,
)
end
rescue => e
Rails.logger.warn("Error enriching transaction #{transaction.id}: #{e.message}")
end
end
end
private
def enrichable?
family.data_enrichment_enabled? || (linked? && Rails.application.config.app_mode.hosted?)
end
def enrichable_transactions
transactions.active
.includes(:merchant, :category)
.where(
"entries.enriched_at IS NULL",
"OR merchant_id IS NULL",
"OR category_id IS NULL"
)
end
end

32
app/models/category.rb
View file

@ -50,11 +50,11 @@ class Category < ApplicationRecord
%w[bus circle-dollar-sign ambulance apple award baby battery lightbulb bed-single beer bluetooth book briefcase building credit-card camera utensils cooking-pot cookie dices drama dog drill drum dumbbell gamepad-2 graduation-cap house hand-helping ice-cream-cone phone piggy-bank pill pizza printer puzzle ribbon shopping-cart shield-plus ticket trees]
end
def bootstrap_defaults
default_categories.each do |name, color, icon|
def bootstrap!
default_categories.each do |name, color, icon, classification|
find_or_create_by!(name: name) do |category|
category.color = color
category.classification = "income" if name == "Income"
category.classification = classification
category.lucide_icon = icon
end
end
@ -71,18 +71,20 @@ class Category < ApplicationRecord
private
def default_categories
[
[ "Income", "#e99537", "circle-dollar-sign" ],
[ "Housing", "#6471eb", "house" ],
[ "Entertainment", "#df4e92", "drama" ],
[ "Food & Drink", "#eb5429", "utensils" ],
[ "Shopping", "#e99537", "shopping-cart" ],
[ "Healthcare", "#4da568", "pill" ],
[ "Insurance", "#6471eb", "piggy-bank" ],
[ "Utilities", "#db5a54", "lightbulb" ],
[ "Transportation", "#df4e92", "bus" ],
[ "Education", "#eb5429", "book" ],
[ "Gifts & Donations", "#61c9ea", "hand-helping" ],
[ "Subscriptions", "#805dee", "credit-card" ]
[ "Income", "#e99537", "circle-dollar-sign", "income" ],
[ "Loan Payments", "#6471eb", "credit-card", "expense" ],
[ "Fees", "#6471eb", "credit-card", "expense" ],
[ "Entertainment", "#df4e92", "drama", "expense" ],
[ "Food & Drink", "#eb5429", "utensils", "expense" ],
[ "Shopping", "#e99537", "shopping-cart", "expense" ],
[ "Home Improvement", "#6471eb", "house", "expense" ],
[ "Healthcare", "#4da568", "pill", "expense" ],
[ "Personal Care", "#4da568", "pill", "expense" ],
[ "Services", "#4da568", "briefcase", "expense" ],
[ "Gifts & Donations", "#61c9ea", "hand-helping", "expense" ],
[ "Transportation", "#df4e92", "bus", "expense" ],
[ "Travel", "#df4e92", "plane", "expense" ],
[ "Rent & Utilities", "#db5a54", "lightbulb", "expense" ]
]
end
end

2
app/models/concerns/accountable.rb
View file

@ -9,6 +9,8 @@ module Accountable
end
included do
include Enrichable
has_one :account, as: :accountable, touch: true
end

63
app/models/concerns/enrichable.rb Normal file
View file

@ -0,0 +1,63 @@
# Enrichable models can have 1+ of their fields enriched by various
# external sources (i.e. Plaid) or internal sources (i.e. Rules)
#
# This module defines how models should, lock, unlock, and edit attributes
# based on the source of the edit. User edits always take highest precedence.
#
# For example:
#
# If a Rule tells us to set the category to "Groceries", but the user later overrides
# a transaction with a category of "Food", we should not override the category again.
#
module Enrichable
extend ActiveSupport::Concern
InvalidAttributeError = Class.new(StandardError)
included do
scope :enrichable, ->(attrs) {
attrs = Array(attrs).map(&:to_s)
json_condition = attrs.each_with_object({}) { |attr, hash| hash[attr] = true }
where.not(Arel.sql("#{table_name}.locked_attributes ?| array[:keys]"), keys: attrs)
}
end
def log_enrichment!(attribute_name:, attribute_value:, source:, metadata: {})
de = DataEnrichment.find_or_create_by!(
enrichable: self,
attribute_name: attribute_name,
source: source,
)
de.value = attribute_value
de.metadata = metadata
de.save!
end
def locked?(attr)
locked_attributes[attr.to_s].present?
end
def enrichable?(attr)
!locked?(attr)
end
def lock!(attr)
update!(locked_attributes: locked_attributes.merge(attr.to_s => Time.current))
end
def unlock!(attr)
update!(locked_attributes: locked_attributes.except(attr.to_s))
end
def lock_saved_attributes!
saved_changes.keys.reject { |attr| ignored_enrichable_attributes.include?(attr) }.each do |attr|
lock!(attr)
end
end
private
def ignored_enrichable_attributes
%w[id updated_at created_at]
end
end

5
app/models/data_enrichment.rb Normal file
View file

@ -0,0 +1,5 @@
class DataEnrichment < ApplicationRecord
belongs_to :enrichable, polymorphic: true
enum :source, { rule: "rule", plaid: "plaid", synth: "synth", ai: "ai" }
end

23
app/models/demo/generator.rb
View file

@ -40,7 +40,7 @@ class Demo::Generator
create_tags!(family)
create_categories!(family)
create_merchants!(family)
create_rules!(family)
puts "tags, categories, merchants created for #{family_name}"
create_credit_card_account!(family)
@ -152,7 +152,7 @@ class Demo::Generator
Security::Price.destroy_all
end
def create_family_and_user!(family_name, user_email, data_enrichment_enabled: false, currency: "USD")
def create_family_and_user!(family_name, user_email, currency: "USD")
base_uuid = "d99e3c6e-d513-4452-8f24-dc263f8528c0"
id = Digest::UUID.uuid_v5(base_uuid, family_name)
@ -161,7 +161,6 @@ class Demo::Generator
name: family_name,
currency: currency,
stripe_subscription_status: "active",
data_enrichment_enabled: data_enrichment_enabled,
locale: "en",
country: "US",
timezone: "America/New_York",
@ -185,6 +184,20 @@ class Demo::Generator
onboarded_at: Time.current
end
def create_rules!(family)
family.rules.create!(
effective_date: 1.year.ago.to_date,
active: true,
resource_type: "transaction",
conditions: [
Rule::Condition.new(condition_type: "transaction_merchant", operator: "=", value: "Whole Foods")
],
actions: [
Rule::Action.new(action_type: "set_transaction_category", value: "Groceries")
]
)
end
def create_tags!(family)
[ "Trips", "Emergency Fund", "Demo Tag" ].each do |tag|
family.tags.create!(name: tag)
@ -192,7 +205,7 @@ class Demo::Generator
end
def create_categories!(family)
family.categories.bootstrap_defaults
family.categories.bootstrap!
food = family.categories.find_by(name: "Food & Drink")
family.categories.create!(name: "Restaurants", parent: food, color: COLORS.sample, lucide_icon: "utensils", classification: "expense")
@ -206,7 +219,7 @@ class Demo::Generator
"Uber", "Netflix", "Spotify", "Delta Airlines", "Airbnb", "Sephora" ]
merchants.each do |merchant|
family.merchants.create!(name: merchant, color: COLORS.sample)
FamilyMerchant.create!(name: merchant, family: family, color: COLORS.sample)
end
end

18
app/models/entry.rb
View file

@ -1,5 +1,5 @@
class Entry < ApplicationRecord
include Monetizable
include Monetizable, Enrichable
monetize :amount
@ -34,6 +34,15 @@ class Entry < ApplicationRecord
)
}
def classification
amount.negative? ? "income" : "expense"
end
def lock_saved_attributes!
super
entryable.lock_saved_attributes!
end
def sync_account_later
sync_start_date = [ date_previously_was, date ].compact.min unless destroyed?
account.sync_later(start_date: sync_start_date)
@ -47,10 +56,6 @@ class Entry < ApplicationRecord
Balance::TrendCalculator.new(self, entries, balances).trend
end
def display_name
enriched_name.presence || name
end
class << self
def search(params)
EntrySearch.new(params).build_query(all)
@ -78,6 +83,9 @@ class Entry < ApplicationRecord
all.each do |entry|
bulk_attributes[:entryable_attributes][:id] = entry.entryable_id if bulk_attributes[:entryable_attributes].present?
entry.update! bulk_attributes
entry.lock_saved_attributes!
entry.entryable.lock!(:tag_ids) if entry.transaction? && entry.transaction.tags.any?
end
end

2
app/models/entry_search.rb
View file

@ -16,7 +16,7 @@ class EntrySearch
return scope if search.blank?
query = scope
query = query.where("entries.name ILIKE :search OR entries.enriched_name ILIKE :search",
query = query.where("entries.name ILIKE :search",
search: "%#{ActiveRecord::Base.sanitize_sql_like(search)}%"
)
query

2
app/models/entryable.rb
View file

@ -8,6 +8,8 @@ module Entryable
end
included do
include Enrichable
has_one :entry, as: :entryable, touch: true
scope :with_entry, -> { joins(:entry) }

35
app/models/family.rb
View file

@ -22,12 +22,13 @@ class Family < ApplicationRecord
has_many :entries, through: :accounts
has_many :transactions, through: :accounts
has_many :rules, dependent: :destroy
has_many :trades, through: :accounts
has_many :holdings, through: :accounts
has_many :tags, dependent: :destroy
has_many :categories, dependent: :destroy
has_many :merchants, dependent: :destroy
has_many :merchants, dependent: :destroy, class_name: "FamilyMerchant"
has_many :budgets, dependent: :destroy
has_many :budget_categories, through: :budgets
@ -35,6 +36,27 @@ class Family < ApplicationRecord
validates :locale, inclusion: { in: I18n.available_locales.map(&:to_s) }
validates :date_format, inclusion: { in: DATE_FORMATS.map(&:last) }
def assigned_merchants
merchant_ids = transactions.where.not(merchant_id: nil).pluck(:merchant_id).uniq
Merchant.where(id: merchant_ids)
end
def auto_categorize_transactions_later(transactions)
AutoCategorizeJob.perform_later(self, transaction_ids: transactions.pluck(:id))
end
def auto_categorize_transactions(transaction_ids)
AutoCategorizer.new(self, transaction_ids: transaction_ids).auto_categorize
end
def auto_detect_transaction_merchants_later(transactions)
AutoDetectMerchantsJob.perform_later(self, transaction_ids: transactions.pluck(:id))
end
def auto_detect_transaction_merchants(transaction_ids)
AutoMerchantDetector.new(self, transaction_ids: transaction_ids).auto_detect
end
def balance_sheet
@balance_sheet ||= BalanceSheet.new(self)
end
@ -46,13 +68,20 @@ class Family < ApplicationRecord
def sync_data(sync, start_date: nil)
update!(last_synced_at: Time.current)
Rails.logger.info("Syncing accounts for family #{id}")
accounts.manual.each do |account|
account.sync_later(start_date: start_date, parent_sync: sync)
end
Rails.logger.info("Syncing plaid items for family #{id}")
plaid_items.each do |plaid_item|
plaid_item.sync_later(start_date: start_date, parent_sync: sync)
end
Rails.logger.info("Applying rules for family #{id}")
rules.each do |rule|
rule.apply_later
end
end
def remove_syncing_notice!
@ -138,4 +167,8 @@ class Family < ApplicationRecord
entries.maximum(:updated_at)
].compact.join("_")
end
def self_hoster?
Rails.application.config.app_mode.self_hosted?
end
end

88
app/models/family/auto_categorizer.rb Normal file
View file

@ -0,0 +1,88 @@
class Family::AutoCategorizer
Error = Class.new(StandardError)
def initialize(family, transaction_ids: [])
@family = family
@transaction_ids = transaction_ids
end
def auto_categorize
raise Error, "No LLM provider for auto-categorization" unless llm_provider
if scope.none?
Rails.logger.info("No transactions to auto-categorize for family #{family.id}")
return
else
Rails.logger.info("Auto-categorizing #{scope.count} transactions for family #{family.id}")
end
result = llm_provider.auto_categorize(
transactions: transactions_input,
user_categories: user_categories_input
)
unless result.success?
Rails.logger.error("Failed to auto-categorize transactions for family #{family.id}: #{result.error.message}")
return
end
scope.each do |transaction|
transaction.lock!(:category_id)
auto_categorization = result.data.find { |c| c.transaction_id == transaction.id }
category_id = user_categories_input.find { |c| c[:name] == auto_categorization&.category_name }&.dig(:id)
if category_id.present?
Family.transaction do
transaction.log_enrichment!(
attribute_name: "category_id",
attribute_value: category_id,
source: "ai",
)
transaction.update!(category_id: category_id)
end
end
end
end
private
attr_reader :family, :transaction_ids
# For now, OpenAI only, but this should work with any LLM concept provider
def llm_provider
Provider::Registry.get_provider(:openai)
end
def user_categories_input
family.categories.map do |category|
{
id: category.id,
name: category.name,
is_subcategory: category.subcategory?,
parent_id: category.parent_id,
classification: category.classification
}
end
end
def transactions_input
scope.map do |transaction|
{
id: transaction.id,
amount: transaction.entry.amount.abs,
classification: transaction.entry.classification,
description: transaction.entry.name,
merchant: transaction.merchant&.name,
hint: transaction.plaid_category_detailed
}
end
end
def scope
family.transactions.where(id: transaction_ids, category_id: nil)
.enrichable(:category_id)
.includes(:category, :merchant, :entry)
end
end

100
app/models/family/auto_merchant_detector.rb Normal file
View file

@ -0,0 +1,100 @@
class Family::AutoMerchantDetector
Error = Class.new(StandardError)
def initialize(family, transaction_ids: [])
@family = family
@transaction_ids = transaction_ids
end
def auto_detect
raise "No LLM provider for auto-detecting merchants" unless llm_provider
if scope.none?
Rails.logger.info("No transactions to auto-detect merchants for family #{family.id}")
return
else
Rails.logger.info("Auto-detecting merchants for #{scope.count} transactions for family #{family.id}")
end
result = llm_provider.auto_detect_merchants(
transactions: transactions_input,
user_merchants: user_merchants_input
)
unless result.success?
Rails.logger.error("Failed to auto-detect merchants for family #{family.id}: #{result.error.message}")
return
end
scope.each do |transaction|
transaction.lock!(:merchant_id)
auto_detection = result.data.find { |c| c.transaction_id == transaction.id }
merchant_id = user_merchants_input.find { |m| m[:name] == auto_detection&.business_name }&.dig(:id)
if merchant_id.nil? && auto_detection&.business_url.present? && auto_detection&.business_name.present?
ai_provider_merchant = ProviderMerchant.find_or_create_by!(
source: "ai",
name: auto_detection.business_name,
website_url: auto_detection.business_url,
) do |pm|
pm.logo_url = "#{default_logo_provider_url}/#{auto_detection.business_url}"
end
end
merchant_id = merchant_id || ai_provider_merchant&.id
if merchant_id.present?
Family.transaction do
transaction.log_enrichment!(
attribute_name: "merchant_id",
attribute_value: merchant_id,
source: "ai",
)
transaction.update!(merchant_id: merchant_id)
end
end
end
end
private
attr_reader :family, :transaction_ids
# For now, OpenAI only, but this should work with any LLM concept provider
def llm_provider
Provider::Registry.get_provider(:openai)
end
def default_logo_provider_url
"https://logo.synthfinance.com"
end
def user_merchants_input
family.merchants.map do |merchant|
{
id: merchant.id,
name: merchant.name
}
end
end
def transactions_input
scope.map do |transaction|
{
id: transaction.id,
amount: transaction.entry.amount.abs,
classification: transaction.entry.classification,
description: transaction.entry.name,
merchant: transaction.merchant&.name
}
end
end
def scope
family.transactions.where(id: transaction_ids, merchant_id: nil)
.enrichable(:merchant_id)
.includes(:merchant, :entry)
end
end

15
app/models/family_merchant.rb Normal file
View file

@ -0,0 +1,15 @@
class FamilyMerchant < Merchant
COLORS = %w[#e99537 #4da568 #6471eb #db5a54 #df4e92 #c44fe9 #eb5429 #61c9ea #805dee #6ad28a]
belongs_to :family
before_validation :set_default_color
validates :color, presence: true
validates :name, uniqueness: { scope: :family }
private
def set_default_color
self.color = COLORS.sample
end
end

11
app/models/merchant.rb
View file

@ -1,11 +1,10 @@
class Merchant < ApplicationRecord
has_many :transactions, dependent: :nullify, class_name: "Transaction"
belongs_to :family
TYPES = %w[FamilyMerchant ProviderMerchant].freeze
validates :name, :color, :family, presence: true
validates :name, uniqueness: { scope: :family }
has_many :transactions, dependent: :nullify
validates :name, presence: true
validates :type, inclusion: { in: TYPES }
scope :alphabetically, -> { order(:name) }
COLORS = %w[#e99537 #4da568 #6471eb #db5a54 #df4e92 #c44fe9 #eb5429 #61c9ea #805dee #6ad28a]
end

38
app/models/plaid_account.rb
View file

@ -83,13 +83,14 @@ class PlaidAccount < ApplicationRecord
def sync_transactions!(added:, modified:, removed:)
added.each do |plaid_txn|
account.entries.find_or_create_by!(plaid_id: plaid_txn.transaction_id) do |t|
t.name = plaid_txn.name
t.name = plaid_txn.merchant_name || plaid_txn.original_description
t.amount = plaid_txn.amount
t.currency = plaid_txn.iso_currency_code
t.date = plaid_txn.date
t.entryable = Transaction.new(
category: get_category(plaid_txn.personal_finance_category.primary),
merchant: get_merchant(plaid_txn.merchant_name)
plaid_category: plaid_txn.personal_finance_category.primary,
plaid_category_detailed: plaid_txn.personal_finance_category.detailed,
merchant: find_or_create_merchant(plaid_txn)
)
end
end
@ -99,7 +100,12 @@ class PlaidAccount < ApplicationRecord
existing_txn.update!(
amount: plaid_txn.amount,
date: plaid_txn.date
date: plaid_txn.date,
entryable_attributes: {
plaid_category: plaid_txn.personal_finance_category.primary,
plaid_category_detailed: plaid_txn.personal_finance_category.detailed,
merchant: find_or_create_merchant(plaid_txn)
}
)
end
@ -125,19 +131,19 @@ class PlaidAccount < ApplicationRecord
end
end
# See https://plaid.com/documents/transactions-personal-finance-category-taxonomy.csv
def get_category(plaid_category)
ignored_categories = [ "BANK_FEES", "TRANSFER_IN", "TRANSFER_OUT", "LOAN_PAYMENTS", "OTHER" ]
def find_or_create_merchant(plaid_txn)
unless plaid_txn.merchant_entity_id.present? && plaid_txn.merchant_name.present?
return nil
end
return nil if ignored_categories.include?(plaid_category)
family.categories.find_or_create_by!(name: plaid_category.titleize)
end
def get_merchant(plaid_merchant_name)
return nil if plaid_merchant_name.blank?
family.merchants.find_or_create_by!(name: plaid_merchant_name)
ProviderMerchant.find_or_create_by!(
source: "plaid",
name: plaid_txn.merchant_name,
) do |m|
m.provider_merchant_id = plaid_txn.merchant_entity_id
m.website_url = plaid_txn.website
m.logo_url = plaid_txn.logo_url
end
end
def derive_plaid_cash_balance(plaid_balances)

31
app/models/plaid_item.rb
View file

@ -80,6 +80,7 @@ class PlaidItem < ApplicationRecord
end
def post_sync(sync)
auto_match_categories!
family.broadcast_refresh
end
@ -88,6 +89,36 @@ class PlaidItem < ApplicationRecord
DestroyJob.perform_later(self)
end
def auto_match_categories!
if family.categories.none?
family.categories.bootstrap!
end
alias_matcher = build_category_alias_matcher(family.categories)
accounts.each do |account|
matchable_transactions = account.transactions
.where(category_id: nil)
.where.not(plaid_category: nil)
.enrichable(:category_id)
matchable_transactions.each do |transaction|
category = alias_matcher.match(transaction.plaid_category_detailed)
if category.present?
PlaidItem.transaction do
transaction.log_enrichment!(
attribute_name: "category_id",
attribute_value: category.id,
source: "plaid"
)
transaction.set_category!(category)
end
end
end
end
end
private
def fetch_and_load_plaid_data
data = {}

4
app/models/plaid_item/provided.rb
View file

@ -15,6 +15,10 @@ module PlaidItem::Provided
end
end
def build_category_alias_matcher(user_categories)
Provider::Plaid::CategoryAliasMatcher.new(user_categories)
end
private
def eu?
raise "eu? is not implemented for #{self.class.name}"

12
app/models/provider/llm_concept.rb
View file

@ -1,6 +1,18 @@
module Provider::LlmConcept
extend ActiveSupport::Concern
AutoCategorization = Data.define(:transaction_id, :category_name)
def auto_categorize(transactions)
raise NotImplementedError, "Subclasses must implement #auto_categorize"
end
AutoDetectedMerchant = Data.define(:transaction_id, :business_name, :business_url)
def auto_detect_merchants(transactions)
raise NotImplementedError, "Subclasses must implement #auto_detect_merchants"
end
ChatMessage = Data.define(:id, :output_text)
ChatStreamChunk = Data.define(:type, :data)
ChatResponse = Data.define(:id, :model, :messages, :function_requests)

24
app/models/provider/openai.rb
View file

@ -14,6 +14,30 @@ class Provider::Openai < Provider
MODELS.include?(model)
end
def auto_categorize(transactions: [], user_categories: [])
with_provider_response do
raise Error, "Too many transactions to auto-categorize. Max is 25 per request." if transactions.size > 25
AutoCategorizer.new(
client,
transactions: transactions,
user_categories: user_categories
).auto_categorize
end
end
def auto_detect_merchants(transactions: [], user_merchants: [])
with_provider_response do
raise Error, "Too many transactions to auto-detect merchants. Max is 25 per request." if transactions.size > 25
AutoMerchantDetector.new(
client,
transactions: transactions,
user_merchants: user_merchants
).auto_detect_merchants
end
end
def chat_response(prompt, model:, instructions: nil, functions: [], function_results: [], streamer: nil, previous_response_id: nil)
with_provider_response do
chat_config = ChatConfig.new(

120
app/models/provider/openai/auto_categorizer.rb Normal file
View file

@ -0,0 +1,120 @@
class Provider::Openai::AutoCategorizer
def initialize(client, transactions: [], user_categories: [])
@client = client
@transactions = transactions
@user_categories = user_categories
end
def auto_categorize
response = client.responses.create(parameters: {
model: "gpt-4.1-mini",
input: [ { role: "developer", content: developer_message } ],
text: {
format: {
type: "json_schema",
name: "auto_categorize_personal_finance_transactions",
strict: true,
schema: json_schema
}
},
instructions: instructions
})
Rails.logger.info("Tokens used to auto-categorize transactions: #{response.dig("usage").dig("total_tokens")}")
build_response(extract_categorizations(response))
end
private
attr_reader :client, :transactions, :user_categories
AutoCategorization = Provider::LlmConcept::AutoCategorization
def build_response(categorizations)
categorizations.map do |categorization|
AutoCategorization.new(
transaction_id: categorization.dig("transaction_id"),
category_name: normalize_category_name(categorization.dig("category_name")),
)
end
end
def normalize_category_name(category_name)
return nil if category_name == "null"
category_name
end
def extract_categorizations(response)
response_json = JSON.parse(response.dig("output")[0].dig("content")[0].dig("text"))
response_json.dig("categorizations")
end
def json_schema
{
type: "object",
properties: {
categorizations: {
type: "array",
description: "An array of auto-categorizations for each transaction",
items: {
type: "object",
properties: {
transaction_id: {
type: "string",
description: "The internal ID of the original transaction",
enum: transactions.map { |t| t[:id] }
},
category_name: {
type: "string",
description: "The matched category name of the transaction, or null if no match",
enum: [ *user_categories.map { |c| c[:name] }, "null" ]
}
},
required: [ "transaction_id", "category_name" ],
additionalProperties: false
}
}
},
required: [ "categorizations" ],
additionalProperties: false
}
end
def developer_message
<<~MESSAGE.strip_heredoc
Here are the user's available categories in JSON format:
```json
#{user_categories.to_json}
```
Use the available categories to auto-categorize the following transactions:
```json
#{transactions.to_json}
```
MESSAGE
end
def instructions
<<~INSTRUCTIONS.strip_heredoc
You are an assistant to a consumer personal finance app. You will be provided a list
of the user's transactions and a list of the user's categories. Your job is to auto-categorize
each transaction.
Closely follow ALL the rules below while auto-categorizing:
- Return 1 result per transaction
- Correlate each transaction by ID (transaction_id)
- Attempt to match the most specific category possible (i.e. subcategory over parent category)
- Category and transaction classifications should match (i.e. if transaction is an "expense", the category must have classification of "expense")
- If you don't know the category, return "null"
- You should always favor "null" over false positives
- Be slightly pessimistic. Only match a category if you're 60%+ confident it is the correct one.
- Each transaction has varying metadata that can be used to determine the category
- Note: "hint" comes from 3rd party aggregators and typically represents a category name that
may or may not match any of the user-supplied categories
INSTRUCTIONS
end
end

146
app/models/provider/openai/auto_merchant_detector.rb Normal file
View file

@ -0,0 +1,146 @@
class Provider::Openai::AutoMerchantDetector
def initialize(client, transactions:, user_merchants:)
@client = client
@transactions = transactions
@user_merchants = user_merchants
end
def auto_detect_merchants
response = client.responses.create(parameters: {
model: "gpt-4.1-mini",
input: [ { role: "developer", content: developer_message } ],
text: {
format: {
type: "json_schema",
name: "auto_detect_personal_finance_merchants",
strict: true,
schema: json_schema
}
},
instructions: instructions
})
Rails.logger.info("Tokens used to auto-detect merchants: #{response.dig("usage").dig("total_tokens")}")
build_response(extract_categorizations(response))
end
private
attr_reader :client, :transactions, :user_merchants
AutoDetectedMerchant = Provider::LlmConcept::AutoDetectedMerchant
def build_response(categorizations)
categorizations.map do |categorization|
AutoDetectedMerchant.new(
transaction_id: categorization.dig("transaction_id"),
business_name: normalize_ai_value(categorization.dig("business_name")),
business_url: normalize_ai_value(categorization.dig("business_url")),
)
end
end
def normalize_ai_value(ai_value)
return nil if ai_value == "null"
ai_value
end
def extract_categorizations(response)
response_json = JSON.parse(response.dig("output")[0].dig("content")[0].dig("text"))
response_json.dig("merchants")
end
def json_schema
{
type: "object",
properties: {
merchants: {
type: "array",
description: "An array of auto-detected merchant businesses for each transaction",
items: {
type: "object",
properties: {
transaction_id: {
type: "string",
description: "The internal ID of the original transaction",
enum: transactions.map { |t| t[:id] }
},
business_name: {
type: [ "string", "null" ],
description: "The detected business name of the transaction, or `null` if uncertain"
},
business_url: {
type: [ "string", "null" ],
description: "The URL of the detected business, or `null` if uncertain"
}
},
required: [ "transaction_id", "business_name", "business_url" ],
additionalProperties: false
}
}
},
required: [ "merchants" ],
additionalProperties: false
}
end
def developer_message
<<~MESSAGE.strip_heredoc
Here are the user's available merchants in JSON format:
```json
#{user_merchants.to_json}
```
Use BOTH your knowledge AND the user-generated merchants to auto-detect the following transactions:
```json
#{transactions.to_json}
```
Return "null" if you are not 80%+ confident in your answer.
MESSAGE
end
def instructions
<<~INSTRUCTIONS.strip_heredoc
You are an assistant to a consumer personal finance app.
Closely follow ALL the rules below while auto-detecting business names and website URLs:
- Return 1 result per transaction
- Correlate each transaction by ID (transaction_id)
- Do not include the subdomain in the business_url (i.e. "amazon.com" not "www.amazon.com")
- User merchants are considered "manual" user-generated merchants and should only be used in 100% clear cases
- Be slightly pessimistic. We favor returning "null" over returning a false positive.
- NEVER return a name or URL for generic transaction names (e.g. "Paycheck", "Laundromat", "Grocery store", "Local diner")
Determining a value:
- First attempt to determine the name + URL from your knowledge of global businesses
- If no certain match, attempt to match one of the user-provided merchants
- If no match, return "null"
Example 1 (known business):
```
Transaction name: "Some Amazon purchases"
Result:
- business_name: "Amazon"
- business_url: "amazon.com"
```
Example 2 (generic business):
```
Transaction name: "local diner"
Result:
- business_name: null
- business_url: null
```
INSTRUCTIONS
end
end

5
app/models/provider/plaid.rb
View file

@ -121,7 +121,10 @@ class Provider::Plaid
while has_more
request = Plaid::TransactionsSyncRequest.new(
access_token: item.access_token,
cursor: cursor
cursor: cursor,
options: {
include_original_description: true
}
)
response = client.transactions_sync(request)

109
app/models/provider/plaid/category_alias_matcher.rb Normal file
View file

@ -0,0 +1,109 @@
# The purpose of this matcher is to auto-match Plaid categories to
# known internal user categories. Since we allow users to define their own
# categories we cannot directly assign Plaid categories as this would overwrite
# user data and create a confusing experience.
#
# Automated category matching in the Maybe app has a hierarchy:
# 1. Naive string matching via CategoryAliasMatcher
# 2. Rules-based matching set by user
# 3. AI-powered matching (also enabled by user via rules)
#
# This class is simply a FAST and CHEAP way to match categories that are high confidence.
# Edge cases will be handled by user-defined rules.
class Provider::Plaid::CategoryAliasMatcher
include Provider::Plaid::CategoryTaxonomy
def initialize(user_categories)
@user_categories = user_categories
end
def match(plaid_detailed_category)
plaid_category_details = get_plaid_category_details(plaid_detailed_category)
return nil unless plaid_category_details
# Try exact name matches first
exact_match = normalized_user_categories.find do |category|
category[:name] == plaid_category_details[:key].to_s
end
return user_categories.find { |c| c.id == exact_match[:id] } if exact_match
# Try detailed aliases matches with fuzzy matching
alias_match = normalized_user_categories.find do |category|
name = category[:name]
plaid_category_details[:aliases].any? do |a|
alias_str = a.to_s
# Try exact match
next true if name == alias_str
# Try plural forms
next true if name.singularize == alias_str || name.pluralize == alias_str
next true if alias_str.singularize == name || alias_str.pluralize == name
# Try common forms
normalized_name = name.gsub(/(and|&|\s+)/, "").strip
normalized_alias = alias_str.gsub(/(and|&|\s+)/, "").strip
normalized_name == normalized_alias
end
end
return user_categories.find { |c| c.id == alias_match[:id] } if alias_match
# Try parent aliases matches with fuzzy matching
parent_match = normalized_user_categories.find do |category|
name = category[:name]
plaid_category_details[:parent_aliases].any? do |a|
alias_str = a.to_s
# Try exact match
next true if name == alias_str
# Try plural forms
next true if name.singularize == alias_str || name.pluralize == alias_str
next true if alias_str.singularize == name || alias_str.pluralize == name
# Try common forms
normalized_name = name.gsub(/(and|&|\s+)/, "").strip
normalized_alias = alias_str.gsub(/(and|&|\s+)/, "").strip
normalized_name == normalized_alias
end
end
return user_categories.find { |c| c.id == parent_match[:id] } if parent_match
nil
end
private
attr_reader :user_categories
def get_plaid_category_details(plaid_category_name)
detailed_plaid_categories.find { |c| c[:key] == plaid_category_name.downcase.to_sym }
end
def detailed_plaid_categories
CATEGORIES_MAP.flat_map do |parent_key, parent_data|
parent_data[:detailed_categories].map do |child_key, child_data|
{
key: child_key,
classification: child_data[:classification],
aliases: child_data[:aliases],
parent_key: parent_key,
parent_aliases: parent_data[:aliases]
}
end
end
end
def normalized_user_categories
user_categories.map do |user_category|
{
id: user_category.id,
classification: user_category.classification,
name: normalize_user_category_name(user_category.name)
}
end
end
def normalize_user_category_name(name)
name.to_s.downcase.gsub(/[^a-z0-9]/, " ").strip
end
end

461
app/models/provider/plaid/category_taxonomy.rb Normal file
View file

@ -0,0 +1,461 @@
# https://plaid.com/documents/transactions-personal-finance-category-taxonomy.csv
module Provider::Plaid::CategoryTaxonomy
CATEGORIES_MAP = {
income: {
classification: :income,
aliases: [ "income", "revenue", "earnings" ],
detailed_categories: {
income_dividends: {
classification: :income,
aliases: [ "dividend", "stock income", "dividend income", "dividend earnings" ]
},
income_interest_earned: {
classification: :income,
aliases: [ "interest", "bank interest", "interest earned", "interest income" ]
},
income_retirement_pension: {
classification: :income,
aliases: [ "retirement", "pension" ]
},
income_tax_refund: {
classification: :income,
aliases: [ "tax refund" ]
},
income_unemployment: {
classification: :income,
aliases: [ "unemployment" ]
},
income_wages: {
classification: :income,
aliases: [ "wage", "salary", "paycheck" ]
},
income_other_income: {
classification: :income,
aliases: [ "other income", "misc income" ]
}
}
},
loan_payments: {
classification: :expense,
aliases: [ "loan payment", "debt payment", "loan", "debt", "payment" ],
detailed_categories: {
loan_payments_car_payment: {
classification: :expense,
aliases: [ "car payment", "auto loan" ]
},
loan_payments_credit_card_payment: {
classification: :expense,
aliases: [ "credit card", "card payment" ]
},
loan_payments_personal_loan_payment: {
classification: :expense,
aliases: [ "personal loan", "loan payment" ]
},
loan_payments_mortgage_payment: {
classification: :expense,
aliases: [ "mortgage", "home loan" ]
},
loan_payments_student_loan_payment: {
classification: :expense,
aliases: [ "student loan", "education loan" ]
},
loan_payments_other_payment: {
classification: :expense,
aliases: [ "loan", "loan payment" ]
}
}
},
bank_fees: {
classification: :expense,
aliases: [ "bank fee", "service charge", "fee", "misc fees" ],
detailed_categories: {
bank_fees_atm_fees: {
classification: :expense,
aliases: [ "atm fee", "withdrawal fee" ]
},
bank_fees_foreign_transaction_fees: {
classification: :expense,
aliases: [ "foreign fee", "international fee" ]
},
bank_fees_insufficient_funds: {
classification: :expense,
aliases: [ "nsf fee", "overdraft" ]
},
bank_fees_interest_charge: {
classification: :expense,
aliases: [ "interest charge", "finance charge" ]
},
bank_fees_overdraft_fees: {
classification: :expense,
aliases: [ "overdraft fee" ]
},
bank_fees_other_bank_fees: {
classification: :expense,
aliases: [ "bank fee", "service charge" ]
}
}
},
entertainment: {
classification: :expense,
aliases: [ "entertainment", "recreation" ],
detailed_categories: {
entertainment_casinos_and_gambling: {
classification: :expense,
aliases: [ "casino", "gambling" ]
},
entertainment_music_and_audio: {
classification: :expense,
aliases: [ "music", "concert" ]
},
entertainment_sporting_events_amusement_parks_and_museums: {
classification: :expense,
aliases: [ "event", "amusement", "museum" ]
},
entertainment_tv_and_movies: {
classification: :expense,
aliases: [ "movie", "streaming" ]
},
entertainment_video_games: {
classification: :expense,
aliases: [ "game", "gaming" ]
},
entertainment_other_entertainment: {
classification: :expense,
aliases: [ "entertainment", "recreation" ]
}
}
},
food_and_drink: {
classification: :expense,
aliases: [ "food", "dining", "food and drink", "food & drink" ],
detailed_categories: {
food_and_drink_beer_wine_and_liquor: {
classification: :expense,
aliases: [ "alcohol", "liquor", "beer", "wine", "bar", "pub" ]
},
food_and_drink_coffee: {
classification: :expense,
aliases: [ "coffee", "cafe", "coffee shop" ]
},
food_and_drink_fast_food: {
classification: :expense,
aliases: [ "fast food", "takeout" ]
},
food_and_drink_groceries: {
classification: :expense,
aliases: [ "grocery", "supermarket", "grocery store" ]
},
food_and_drink_restaurant: {
classification: :expense,
aliases: [ "restaurant", "dining" ]
},
food_and_drink_vending_machines: {
classification: :expense,
aliases: [ "vending" ]
},
food_and_drink_other_food_and_drink: {
classification: :expense,
aliases: [ "food", "drink" ]
}
}
},
general_merchandise: {
classification: :expense,
aliases: [ "shopping", "retail" ],
detailed_categories: {
general_merchandise_bookstores_and_newsstands: {
classification: :expense,
aliases: [ "book", "newsstand" ]
},
general_merchandise_clothing_and_accessories: {
classification: :expense,
aliases: [ "clothing", "apparel" ]
},
general_merchandise_convenience_stores: {
classification: :expense,
aliases: [ "convenience" ]
},
general_merchandise_department_stores: {
classification: :expense,
aliases: [ "department store" ]
},
general_merchandise_discount_stores: {
classification: :expense,
aliases: [ "discount store" ]
},
general_merchandise_electronics: {
classification: :expense,
aliases: [ "electronic", "computer" ]
},
general_merchandise_gifts_and_novelties: {
classification: :expense,
aliases: [ "gift", "souvenir" ]
},
general_merchandise_office_supplies: {
classification: :expense,
aliases: [ "office supply" ]
},
general_merchandise_online_marketplaces: {
classification: :expense,
aliases: [ "online shopping" ]
},
general_merchandise_pet_supplies: {
classification: :expense,
aliases: [ "pet supply", "pet food" ]
},
general_merchandise_sporting_goods: {
classification: :expense,
aliases: [ "sporting good", "sport" ]
},
general_merchandise_superstores: {
classification: :expense,
aliases: [ "superstore", "retail" ]
},
general_merchandise_tobacco_and_vape: {
classification: :expense,
aliases: [ "tobacco", "smoke" ]
},
general_merchandise_other_general_merchandise: {
classification: :expense,
aliases: [ "shopping", "merchandise" ]
}
}
},
home_improvement: {
classification: :expense,
aliases: [ "home", "house", "house renovation", "home improvement", "renovation" ],
detailed_categories: {
home_improvement_furniture: {
classification: :expense,
aliases: [ "furniture", "furnishing" ]
},
home_improvement_hardware: {
classification: :expense,
aliases: [ "hardware", "tool" ]
},
home_improvement_repair_and_maintenance: {
classification: :expense,
aliases: [ "repair", "maintenance" ]
},
home_improvement_security: {
classification: :expense,
aliases: [ "security", "alarm" ]
},
home_improvement_other_home_improvement: {
classification: :expense,
aliases: [ "home improvement", "renovation" ]
}
}
},
medical: {
classification: :expense,
aliases: [ "medical", "healthcare", "health" ],
detailed_categories: {
medical_dental_care: {
classification: :expense,
aliases: [ "dental", "dentist" ]
},
medical_eye_care: {
classification: :expense,
aliases: [ "eye", "optometrist" ]
},
medical_nursing_care: {
classification: :expense,
aliases: [ "nursing", "care" ]
},
medical_pharmacies_and_supplements: {
classification: :expense,
aliases: [ "pharmacy", "prescription" ]
},
medical_primary_care: {
classification: :expense,
aliases: [ "doctor", "medical" ]
},
medical_veterinary_services: {
classification: :expense,
aliases: [ "vet", "veterinary" ]
},
medical_other_medical: {
classification: :expense,
aliases: [ "medical", "healthcare" ]
}
}
},
personal_care: {
classification: :expense,
aliases: [ "personal care", "grooming" ],
detailed_categories: {
personal_care_gyms_and_fitness_centers: {
classification: :expense,
aliases: [ "gym", "fitness", "exercise", "sport" ]
},
personal_care_hair_and_beauty: {
classification: :expense,
aliases: [ "salon", "beauty" ]
},
personal_care_laundry_and_dry_cleaning: {
classification: :expense,
aliases: [ "laundry", "cleaning" ]
},
personal_care_other_personal_care: {
classification: :expense,
aliases: [ "personal care", "grooming" ]
}
}
},
general_services: {
classification: :expense,
aliases: [ "service", "professional service" ],
detailed_categories: {
general_services_accounting_and_financial_planning: {
classification: :expense,
aliases: [ "accountant", "financial advisor" ]
},
general_services_automotive: {
classification: :expense,
aliases: [ "auto repair", "mechanic", "vehicle", "car", "car care", "car maintenance", "vehicle maintenance" ]
},
general_services_childcare: {
classification: :expense,
aliases: [ "childcare", "daycare" ]
},
general_services_consulting_and_legal: {
classification: :expense,
aliases: [ "legal", "attorney" ]
},
general_services_education: {
classification: :expense,
aliases: [ "education", "tuition" ]
},
general_services_insurance: {
classification: :expense,
aliases: [ "insurance", "premium" ]
},
general_services_postage_and_shipping: {
classification: :expense,
aliases: [ "shipping", "postage" ]
},
general_services_storage: {
classification: :expense,
aliases: [ "storage" ]
},
general_services_other_general_services: {
classification: :expense,
aliases: [ "service" ]
}
}
},
government_and_non_profit: {
classification: :expense,
aliases: [ "government", "non-profit" ],
detailed_categories: {
government_and_non_profit_donations: {
classification: :expense,
aliases: [ "donation", "charity", "charitable", "charitable donation", "giving", "gifts and donations", "gifts & donations" ]
},
government_and_non_profit_government_departments_and_agencies: {
classification: :expense,
aliases: [ "government", "agency" ]
},
government_and_non_profit_tax_payment: {
classification: :expense,
aliases: [ "tax payment", "tax" ]
},
government_and_non_profit_other_government_and_non_profit: {
classification: :expense,
aliases: [ "government", "non-profit" ]
}
}
},
transportation: {
classification: :expense,
aliases: [ "transportation", "travel" ],
detailed_categories: {
transportation_bikes_and_scooters: {
classification: :expense,
aliases: [ "bike", "scooter" ]
},
transportation_gas: {
classification: :expense,
aliases: [ "gas", "fuel" ]
},
transportation_parking: {
classification: :expense,
aliases: [ "parking" ]
},
transportation_public_transit: {
classification: :expense,
aliases: [ "transit", "bus" ]
},
transportation_taxis_and_ride_shares: {
classification: :expense,
aliases: [ "taxi", "rideshare" ]
},
transportation_tolls: {
classification: :expense,
aliases: [ "toll" ]
},
transportation_other_transportation: {
classification: :expense,
aliases: [ "transportation", "travel" ]
}
}
},
travel: {
classification: :expense,
aliases: [ "travel", "vacation", "trip", "sabbatical" ],
detailed_categories: {
travel_flights: {
classification: :expense,
aliases: [ "flight", "airfare" ]
},
travel_lodging: {
classification: :expense,
aliases: [ "hotel", "lodging" ]
},
travel_rental_cars: {
classification: :expense,
aliases: [ "rental car" ]
},
travel_other_travel: {
classification: :expense,
aliases: [ "travel", "trip" ]
}
}
},
rent_and_utilities: {
classification: :expense,
aliases: [ "utilities", "housing", "house", "home", "rent", "rent & utilities" ],
detailed_categories: {
rent_and_utilities_gas_and_electricity: {
classification: :expense,
aliases: [ "utility", "electric" ]
},
rent_and_utilities_internet_and_cable: {
classification: :expense,
aliases: [ "internet", "cable" ]
},
rent_and_utilities_rent: {
classification: :expense,
aliases: [ "rent", "lease" ]
},
rent_and_utilities_sewage_and_waste_management: {
classification: :expense,
aliases: [ "sewage", "waste" ]
},
rent_and_utilities_telephone: {
classification: :expense,
aliases: [ "phone", "telephone" ]
},
rent_and_utilities_water: {
classification: :expense,
aliases: [ "water" ]
},
rent_and_utilities_other_utilities: {
classification: :expense,
aliases: [ "utility" ]
}
}
}
}
end

29
app/models/provider/synth.rb
View file

@ -159,38 +159,9 @@ class Provider::Synth < Provider
end
end
# ================================
# Transactions
# ================================
def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
with_provider_response do
params = {
description: description,
amount: amount,
date: date,
city: city,
state: state,
country: country
}.compact
response = client.get("#{base_url}/enrich", params)
parsed = JSON.parse(response.body)
TransactionEnrichmentData.new(
name: parsed.dig("merchant"),
icon_url: parsed.dig("icon"),
category: parsed.dig("category")
)
end
end
private
attr_reader :api_key
TransactionEnrichmentData = Data.define(:name, :icon_url, :category)
def base_url
ENV["SYNTH_URL"] || "https://api.synthfinance.com"
end

6
app/models/provider_merchant.rb Normal file
View file

@ -0,0 +1,6 @@
class ProviderMerchant < Merchant
enum :source, { plaid: "plaid", synth: "synth", ai: "ai" }
validates :name, uniqueness: { scope: [ :source ] }
validates :source, presence: true
end

90
app/models/rule.rb Normal file
View file

@ -0,0 +1,90 @@
class Rule < ApplicationRecord
UnsupportedResourceTypeError = Class.new(StandardError)
belongs_to :family
has_many :conditions, dependent: :destroy
has_many :actions, dependent: :destroy
accepts_nested_attributes_for :conditions, allow_destroy: true
accepts_nested_attributes_for :actions, allow_destroy: true
validates :resource_type, presence: true
validate :no_nested_compound_conditions
# Every rule must have at least 1 action
validate :min_actions
validate :no_duplicate_actions
def action_executors
registry.action_executors
end
def condition_filters
registry.condition_filters
end
def registry
@registry ||= case resource_type
when "transaction"
Rule::Registry::TransactionResource.new(self)
else
raise UnsupportedResourceTypeError, "Unsupported resource type: #{resource_type}"
end
end
def affected_resource_count
matching_resources_scope.count
end
def apply(ignore_attribute_locks: false)
actions.each do |action|
action.apply(matching_resources_scope, ignore_attribute_locks: ignore_attribute_locks)
end
end
def apply_later(ignore_attribute_locks: false)
RuleJob.perform_later(self, ignore_attribute_locks: ignore_attribute_locks)
end
private
def matching_resources_scope
scope = registry.resource_scope
# 1. Prepare the query with joins required by conditions
conditions.each do |condition|
scope = condition.prepare(scope)
end
# 2. Apply the conditions to the query
conditions.each do |condition|
scope = condition.apply(scope)
end
scope
end
def min_actions
if actions.reject(&:marked_for_destruction?).empty?
errors.add(:base, "must have at least one action")
end
end
def no_duplicate_actions
action_types = actions.reject(&:marked_for_destruction?).map(&:action_type)
errors.add(:base, "Rule cannot have duplicate actions #{action_types.inspect}") if action_types.uniq.count != action_types.count
end
# Validation: To keep rules simple and easy to understand, we don't allow nested compound conditions.
def no_nested_compound_conditions
return true if conditions.none? { |condition| condition.compound? }
conditions.each do |condition|
if condition.compound?
if condition.sub_conditions.any? { |sub_condition| sub_condition.compound? }
errors.add(:base, "Compound conditions cannot be nested")
end
end
end
end
end

17
app/models/rule/action.rb Normal file
View file

@ -0,0 +1,17 @@
class Rule::Action < ApplicationRecord
belongs_to :rule
validates :action_type, presence: true
def apply(resource_scope, ignore_attribute_locks: false)
executor.execute(resource_scope, value: value, ignore_attribute_locks: ignore_attribute_locks)
end
def options
executor.options
end
def executor
rule.registry.get_executor!(action_type)
end
end

43
app/models/rule/action_executor.rb Normal file
View file

@ -0,0 +1,43 @@
class Rule::ActionExecutor
TYPES = [ "select", "function" ]
def initialize(rule)
@rule = rule
end
def key
self.class.name.demodulize.underscore
end
def label
key.humanize
end
def type
"function"
end
def options
nil
end
def execute(scope, value: nil, ignore_attribute_locks: false)
raise NotImplementedError, "Action executor #{self.class.name} must implement #execute"
end
def as_json
{
type: type,
key: key,
label: label,
options: options
}
end
private
attr_reader :rule
def family
rule.family
end
end

23
app/models/rule/action_executor/auto_categorize.rb Normal file
View file

@ -0,0 +1,23 @@
class Rule::ActionExecutor::AutoCategorize < Rule::ActionExecutor
def label
if rule.family.self_hoster?
"Auto-categorize transactions with AI ($$)"
else
"Auto-categorize transactions"
end
end
def execute(transaction_scope, value: nil, ignore_attribute_locks: false)
enrichable_transactions = transaction_scope.enrichable(:category_id)
if enrichable_transactions.empty?
Rails.logger.info("No transactions to auto-categorize for #{rule.title} #{rule.id}")
return
end
enrichable_transactions.in_batches(of: 20).each_with_index do |transactions, idx|
Rails.logger.info("Scheduling auto-categorization for batch #{idx + 1} of #{enrichable_transactions.count}")
rule.family.auto_categorize_transactions_later(transactions)
end
end
end

23
app/models/rule/action_executor/auto_detect_merchants.rb Normal file
View file

@ -0,0 +1,23 @@
class Rule::ActionExecutor::AutoDetectMerchants < Rule::ActionExecutor
def label
if rule.family.self_hoster?
"Auto-detect merchants with AI ($$)"
else
"Auto-detect merchants"
end
end
def execute(transaction_scope, value: nil, ignore_attribute_locks: false)
enrichable_transactions = transaction_scope.enrichable(:merchant_id)
if enrichable_transactions.empty?
Rails.logger.info("No transactions to auto-detect merchants for #{rule.title} #{rule.id}")
return
end
enrichable_transactions.in_batches(of: 20).each_with_index do |transactions, idx|
Rails.logger.info("Scheduling auto-merchant-enrichment for batch #{idx + 1} of #{enrichable_transactions.count}")
rule.family.auto_detect_transaction_merchants_later(transactions)
end
end
end

31
app/models/rule/action_executor/set_transaction_category.rb Normal file
View file

@ -0,0 +1,31 @@
class Rule::ActionExecutor::SetTransactionCategory < Rule::ActionExecutor
def type
"select"
end
def options
family.categories.pluck(:name, :id)
end
def execute(transaction_scope, value: nil, ignore_attribute_locks: false)
category = family.categories.find_by_id(value)
scope = transaction_scope
unless ignore_attribute_locks
scope = scope.enrichable(:category_id)
end
scope.each do |txn|
Rule.transaction do
txn.log_enrichment!(
attribute_name: "category_id",
attribute_value: category.id,
source: "rule"
)
txn.update!(category: category)
end
end
end
end

31
app/models/rule/action_executor/set_transaction_tags.rb Normal file
View file

@ -0,0 +1,31 @@
class Rule::ActionExecutor::SetTransactionTags < Rule::ActionExecutor
def type
"select"
end
def options
family.tags.pluck(:name, :id)
end
def execute(transaction_scope, value: nil, ignore_attribute_locks: false)
tag = family.tags.find_by_id(value)
scope = transaction_scope
unless ignore_attribute_locks
scope = scope.enrichable(:tag_ids)
end
rows = scope.each do |txn|
Rule.transaction do
txn.log_enrichment!(
attribute_name: "tag_ids",
attribute_value: [ tag.id ],
source: "rule"
)
txn.update!(tag_ids: [ tag.id ])
end
end
end
end

62
app/models/rule/condition.rb Normal file
View file

@ -0,0 +1,62 @@
class Rule::Condition < ApplicationRecord
belongs_to :rule, optional: -> { where.not(parent_id: nil) }
belongs_to :parent, class_name: "Rule::Condition", optional: true, inverse_of: :sub_conditions
has_many :sub_conditions, class_name: "Rule::Condition", foreign_key: :parent_id, dependent: :destroy, inverse_of: :parent
validates :condition_type, presence: true
validates :operator, presence: true
validates :value, presence: true, unless: -> { compound? }
accepts_nested_attributes_for :sub_conditions, allow_destroy: true
# We don't store rule_id on sub_conditions, so "walk up" to the parent rule
def rule
parent&.rule || super
end
def compound?
condition_type == "compound"
end
def apply(scope)
if compound?
build_compound_scope(scope)
else
filter.apply(scope, operator, value)
end
end
def prepare(scope)
if compound?
sub_conditions.reduce(scope) { |s, sub| sub.prepare(s) }
else
filter.prepare(scope)
end
end
def options
filter.options
end
def operators
filter.operators
end
def filter
rule.registry.get_filter!(condition_type)
end
private
def build_compound_scope(scope)
if operator == "or"
combined_scope = sub_conditions
.map { |sub| sub.apply(scope) }
.reduce { |acc, s| acc.or(s) }
combined_scope || scope
else
sub_conditions.reduce(scope) { |s, sub| sub.apply(s) }
end
end
end

87
app/models/rule/condition_filter.rb Normal file
View file

@ -0,0 +1,87 @@
class Rule::ConditionFilter
UnsupportedOperatorError = Class.new(StandardError)
TYPES = [ "text", "number", "select" ]
OPERATORS_MAP = {
"text" => [ [ "Contains", "like" ], [ "Equal to", "=" ] ],
"number" => [ [ "Greater than", ">" ], [ "Greater or equal to", ">=" ], [ "Less than", "<" ], [ "Less than or equal to", "<=" ], [ "Is equal to", "=" ] ],
"select" => [ [ "Equal to", "=" ] ]
}
def initialize(rule)
@rule = rule
end
def type
"text"
end
def number_step
family_currency = Money::Currency.new(family.currency)
family_currency.step
end
def key
self.class.name.demodulize.underscore
end
def label
key.humanize
end
def options
nil
end
def operators
OPERATORS_MAP.dig(type)
end
# Matchers can prepare the scope with joins by implementing this method
def prepare(scope)
scope
end
# Applies the condition to the scope
def apply(scope, operator, value)
raise NotImplementedError, "Condition #{self.class.name} must implement #apply"
end
def as_json
{
type: type,
key: key,
label: label,
operators: operators,
options: options,
number_step: number_step
}
end
private
attr_reader :rule
def family
rule.family
end
def build_sanitized_where_condition(field, operator, value)
sanitized_value = operator == "like" ? "%#{ActiveRecord::Base.sanitize_sql_like(value)}%" : value
ActiveRecord::Base.sanitize_sql_for_conditions([
"#{field} #{sanitize_operator(operator)} ?",
sanitized_value
])
end
def sanitize_operator(operator)
raise UnsupportedOperatorError, "Unsupported operator: #{operator} for type: #{type}" unless operators.map(&:last).include?(operator)
if operator == "like"
"ILIKE"
else
operator
end
end
end

14
app/models/rule/condition_filter/transaction_amount.rb Normal file
View file

@ -0,0 +1,14 @@
class Rule::ConditionFilter::TransactionAmount < Rule::ConditionFilter
def type
"number"
end
def prepare(scope)
scope.with_entry
end
def apply(scope, operator, value)
expression = build_sanitized_where_condition("ABS(entries.amount)", operator, value.to_d)
scope.where(expression)
end
end

18
app/models/rule/condition_filter/transaction_merchant.rb Normal file
View file

@ -0,0 +1,18 @@
class Rule::ConditionFilter::TransactionMerchant < Rule::ConditionFilter
def type
"select"
end
def options
family.assigned_merchants.pluck(:name, :id)
end
def prepare(scope)
scope.left_joins(:merchant)
end
def apply(scope, operator, value)
expression = build_sanitized_where_condition("merchants.id", operator, value)
scope.where(expression)
end
end

10
app/models/rule/condition_filter/transaction_name.rb Normal file
View file

@ -0,0 +1,10 @@
class Rule::ConditionFilter::TransactionName < Rule::ConditionFilter
def prepare(scope)
scope.with_entry
end
def apply(scope, operator, value)
expression = build_sanitized_where_condition("entries.name", operator, value)
scope.where(expression)
end
end

46
app/models/rule/registry.rb Normal file
View file

@ -0,0 +1,46 @@
class Rule::Registry
UnsupportedActionError = Class.new(StandardError)
UnsupportedConditionError = Class.new(StandardError)
def initialize(rule)
@rule = rule
end
def resource_scope
raise NotImplementedError, "#{self.class.name} must implement #resource_scope"
end
def condition_filters
[]
end
def action_executors
[]
end
def get_filter!(key)
filter = condition_filters.find { |filter| filter.key == key }
raise UnsupportedConditionError, "Unsupported condition type: #{key}" unless filter
filter
end
def get_executor!(key)
executor = action_executors.find { |executor| executor.key == key }
raise UnsupportedActionError, "Unsupported action type: #{key}" unless executor
executor
end
def as_json
{
filters: condition_filters.map(&:as_json),
executors: action_executors.map(&:as_json)
}
end
private
attr_reader :rule
def family
rule.family
end
end

32
app/models/rule/registry/transaction_resource.rb Normal file
View file

@ -0,0 +1,32 @@
class Rule::Registry::TransactionResource < Rule::Registry
def resource_scope
family.transactions.active.with_entry.where(entry: { date: rule.effective_date.. })
end
def condition_filters
[
Rule::ConditionFilter::TransactionName.new(rule),
Rule::ConditionFilter::TransactionAmount.new(rule),
Rule::ConditionFilter::TransactionMerchant.new(rule)
]
end
def action_executors
enabled_executors = [
Rule::ActionExecutor::SetTransactionCategory.new(rule),
Rule::ActionExecutor::SetTransactionTags.new(rule)
]
if ai_enabled?
enabled_executors << Rule::ActionExecutor::AutoCategorize.new(rule)
enabled_executors << Rule::ActionExecutor::AutoDetectMerchants.new(rule)
end
enabled_executors
end
private
def ai_enabled?
Provider::Registry.get_provider(:openai).present?
end
end

12
app/models/sync.rb
View file

@ -23,19 +23,19 @@ class Sync < ApplicationRecord
update!(data: data) if data
complete! unless has_pending_child_syncs?
rescue StandardError => error
fail! error
raise error if Rails.env.development?
ensure
Rails.logger.info("Sync completed, starting post-sync")
syncable.post_sync(self) unless has_pending_child_syncs?
if has_parent?
notify_parent_of_completion!
else
syncable.post_sync(self)
end
Rails.logger.info("Post-sync completed")
rescue StandardError => error
fail! error
raise error if Rails.env.development?
end
end
end

15
app/models/trade_builder.rb
View file

@ -15,6 +15,21 @@ class TradeBuilder
buildable.save
end
def lock_saved_attributes!
if buildable.is_a?(Transfer)
buildable.inflow_transaction.entry.lock_saved_attributes!
buildable.outflow_transaction.entry.lock_saved_attributes!
else
buildable.lock_saved_attributes!
end
end
def entryable
return nil if buildable.is_a?(Transfer)
buildable.entryable
end
def errors
buildable.errors
end

12
app/models/transaction.rb
View file

@ -1,5 +1,5 @@
class Transaction < ApplicationRecord
include Entryable, Transferable, Provided
include Entryable, Transferable, Ruleable
belongs_to :category, optional: true
belongs_to :merchant, optional: true
@ -14,4 +14,14 @@ class Transaction < ApplicationRecord
Search.new(params).build_query(all)
end
end
def set_category!(category)
if category.is_a?(String)
category = entry.account.family.categories.find_or_create_by!(
name: category
)
end
update!(category: category)
end
end

20
app/models/transaction/provided.rb
View file

@ -1,20 +0,0 @@
module Transaction::Provided
extend ActiveSupport::Concern
def fetch_enrichment_info
return nil unless provider
response = provider.enrich_transaction(
entry.name,
amount: entry.amount,
date: entry.date
)
response.data
end
private
def provider
Provider::Registry.get_provider(:synth)
end
end

17
app/models/transaction/ruleable.rb Normal file
View file

@ -0,0 +1,17 @@
module Transaction::Ruleable
extend ActiveSupport::Concern
def eligible_for_category_rule?
rules.joins(:actions).where(
actions: {
action_type: "set_transaction_category",
value: category_id
}
).empty?
end
private
def rules
entry.account.family.rules
end
end

2
app/models/transfer.rb
View file

@ -29,7 +29,6 @@ class Transfer < ApplicationRecord
currency: converted_amount.currency.iso_code,
date: date,
name: "Transfer from #{from_account.name}",
entryable: Transaction.new
)
),
outflow_transaction: Transaction.new(
@ -38,7 +37,6 @@ class Transfer < ApplicationRecord
currency: from_account.currency,
date: date,
name: "Transfer to #{to_account.name}",
entryable: Transaction.new
)
),
status: "confirmed"