Add account data enrichment (#1532)

* Add data enrichment

* Make data enrichment optional for self-hosters

* Add categories to data enrichment

* Only update category and merchant if nil

* Fix name overrides

* Lint fixes

app/models/account.rb

@@ -126,6 +126,14 @@ class Account < ApplicationRecord
     classification == "asset" ? "up" : "down"
   end
 
+  def enrich_data
+    DataEnricher.new(self).run
+  end
+
+  def enrich_data_later
+    EnrichDataJob.perform_later(self)
+  end
+
   def update_with_sync!(attributes)
     transaction do
       update!(attributes)

app/models/account/data_enricher.rb

@@ -0,0 +1,61 @@
+class Account::DataEnricher
+  include Providable
+
+  attr_reader :account
+
+  def initialize(account)
+    @account = account
+  end
+
+  def run
+    enrich_transactions
+  end
+
+  private
+    def enrich_transactions
+      candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ])
+
+      Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}")
+
+      merchants = {}
+      categories = {}
+
+      candidates.each do |entry|
+        if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil?
+          begin
+            info = self.class.synth_provider.enrich_transaction(entry.name).info
+
+            next unless info.present?
+
+            if info.name.present?
+              merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
+
+              if info.icon_url.present?
+                merchant.icon_url = info.icon_url
+              end
+            end
+
+            if info.category.present?
+              category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category)
+            end
+
+            entryable_attributes = { id: entry.entryable_id }
+            entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
+            entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil?
+
+            Account.transaction do
+              merchant.save! if merchant.present?
+              category.save! if category.present?
+              entry.update!(
+                enriched_at: Time.current,
+                name: entry.enriched_at.nil? ? info.name : entry.name,
+                entryable_attributes: entryable_attributes
+              )
+            end
+          rescue => e
+            Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
+          end
+        end
+      end
+    end
+end

app/models/account/syncer.rb

@@ -10,6 +10,12 @@ class Account::Syncer
     account.reload
     update_account_info(balances, holdings) unless account.plaid_account_id.present?
     convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency
+
+    if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed?
+      account.enrich_data_later
+    else
+      Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}")
+    end
   end
 
   private

app/models/concerns/providable.rb

@@ -23,8 +23,10 @@ module Providable
     end
 
     def synth_provider
-      api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
-      api_key.present? ? Provider::Synth.new(api_key) : nil
+      @synth_provider ||= begin
+        api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+        api_key.present? ? Provider::Synth.new(api_key) : nil
+      end
     end
 
     private

app/models/provider/synth.rb

@@ -167,6 +167,35 @@ class Provider::Synth
       raw_response: response
   end
 
+  def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
+    params = {
+      description: description,
+      amount: amount,
+      date: date,
+      city: city,
+      state: state,
+      country: country
+    }.compact
+
+    response = client.get("#{base_url}/enrich", params)
+
+    parsed = JSON.parse(response.body)
+
+    EnrichTransactionResponse.new \
+      info: EnrichTransactionInfo.new(
+        name: parsed.dig("merchant"),
+        icon_url: parsed.dig("icon"),
+        category: parsed.dig("category")
+      ),
+      success?: true,
+      raw_response: response
+  rescue StandardError => error
+    EnrichTransactionResponse.new \
+      success?: false,
+      error: error,
+      raw_response: error
+  end
+
   private
 
     attr_reader :api_key
@@ -177,6 +206,8 @@ class Provider::Synth
     UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
     SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
     SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
 
     def base_url
       ENV["SYNTH_URL"] || "https://api.synthfinance.com"

app/models/setting.rb

@@ -17,6 +17,10 @@ class Setting < RailsSettings::Base
         default: ENV.fetch("UPGRADES_TARGET", "release"),
         validates: { inclusion: { in: %w[release commit] } }
 
+  field :data_enrichment_enabled,
+        type: :boolean,
+        default: true
+
   field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]
 
   field :require_invite_for_signup, type: :boolean, default: false