Add account data enrichment (#1532)

* Add data enrichment

* Make data enrichment optional for self-hosters

* Add categories to data enrichment

* Only update category and merchant if nil

* Fix name overrides

* Lint fixes

app/controllers/settings/hostings_controller.rb

@@ -26,6 +26,10 @@ class Settings::HostingsController < SettingsController
       Setting.synth_api_key = hosting_params[:synth_api_key]
     end
 
+    if hosting_params.key?(:data_enrichment_enabled)
+      Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled]
+    end
+
     redirect_to settings_hosting_path, notice: t(".success")
   rescue ActiveRecord::RecordInvalid => error
     flash.now[:alert] = t(".failure")
@@ -34,7 +38,7 @@ class Settings::HostingsController < SettingsController
 
   private
     def hosting_params
-      params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key)
+      params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled)
     end
 
     def raise_if_not_self_hosted

app/jobs/enrich_data_job.rb

@@ -0,0 +1,7 @@
+class EnrichDataJob < ApplicationJob
+  queue_as :default
+
+  def perform(account)
+    account.enrich_data
+  end
+end

app/models/account.rb

@@ -126,6 +126,14 @@ class Account < ApplicationRecord
     classification == "asset" ? "up" : "down"
   end
 
+  def enrich_data
+    DataEnricher.new(self).run
+  end
+
+  def enrich_data_later
+    EnrichDataJob.perform_later(self)
+  end
+
   def update_with_sync!(attributes)
     transaction do
       update!(attributes)

app/models/account/data_enricher.rb

@@ -0,0 +1,61 @@
+class Account::DataEnricher
+  include Providable
+
+  attr_reader :account
+
+  def initialize(account)
+    @account = account
+  end
+
+  def run
+    enrich_transactions
+  end
+
+  private
+    def enrich_transactions
+      candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ])
+
+      Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}")
+
+      merchants = {}
+      categories = {}
+
+      candidates.each do |entry|
+        if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil?
+          begin
+            info = self.class.synth_provider.enrich_transaction(entry.name).info
+
+            next unless info.present?
+
+            if info.name.present?
+              merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
+
+              if info.icon_url.present?
+                merchant.icon_url = info.icon_url
+              end
+            end
+
+            if info.category.present?
+              category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category)
+            end
+
+            entryable_attributes = { id: entry.entryable_id }
+            entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
+            entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil?
+
+            Account.transaction do
+              merchant.save! if merchant.present?
+              category.save! if category.present?
+              entry.update!(
+                enriched_at: Time.current,
+                name: entry.enriched_at.nil? ? info.name : entry.name,
+                entryable_attributes: entryable_attributes
+              )
+            end
+          rescue => e
+            Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
+          end
+        end
+      end
+    end
+end

app/models/account/syncer.rb

@@ -10,6 +10,12 @@ class Account::Syncer
     account.reload
     update_account_info(balances, holdings) unless account.plaid_account_id.present?
     convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency
+
+    if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed?
+      account.enrich_data_later
+    else
+      Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}")
+    end
   end
 
   private

app/models/concerns/providable.rb

@@ -23,8 +23,10 @@ module Providable
     end
 
     def synth_provider
-      api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+      @synth_provider ||= begin
-      api_key.present? ? Provider::Synth.new(api_key) : nil
+        api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+        api_key.present? ? Provider::Synth.new(api_key) : nil
+      end
     end
 
     private

app/models/provider/synth.rb

@@ -167,6 +167,35 @@ class Provider::Synth
       raw_response: response
   end
 
+  def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
+    params = {
+      description: description,
+      amount: amount,
+      date: date,
+      city: city,
+      state: state,
+      country: country
+    }.compact
+
+    response = client.get("#{base_url}/enrich", params)
+
+    parsed = JSON.parse(response.body)
+
+    EnrichTransactionResponse.new \
+      info: EnrichTransactionInfo.new(
+        name: parsed.dig("merchant"),
+        icon_url: parsed.dig("icon"),
+        category: parsed.dig("category")
+      ),
+      success?: true,
+      raw_response: response
+  rescue StandardError => error
+    EnrichTransactionResponse.new \
+      success?: false,
+      error: error,
+      raw_response: error
+  end
+
   private
 
     attr_reader :api_key
@@ -177,6 +206,8 @@ class Provider::Synth
     UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
     SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
     SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
 
     def base_url
       ENV["SYNTH_URL"] || "https://api.synthfinance.com"

app/models/setting.rb

@@ -17,6 +17,10 @@ class Setting < RailsSettings::Base
         default: ENV.fetch("UPGRADES_TARGET", "release"),
         validates: { inclusion: { in: %w[release commit] } }
 
+  field :data_enrichment_enabled,
+        type: :boolean,
+        default: true
+
   field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]
 
   field :require_invite_for_signup, type: :boolean, default: false

app/views/account/transactions/_transaction.html.erb

@@ -11,15 +11,17 @@
 
     <div class="max-w-full">
       <%= content_tag :div, class: ["flex items-center gap-2"] do %>
-        <div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full bg-gray-600/5 text-gray-600">
+        <% if entry.account_transaction.merchant&.icon_url %>
-          <%= transaction.name.first.upcase %>
+          <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %>
-        </div>
+        <% else %>
+          <%= render "shared/circle_logo", name: entry.name, size: "sm" %>
+        <% end %>
 
         <div class="truncate">
           <% if entry.new_record? %>
-            <%= content_tag :p, transaction.name %>
+            <%= content_tag :p, entry.name %>
           <% else %>
-            <%= link_to transaction.name,
+            <%= link_to entry.name,
                         entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry),
                         data: { turbo_frame: "drawer", turbo_prefetch: false },
                         class: "hover:underline hover:text-gray-800" %>

app/views/merchants/_merchant.html.erb

@@ -2,7 +2,14 @@
 
 <div class="flex justify-between items-center p-4 bg-white">
   <div class="flex w-full items-center gap-2.5">
-    <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+    <% if merchant.icon_url %>
+      <div class="w-8 h-8 rounded-full flex justify-center items-center">
+        <%= image_tag merchant.icon_url, class: "w-8 h-8 rounded-full" %>
+      </div>
+    <% else %>
+      <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+    <% end %>
+
     <p class="text-gray-900 text-sm truncate">
       <%= merchant.name %>
     </p>

app/views/settings/hostings/_data_enrichment_settings.html.erb

@@ -0,0 +1,18 @@
+<div class="space-y-4">
+  <div class="flex items-center justify-between">
+    <div class="space-y-1">
+      <p class="text-sm"><%= t(".title") %></p>
+      <p class="text-gray-500 text-sm"><%= t(".description") %></p>
+    </div>
+
+    <%= styled_form_with model: Setting.new,
+                       url: settings_hosting_path,
+                       method: :patch,
+                       data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %>
+      <div class="relative inline-block select-none">
+        <%= form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %>
+        <%= form.label :data_enrichment_enabled, "&nbsp;".html_safe, class: "maybe-switch" %>
+      </div>
+    <% end %>
+  </div>
+</div>

app/views/settings/hostings/show.html.erb

@@ -10,6 +10,7 @@
       <%= render "settings/hostings/upgrade_settings" %>
       <%= render "settings/hostings/provider_settings" %>
       <%= render "settings/hostings/synth_settings" %>
+      <%= render "settings/hostings/data_enrichment_settings" %>
     </div>
   <% end %>
 

config/locales/views/settings/hostings/en.yml

@@ -2,6 +2,9 @@
 en:
   settings:
     hostings:
+      data_enrichment_settings:
+        description: Enable data enrichment for your accounts such as merchant info, transaction description cleanup, and more
+        title: Data Enrichment
       invite_code_settings:
         description: Every new user that joins your instance of Maybe can only do
           so via an invite code

db/migrate/20241212141453_add_merchant_logo.rb

@@ -0,0 +1,8 @@
+class AddMerchantLogo < ActiveRecord::Migration[7.2]
+  def change
+    add_column :merchants, :icon_url, :string
+    add_column :merchants, :enriched_at, :datetime
+
+    add_column :account_entries, :enriched_at, :datetime
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
+ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -48,6 +48,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
     t.text "notes"
     t.boolean "excluded", default: false
     t.string "plaid_id"
+    t.datetime "enriched_at"
     t.index ["account_id"], name: "index_account_entries_on_account_id"
     t.index ["import_id"], name: "index_account_entries_on_import_id"
     t.index ["transfer_id"], name: "index_account_entries_on_transfer_id"
@@ -452,6 +453,8 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do
     t.uuid "family_id", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "icon_url"
+    t.datetime "enriched_at"
     t.index ["family_id"], name: "index_merchants_on_family_id"
   end
 

test/jobs/enrich_data_job_test.rb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class EnrichDataJobTest < ActiveJob::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end