From fe199f235784a96d9b25efe846d87e6f5fb010e7 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Fri, 13 Dec 2024 17:22:27 -0500 Subject: [PATCH 001/558] Add account data enrichment (#1532) * Add data enrichment * Make data enrichment optional for self-hosters * Add categories to data enrichment * Only update category and merchant if nil * Fix name overrides * Lint fixes --- .../settings/hostings_controller.rb | 6 +- app/jobs/enrich_data_job.rb | 7 +++ app/models/account.rb | 8 +++ app/models/account/data_enricher.rb | 61 +++++++++++++++++++ app/models/account/syncer.rb | 6 ++ app/models/concerns/providable.rb | 6 +- app/models/provider/synth.rb | 31 ++++++++++ app/models/setting.rb | 4 ++ .../transactions/_transaction.html.erb | 12 ++-- app/views/merchants/_merchant.html.erb | 9 ++- .../_data_enrichment_settings.html.erb | 18 ++++++ app/views/settings/hostings/show.html.erb | 1 + config/locales/views/settings/hostings/en.yml | 3 + .../20241212141453_add_merchant_logo.rb | 8 +++ db/schema.rb | 5 +- test/jobs/enrich_data_job_test.rb | 7 +++ 16 files changed, 182 insertions(+), 10 deletions(-) create mode 100644 app/jobs/enrich_data_job.rb create mode 100644 app/models/account/data_enricher.rb create mode 100644 app/views/settings/hostings/_data_enrichment_settings.html.erb create mode 100644 db/migrate/20241212141453_add_merchant_logo.rb create mode 100644 test/jobs/enrich_data_job_test.rb diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb index 222ae018..97b8de92 100644 --- a/app/controllers/settings/hostings_controller.rb +++ b/app/controllers/settings/hostings_controller.rb @@ -26,6 +26,10 @@ class Settings::HostingsController < SettingsController Setting.synth_api_key = hosting_params[:synth_api_key] end + if hosting_params.key?(:data_enrichment_enabled) + Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled] + end + redirect_to settings_hosting_path, notice: t(".success") rescue ActiveRecord::RecordInvalid => error flash.now[:alert] = t(".failure") @@ -34,7 +38,7 @@ class Settings::HostingsController < SettingsController private def hosting_params - params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key) + params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled) end def raise_if_not_self_hosted diff --git a/app/jobs/enrich_data_job.rb b/app/jobs/enrich_data_job.rb new file mode 100644 index 00000000..97286b82 --- /dev/null +++ b/app/jobs/enrich_data_job.rb @@ -0,0 +1,7 @@ +class EnrichDataJob < ApplicationJob + queue_as :default + + def perform(account) + account.enrich_data + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 05931b7b..400e8bea 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -126,6 +126,14 @@ class Account < ApplicationRecord classification == "asset" ? "up" : "down" end + def enrich_data + DataEnricher.new(self).run + end + + def enrich_data_later + EnrichDataJob.perform_later(self) + end + def update_with_sync!(attributes) transaction do update!(attributes) diff --git a/app/models/account/data_enricher.rb b/app/models/account/data_enricher.rb new file mode 100644 index 00000000..4beb0b92 --- /dev/null +++ b/app/models/account/data_enricher.rb @@ -0,0 +1,61 @@ +class Account::DataEnricher + include Providable + + attr_reader :account + + def initialize(account) + @account = account + end + + def run + enrich_transactions + end + + private + def enrich_transactions + candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ]) + + Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}") + + merchants = {} + categories = {} + + candidates.each do |entry| + if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil? + begin + info = self.class.synth_provider.enrich_transaction(entry.name).info + + next unless info.present? + + if info.name.present? + merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name) + + if info.icon_url.present? + merchant.icon_url = info.icon_url + end + end + + if info.category.present? + category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category) + end + + entryable_attributes = { id: entry.entryable_id } + entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil? + entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil? + + Account.transaction do + merchant.save! if merchant.present? + category.save! if category.present? + entry.update!( + enriched_at: Time.current, + name: entry.enriched_at.nil? ? info.name : entry.name, + entryable_attributes: entryable_attributes + ) + end + rescue => e + Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}") + end + end + end + end +end diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb index d42ff431..9160e64f 100644 --- a/app/models/account/syncer.rb +++ b/app/models/account/syncer.rb @@ -10,6 +10,12 @@ class Account::Syncer account.reload update_account_info(balances, holdings) unless account.plaid_account_id.present? convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency + + if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed? + account.enrich_data_later + else + Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}") + end end private diff --git a/app/models/concerns/providable.rb b/app/models/concerns/providable.rb index 996efff8..4a8de8c0 100644 --- a/app/models/concerns/providable.rb +++ b/app/models/concerns/providable.rb @@ -23,8 +23,10 @@ module Providable end def synth_provider - api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"] - api_key.present? ? Provider::Synth.new(api_key) : nil + @synth_provider ||= begin + api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"] + api_key.present? ? Provider::Synth.new(api_key) : nil + end end private diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb index c212e992..b7735575 100644 --- a/app/models/provider/synth.rb +++ b/app/models/provider/synth.rb @@ -167,6 +167,35 @@ class Provider::Synth raw_response: response end + def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil) + params = { + description: description, + amount: amount, + date: date, + city: city, + state: state, + country: country + }.compact + + response = client.get("#{base_url}/enrich", params) + + parsed = JSON.parse(response.body) + + EnrichTransactionResponse.new \ + info: EnrichTransactionInfo.new( + name: parsed.dig("merchant"), + icon_url: parsed.dig("icon"), + category: parsed.dig("category") + ), + success?: true, + raw_response: response + rescue StandardError => error + EnrichTransactionResponse.new \ + success?: false, + error: error, + raw_response: error + end + private attr_reader :api_key @@ -177,6 +206,8 @@ class Provider::Synth UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true + EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true + EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true def base_url ENV["SYNTH_URL"] || "https://api.synthfinance.com" diff --git a/app/models/setting.rb b/app/models/setting.rb index d576fbea..eb1a9369 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -17,6 +17,10 @@ class Setting < RailsSettings::Base default: ENV.fetch("UPGRADES_TARGET", "release"), validates: { inclusion: { in: %w[release commit] } } + field :data_enrichment_enabled, + type: :boolean, + default: true + field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"] field :require_invite_for_signup, type: :boolean, default: false diff --git a/app/views/account/transactions/_transaction.html.erb b/app/views/account/transactions/_transaction.html.erb index dc2a026a..9a37a958 100644 --- a/app/views/account/transactions/_transaction.html.erb +++ b/app/views/account/transactions/_transaction.html.erb @@ -11,15 +11,17 @@
<%= content_tag :div, class: ["flex items-center gap-2"] do %> -
- <%= transaction.name.first.upcase %> -
+ <% if entry.account_transaction.merchant&.icon_url %> + <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %> + <% else %> + <%= render "shared/circle_logo", name: entry.name, size: "sm" %> + <% end %>
<% if entry.new_record? %> - <%= content_tag :p, transaction.name %> + <%= content_tag :p, entry.name %> <% else %> - <%= link_to transaction.name, + <%= link_to entry.name, entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> diff --git a/app/views/merchants/_merchant.html.erb b/app/views/merchants/_merchant.html.erb index 2b964140..a454d41a 100644 --- a/app/views/merchants/_merchant.html.erb +++ b/app/views/merchants/_merchant.html.erb @@ -2,7 +2,14 @@
- <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %> + <% if merchant.icon_url %> +
+ <%= image_tag merchant.icon_url, class: "w-8 h-8 rounded-full" %> +
+ <% else %> + <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %> + <% end %> +

<%= merchant.name %>

diff --git a/app/views/settings/hostings/_data_enrichment_settings.html.erb b/app/views/settings/hostings/_data_enrichment_settings.html.erb new file mode 100644 index 00000000..6d409923 --- /dev/null +++ b/app/views/settings/hostings/_data_enrichment_settings.html.erb @@ -0,0 +1,18 @@ +
+
+
+

<%= t(".title") %>

+

<%= t(".description") %>

+
+ + <%= styled_form_with model: Setting.new, + url: settings_hosting_path, + method: :patch, + data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %> +
+ <%= form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %> + <%= form.label :data_enrichment_enabled, " ".html_safe, class: "maybe-switch" %> +
+ <% end %> +
+
diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erb index ba4b7d5d..a2af0bed 100644 --- a/app/views/settings/hostings/show.html.erb +++ b/app/views/settings/hostings/show.html.erb @@ -10,6 +10,7 @@ <%= render "settings/hostings/upgrade_settings" %> <%= render "settings/hostings/provider_settings" %> <%= render "settings/hostings/synth_settings" %> + <%= render "settings/hostings/data_enrichment_settings" %>
<% end %> diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index 90a89fd7..6b34a6cd 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -2,6 +2,9 @@ en: settings: hostings: + data_enrichment_settings: + description: Enable data enrichment for your accounts such as merchant info, transaction description cleanup, and more + title: Data Enrichment invite_code_settings: description: Every new user that joins your instance of Maybe can only do so via an invite code diff --git a/db/migrate/20241212141453_add_merchant_logo.rb b/db/migrate/20241212141453_add_merchant_logo.rb new file mode 100644 index 00000000..81bd198c --- /dev/null +++ b/db/migrate/20241212141453_add_merchant_logo.rb @@ -0,0 +1,8 @@ +class AddMerchantLogo < ActiveRecord::Migration[7.2] + def change + add_column :merchants, :icon_url, :string + add_column :merchants, :enriched_at, :datetime + + add_column :account_entries, :enriched_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index 31183733..5fd3f26d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do +ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -48,6 +48,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do t.text "notes" t.boolean "excluded", default: false t.string "plaid_id" + t.datetime "enriched_at" t.index ["account_id"], name: "index_account_entries_on_account_id" t.index ["import_id"], name: "index_account_entries_on_import_id" t.index ["transfer_id"], name: "index_account_entries_on_transfer_id" @@ -452,6 +453,8 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_07_002408) do t.uuid "family_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "icon_url" + t.datetime "enriched_at" t.index ["family_id"], name: "index_merchants_on_family_id" end diff --git a/test/jobs/enrich_data_job_test.rb b/test/jobs/enrich_data_job_test.rb new file mode 100644 index 00000000..067767f6 --- /dev/null +++ b/test/jobs/enrich_data_job_test.rb @@ -0,0 +1,7 @@ +require "test_helper" + +class EnrichDataJobTest < ActiveJob::TestCase + # test "the truth" do + # assert true + # end +end From 913008995016c95e279c2c4842c5457175991388 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Mon, 16 Dec 2024 10:37:59 -0500 Subject: [PATCH 002/558] Make data enrichment opt-in --- app/models/setting.rb | 2 +- config/locales/views/settings/hostings/en.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/setting.rb b/app/models/setting.rb index eb1a9369..fe047cbb 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -19,7 +19,7 @@ class Setting < RailsSettings::Base field :data_enrichment_enabled, type: :boolean, - default: true + default: false field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"] diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index 6b34a6cd..dcd65f06 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -3,7 +3,7 @@ en: settings: hostings: data_enrichment_settings: - description: Enable data enrichment for your accounts such as merchant info, transaction description cleanup, and more + description: Enable data enrichment for your account transactions. This will incur additional Synth credits. title: Data Enrichment invite_code_settings: description: Every new user that joins your instance of Maybe can only do From 45add7512b000120a0e740f987ed0397b15296a1 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Mon, 16 Dec 2024 12:52:11 -0500 Subject: [PATCH 003/558] Handle nil name for entries (#1550) * Handle nil name for entries * Fix tests --- .erb-lint.yml => .erb_lint.yml | 0 app/models/account/data_enricher.rb | 4 ++- app/models/account/transaction.rb | 2 +- app/models/account/valuation.rb | 4 +++ .../transactions/_transaction.html.erb | 6 ++--- .../account/valuations/_valuation.html.erb | 4 +-- app/views/shared/_circle_logo.html.erb | 2 +- bin/erblint | 27 ------------------- .../locales/views/account/valuations/en.yml | 2 -- config/locales/views/settings/hostings/en.yml | 3 ++- 10 files changed, 16 insertions(+), 38 deletions(-) rename .erb-lint.yml => .erb_lint.yml (100%) delete mode 100755 bin/erblint diff --git a/.erb-lint.yml b/.erb_lint.yml similarity index 100% rename from .erb-lint.yml rename to .erb_lint.yml diff --git a/app/models/account/data_enricher.rb b/app/models/account/data_enricher.rb index 4beb0b92..e0615cc1 100644 --- a/app/models/account/data_enricher.rb +++ b/app/models/account/data_enricher.rb @@ -23,6 +23,8 @@ class Account::DataEnricher candidates.each do |entry| if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil? begin + next unless entry.name.present? + info = self.class.synth_provider.enrich_transaction(entry.name).info next unless info.present? @@ -48,7 +50,7 @@ class Account::DataEnricher category.save! if category.present? entry.update!( enriched_at: Time.current, - name: entry.enriched_at.nil? ? info.name : entry.name, + name: entry.enriched_at.nil? && info.name ? info.name : entry.name, entryable_attributes: entryable_attributes ) end diff --git a/app/models/account/transaction.rb b/app/models/account/transaction.rb index 91f24f70..fbf2aa9e 100644 --- a/app/models/account/transaction.rb +++ b/app/models/account/transaction.rb @@ -49,7 +49,7 @@ class Account::Transaction < ApplicationRecord end def name - entry.name || "(no description)" + entry.name || (entry.amount.positive? ? "Expense" : "Income") end def eod_balance diff --git a/app/models/account/valuation.rb b/app/models/account/valuation.rb index 93ebf5ff..5a4d1b8f 100644 --- a/app/models/account/valuation.rb +++ b/app/models/account/valuation.rb @@ -10,4 +10,8 @@ class Account::Valuation < ApplicationRecord false end end + + def name + "Balance update" + end end diff --git a/app/views/account/transactions/_transaction.html.erb b/app/views/account/transactions/_transaction.html.erb index 9a37a958..d63ddf8b 100644 --- a/app/views/account/transactions/_transaction.html.erb +++ b/app/views/account/transactions/_transaction.html.erb @@ -14,14 +14,14 @@ <% if entry.account_transaction.merchant&.icon_url %> <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %> <% else %> - <%= render "shared/circle_logo", name: entry.name, size: "sm" %> + <%= render "shared/circle_logo", name: transaction.name, size: "sm" %> <% end %>
<% if entry.new_record? %> - <%= content_tag :p, entry.name %> + <%= content_tag :p, transaction.name %> <% else %> - <%= link_to entry.name, + <%= link_to transaction.name, entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> diff --git a/app/views/account/valuations/_valuation.html.erb b/app/views/account/valuations/_valuation.html.erb index fc4c05d0..0761fd17 100644 --- a/app/views/account/valuations/_valuation.html.erb +++ b/app/views/account/valuations/_valuation.html.erb @@ -18,9 +18,9 @@
<% if entry.new_record? %> - <%= content_tag :p, entry.name %> + <%= content_tag :p, entry.entryable.name %> <% else %> - <%= link_to entry.name || t(".balance_update"), + <%= link_to entry.entryable.name, account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> diff --git a/app/views/shared/_circle_logo.html.erb b/app/views/shared/_circle_logo.html.erb index 17f1948a..e8dd7c1f 100644 --- a/app/views/shared/_circle_logo.html.erb +++ b/app/views/shared/_circle_logo.html.erb @@ -9,5 +9,5 @@ <%= tag.div style: mixed_hex_styles(hex), class: [size_classes[size], "flex shrink-0 items-center justify-center rounded-full"] do %> - <%= tag.span name[0].upcase, class: ["font-medium", size == "sm" ? "text-xs" : "text-sm"] %> + <%= tag.span (name.presence&.first || "T").upcase, class: ["font-medium", size == "sm" ? "text-xs" : "text-sm"] %> <% end %> diff --git a/bin/erblint b/bin/erblint deleted file mode 100755 index 63d662cf..00000000 --- a/bin/erblint +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'erblint' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) - -bundle_binstub = File.expand_path("bundle", __dir__) - -if File.file?(bundle_binstub) - if File.read(bundle_binstub, 300).include?("This file was generated by Bundler") - load(bundle_binstub) - else - abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. -Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") - end -end - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("erb_lint", "erblint") diff --git a/config/locales/views/account/valuations/en.yml b/config/locales/views/account/valuations/en.yml index ba8637c6..c157b6d6 100644 --- a/config/locales/views/account/valuations/en.yml +++ b/config/locales/views/account/valuations/en.yml @@ -2,8 +2,6 @@ en: account: valuations: - valuation: - balance_update: Balance update form: amount: Amount submit: Add balance update diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index dcd65f06..b3f04fc9 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -3,7 +3,8 @@ en: settings: hostings: data_enrichment_settings: - description: Enable data enrichment for your account transactions. This will incur additional Synth credits. + description: Enable data enrichment for your account transactions. This will + incur additional Synth credits. title: Data Enrichment invite_code_settings: description: Every new user that joins your instance of Maybe can only do From f7e86d4c90a51923b0bf918b3ef0356537fade13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:52:29 -0500 Subject: [PATCH 004/558] Bump rails from 7.2.2 to 7.2.2.1 (#1546) Bumps [rails](https://github.com/rails/rails) from 7.2.2 to 7.2.2.1. - [Release notes](https://github.com/rails/rails/releases) - [Commits](https://github.com/rails/rails/compare/v7.2.2...v7.2.2.1) --- updated-dependencies: - dependency-name: rails dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 134 +++++++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7e2318e0..88371eff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,29 +8,29 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (7.2.2) - actionpack (= 7.2.2) - activesupport (= 7.2.2) + actioncable (7.2.2.1) + actionpack (= 7.2.2.1) + activesupport (= 7.2.2.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.2.2) - actionpack (= 7.2.2) - activejob (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + actionmailbox (7.2.2.1) + actionpack (= 7.2.2.1) + activejob (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) mail (>= 2.8.0) - actionmailer (7.2.2) - actionpack (= 7.2.2) - actionview (= 7.2.2) - activejob (= 7.2.2) - activesupport (= 7.2.2) + actionmailer (7.2.2.1) + actionpack (= 7.2.2.1) + actionview (= 7.2.2.1) + activejob (= 7.2.2.1) + activesupport (= 7.2.2.1) mail (>= 2.8.0) rails-dom-testing (~> 2.2) - actionpack (7.2.2) - actionview (= 7.2.2) - activesupport (= 7.2.2) + actionpack (7.2.2.1) + actionview (= 7.2.2.1) + activesupport (= 7.2.2.1) nokogiri (>= 1.8.5) racc rack (>= 2.2.4, < 3.2) @@ -39,35 +39,35 @@ GEM rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) useragent (~> 0.16) - actiontext (7.2.2) - actionpack (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + actiontext (7.2.2.1) + actionpack (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.2.2) - activesupport (= 7.2.2) + actionview (7.2.2.1) + activesupport (= 7.2.2.1) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.2.2) - activesupport (= 7.2.2) + activejob (7.2.2.1) + activesupport (= 7.2.2.1) globalid (>= 0.3.6) - activemodel (7.2.2) - activesupport (= 7.2.2) - activerecord (7.2.2) - activemodel (= 7.2.2) - activesupport (= 7.2.2) + activemodel (7.2.2.1) + activesupport (= 7.2.2.1) + activerecord (7.2.2.1) + activemodel (= 7.2.2.1) + activesupport (= 7.2.2.1) timeout (>= 0.4.0) - activestorage (7.2.2) - actionpack (= 7.2.2) - activejob (= 7.2.2) - activerecord (= 7.2.2) - activesupport (= 7.2.2) + activestorage (7.2.2.1) + actionpack (= 7.2.2.1) + activejob (= 7.2.2.1) + activerecord (= 7.2.2.1) + activesupport (= 7.2.2.1) marcel (~> 1.0) - activesupport (7.2.2) + activesupport (7.2.2.1) base64 benchmark (>= 0.3) bigdecimal @@ -133,7 +133,7 @@ GEM rexml crass (1.0.6) csv (3.3.0) - date (3.4.0) + date (3.4.1) debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) @@ -218,7 +218,7 @@ GEM intercom-rails (1.0.1) activesupport (> 4.0) io-console (0.8.0) - irb (1.14.1) + irb (1.14.2) rdoc (>= 4.0.0) reline (>= 0.4.2) jmespath (1.6.2) @@ -234,7 +234,7 @@ GEM listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - logger (1.6.2) + logger (1.6.3) loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -254,7 +254,7 @@ GEM multipart-post (2.4.1) net-http (0.5.0) uri - net-imap (0.5.0) + net-imap (0.5.1) date net-protocol net-pop (0.1.2) @@ -264,17 +264,17 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.4) - nokogiri (1.17.0-aarch64-linux) + nokogiri (1.17.2-aarch64-linux) racc (~> 1.4) - nokogiri (1.17.0-arm-linux) + nokogiri (1.17.2-arm-linux) racc (~> 1.4) - nokogiri (1.17.0-arm64-darwin) + nokogiri (1.17.2-arm64-darwin) racc (~> 1.4) - nokogiri (1.17.0-x86-linux) + nokogiri (1.17.2-x86-linux) racc (~> 1.4) - nokogiri (1.17.0-x86_64-darwin) + nokogiri (1.17.2-x86_64-darwin) racc (~> 1.4) - nokogiri (1.17.0-x86_64-linux) + nokogiri (1.17.2-x86_64-linux) racc (~> 1.4) octokit (9.2.0) faraday (>= 1, < 3) @@ -309,25 +309,25 @@ GEM rack (>= 1.3) rackup (2.2.1) rack (>= 3) - rails (7.2.2) - actioncable (= 7.2.2) - actionmailbox (= 7.2.2) - actionmailer (= 7.2.2) - actionpack (= 7.2.2) - actiontext (= 7.2.2) - actionview (= 7.2.2) - activejob (= 7.2.2) - activemodel (= 7.2.2) - activerecord (= 7.2.2) - activestorage (= 7.2.2) - activesupport (= 7.2.2) + rails (7.2.2.1) + actioncable (= 7.2.2.1) + actionmailbox (= 7.2.2.1) + actionmailer (= 7.2.2.1) + actionpack (= 7.2.2.1) + actiontext (= 7.2.2.1) + actionview (= 7.2.2.1) + activejob (= 7.2.2.1) + activemodel (= 7.2.2.1) + activerecord (= 7.2.2.1) + activestorage (= 7.2.2.1) + activesupport (= 7.2.2.1) bundler (>= 1.15.0) - railties (= 7.2.2) + railties (= 7.2.2.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.1) + rails-html-sanitizer (1.6.2) loofah (~> 2.21) nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) rails-i18n (7.0.9) @@ -336,9 +336,9 @@ GEM rails-settings-cached (2.9.5) activerecord (>= 5.0.0) railties (>= 5.0.0) - railties (7.2.2) - actionpack (= 7.2.2) - activesupport (= 7.2.2) + railties (7.2.2.1) + actionpack (= 7.2.2.1) + activesupport (= 7.2.2.1) irb (~> 1.13) rackup (>= 1.0.0) rake (>= 12.2) @@ -351,7 +351,7 @@ GEM ffi (~> 1.0) rbs (3.6.1) logger - rdoc (6.8.1) + rdoc (6.9.0) psych (>= 4.0.0) redcarpet (3.6.0) regexp_parser (2.9.2) @@ -402,7 +402,7 @@ GEM sawyer (0.9.2) addressable (>= 2.3.5) faraday (>= 0.17.3, < 3) - securerandom (0.4.0) + securerandom (0.4.1) selenium-webdriver (4.27.0) base64 (~> 0.2) logger (~> 1.4) @@ -440,7 +440,7 @@ GEM terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) thor (1.3.2) - timeout (0.4.2) + timeout (0.4.3) turbo-rails (2.0.11) actionpack (>= 6.0.0) railties (>= 6.0.0) From f7ce2cdf8912bd28e3c8ff390a495df244ed9f90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:00:08 -0500 Subject: [PATCH 005/558] Bump mocha from 2.7.0 to 2.7.1 (#1544) Bumps [mocha](https://github.com/freerange/mocha) from 2.7.0 to 2.7.1. - [Changelog](https://github.com/freerange/mocha/blob/main/RELEASE.md) - [Commits](https://github.com/freerange/mocha/compare/v2.7.0...v2.7.1) --- updated-dependencies: - dependency-name: mocha dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 88371eff..90564356 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -248,7 +248,7 @@ GEM mini_magick (4.13.2) mini_mime (1.1.5) minitest (5.25.4) - mocha (2.7.0) + mocha (2.7.1) ruby2_keywords (>= 0.0.5) msgpack (1.7.2) multipart-post (2.4.1) From 0d09f2e3e91204a119bcc2573c5d123e7e0b2c6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:00:25 -0500 Subject: [PATCH 006/558] Bump csv from 3.3.0 to 3.3.1 (#1543) Bumps [csv](https://github.com/ruby/csv) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/ruby/csv/releases) - [Changelog](https://github.com/ruby/csv/blob/master/NEWS.md) - [Commits](https://github.com/ruby/csv/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: csv dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 90564356..bfddd858 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -132,7 +132,7 @@ GEM bigdecimal rexml crass (1.0.6) - csv (3.3.0) + csv (3.3.1) date (3.4.1) debug (1.9.2) irb (~> 1.10) From 54e46c1b4ed08470336b1e56827d4fd5606f51a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:01:05 -0500 Subject: [PATCH 007/558] Bump faraday from 2.12.1 to 2.12.2 (#1542) Bumps [faraday](https://github.com/lostisland/faraday) from 2.12.1 to 2.12.2. - [Release notes](https://github.com/lostisland/faraday/releases) - [Changelog](https://github.com/lostisland/faraday/blob/main/CHANGELOG.md) - [Commits](https://github.com/lostisland/faraday/compare/v2.12.1...v2.12.2) --- updated-dependencies: - dependency-name: faraday dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bfddd858..6560b704 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -155,7 +155,7 @@ GEM tzinfo faker (3.5.1) i18n (>= 1.8.11, < 2) - faraday (2.12.1) + faraday (2.12.2) faraday-net_http (>= 2.0, < 3.5) json logger @@ -222,7 +222,7 @@ GEM rdoc (>= 4.0.0) reline (>= 0.4.2) jmespath (1.6.2) - json (2.8.2) + json (2.9.0) jwt (2.9.3) base64 language_server-protocol (3.17.0.3) @@ -252,7 +252,7 @@ GEM ruby2_keywords (>= 0.0.5) msgpack (1.7.2) multipart-post (2.4.1) - net-http (0.5.0) + net-http (0.6.0) uri net-imap (0.5.1) date From bb9fa56addc52eea96684c318018d8b895347f9c Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Mon, 16 Dec 2024 13:21:30 -0500 Subject: [PATCH 008/558] Fix date format validation error (#1551) * Fix date format validation error * Order trades, fix flaky test --- app/helpers/application_helper.rb | 2 +- app/models/account/holding_calculator.rb | 2 +- app/models/family.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 8bf3cf28..24c962df 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -4,7 +4,7 @@ module ApplicationHelper def date_format_options [ [ "DD-MM-YYYY", "%d-%m-%Y" ], - [ "DD.MM.YY", "%d.%m.%Y" ], + [ "DD.MM.YYYY", "%d.%m.%Y" ], [ "MM-DD-YYYY", "%m-%d-%Y" ], [ "YYYY-MM-DD", "%Y-%m-%d" ], [ "DD/MM/YYYY", "%d/%m/%Y" ], diff --git a/app/models/account/holding_calculator.rb b/app/models/account/holding_calculator.rb index b1f6fc2f..5f6673de 100644 --- a/app/models/account/holding_calculator.rb +++ b/app/models/account/holding_calculator.rb @@ -98,7 +98,7 @@ class Account::HoldingCalculator end def trades - @trades ||= account.entries.includes(entryable: :security).account_trades.to_a + @trades ||= account.entries.includes(entryable: :security).account_trades.chronological.to_a end def portfolio_start_date diff --git a/app/models/family.rb b/app/models/family.rb index bae32843..69ac5eb7 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -1,7 +1,7 @@ class Family < ApplicationRecord include Plaidable, Syncable - DATE_FORMATS = [ "%m-%d-%Y", "%d-%m-%Y", "%Y-%m-%d", "%d/%m/%Y", "%Y/%m/%d", "%m/%d/%Y", "%e/%m/%Y", "%Y.%m.%d" ] + DATE_FORMATS = [ "%m-%d-%Y", "%d.%m.%Y", "%d-%m-%Y", "%Y-%m-%d", "%d/%m/%Y", "%Y/%m/%d", "%m/%d/%Y", "%e/%m/%Y", "%Y.%m.%d" ] include Providable From 7508ae55ac65afec524b8b2b3e16041f4dab8847 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:42:11 -0500 Subject: [PATCH 009/558] Bump dotenv-rails from 3.1.4 to 3.1.6 (#1540) Bumps [dotenv-rails](https://github.com/bkeepers/dotenv) from 3.1.4 to 3.1.6. - [Release notes](https://github.com/bkeepers/dotenv/releases) - [Changelog](https://github.com/bkeepers/dotenv/blob/main/Changelog.md) - [Commits](https://github.com/bkeepers/dotenv/compare/v3.1.4...v3.1.6) --- updated-dependencies: - dependency-name: dotenv-rails dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Gollwitzer --- Gemfile.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6560b704..a8ea5877 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -138,9 +138,9 @@ GEM irb (~> 1.10) reline (>= 0.3.8) docile (1.4.0) - dotenv (3.1.4) - dotenv-rails (3.1.4) - dotenv (= 3.1.4) + dotenv (3.1.6) + dotenv-rails (3.1.6) + dotenv (= 3.1.6) railties (>= 6.1) drb (2.2.1) erb_lint (0.7.0) @@ -351,11 +351,11 @@ GEM ffi (~> 1.0) rbs (3.6.1) logger - rdoc (6.9.0) + rdoc (6.9.1) psych (>= 4.0.0) redcarpet (3.6.0) regexp_parser (2.9.2) - reline (0.5.12) + reline (0.6.0) io-console (~> 0.5) rexml (3.3.9) rubocop (1.67.0) From ae30176816de5080c68e1d2a58d399d2ad0c1983 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:42:27 -0500 Subject: [PATCH 010/558] Bump aws-sdk-s3 from 1.176.0 to 1.176.1 (#1545) Bumps [aws-sdk-s3](https://github.com/aws/aws-sdk-ruby) from 1.176.0 to 1.176.1. - [Release notes](https://github.com/aws/aws-sdk-ruby/releases) - [Changelog](https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-s3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-ruby/commits) --- updated-dependencies: - dependency-name: aws-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Gollwitzer --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a8ea5877..2843a0c1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -83,7 +83,7 @@ GEM public_suffix (>= 2.0.2, < 7.0) ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.1018.0) + aws-partitions (1.1023.0) aws-sdk-core (3.214.0) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.992.0) @@ -92,7 +92,7 @@ GEM aws-sdk-kms (1.96.0) aws-sdk-core (~> 3, >= 3.210.0) aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.176.0) + aws-sdk-s3 (1.176.1) aws-sdk-core (~> 3, >= 3.210.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.5) From 6034dfe5f5347a0d8f1d6b9b083bf223ba86520f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 13:42:36 -0500 Subject: [PATCH 011/558] Bump good_job from 4.5.1 to 4.6.0 (#1541) Bumps [good_job](https://github.com/bensheldon/good_job) from 4.5.1 to 4.6.0. - [Release notes](https://github.com/bensheldon/good_job/releases) - [Changelog](https://github.com/bensheldon/good_job/blob/main/CHANGELOG.md) - [Commits](https://github.com/bensheldon/good_job/compare/v4.5.1...v4.6.0) --- updated-dependencies: - dependency-name: good_job dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Gollwitzer --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2843a0c1..0cfab904 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -176,7 +176,7 @@ GEM raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) - good_job (4.5.1) + good_job (4.6.0) activejob (>= 6.1.0) activerecord (>= 6.1.0) concurrent-ruby (>= 1.3.1) From ba878c3d8b392dce46c1bf8dab14bf7ff4733f7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:00:08 -0500 Subject: [PATCH 012/558] Bump rails-settings-cached from 2.9.5 to 2.9.6 (#1547) Bumps [rails-settings-cached](https://github.com/huacnlee/rails-settings-cached) from 2.9.5 to 2.9.6. - [Release notes](https://github.com/huacnlee/rails-settings-cached/releases) - [Changelog](https://github.com/huacnlee/rails-settings-cached/blob/main/CHANGELOG.md) - [Commits](https://github.com/huacnlee/rails-settings-cached/compare/v2.9.5...v2.9.6) --- updated-dependencies: - dependency-name: rails-settings-cached dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Zach Gollwitzer --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0cfab904..1a80b26c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -333,7 +333,7 @@ GEM rails-i18n (7.0.9) i18n (>= 0.7, < 2) railties (>= 6.0.0, < 8) - rails-settings-cached (2.9.5) + rails-settings-cached (2.9.6) activerecord (>= 5.0.0) railties (>= 5.0.0) railties (7.2.2.1) From 68617514b004e71b8750f9506de2dc2a9629992a Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Tue, 17 Dec 2024 09:58:08 -0500 Subject: [PATCH 013/558] Make transaction enrichment opt-in for all users (#1552) --- .../settings/hostings_controller.rb | 6 +---- app/controllers/users_controller.rb | 2 +- app/models/account/syncer.rb | 2 +- app/models/setting.rb | 4 ---- .../_data_enrichment_settings.html.erb | 18 -------------- app/views/settings/hostings/show.html.erb | 1 - .../_data_enrichment_settings.html.erb | 24 +++++++++++++++++++ app/views/settings/preferences/show.html.erb | 5 ++++ config/locales/views/settings/en.yml | 7 ++++++ config/locales/views/settings/hostings/en.yml | 4 ---- .../20241217141716_add_enrichment_setting.rb | 5 ++++ db/schema.rb | 3 ++- 12 files changed, 46 insertions(+), 35 deletions(-) delete mode 100644 app/views/settings/hostings/_data_enrichment_settings.html.erb create mode 100644 app/views/settings/preferences/_data_enrichment_settings.html.erb create mode 100644 db/migrate/20241217141716_add_enrichment_setting.rb diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb index 97b8de92..222ae018 100644 --- a/app/controllers/settings/hostings_controller.rb +++ b/app/controllers/settings/hostings_controller.rb @@ -26,10 +26,6 @@ class Settings::HostingsController < SettingsController Setting.synth_api_key = hosting_params[:synth_api_key] end - if hosting_params.key?(:data_enrichment_enabled) - Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled] - end - redirect_to settings_hosting_path, notice: t(".success") rescue ActiveRecord::RecordInvalid => error flash.now[:alert] = t(".failure") @@ -38,7 +34,7 @@ class Settings::HostingsController < SettingsController private def hosting_params - params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled) + params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key) end def raise_if_not_self_hosted diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index beb85197..55b75581 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -41,7 +41,7 @@ class UsersController < ApplicationController def user_params params.require(:user).permit( :first_name, :last_name, :profile_image, :redirect_to, :delete_profile_image, :onboarded_at, - family_attributes: [ :name, :currency, :country, :locale, :date_format, :timezone, :id ] + family_attributes: [ :name, :currency, :country, :locale, :date_format, :timezone, :id, :data_enrichment_enabled ] ) end diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb index 9160e64f..df5a7b03 100644 --- a/app/models/account/syncer.rb +++ b/app/models/account/syncer.rb @@ -11,7 +11,7 @@ class Account::Syncer update_account_info(balances, holdings) unless account.plaid_account_id.present? convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency - if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed? + if account.family.data_enrichment_enabled? account.enrich_data_later else Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}") diff --git a/app/models/setting.rb b/app/models/setting.rb index fe047cbb..d576fbea 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -17,10 +17,6 @@ class Setting < RailsSettings::Base default: ENV.fetch("UPGRADES_TARGET", "release"), validates: { inclusion: { in: %w[release commit] } } - field :data_enrichment_enabled, - type: :boolean, - default: false - field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"] field :require_invite_for_signup, type: :boolean, default: false diff --git a/app/views/settings/hostings/_data_enrichment_settings.html.erb b/app/views/settings/hostings/_data_enrichment_settings.html.erb deleted file mode 100644 index 6d409923..00000000 --- a/app/views/settings/hostings/_data_enrichment_settings.html.erb +++ /dev/null @@ -1,18 +0,0 @@ -
-
-
-

<%= t(".title") %>

-

<%= t(".description") %>

-
- - <%= styled_form_with model: Setting.new, - url: settings_hosting_path, - method: :patch, - data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %> -
- <%= form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %> - <%= form.label :data_enrichment_enabled, " ".html_safe, class: "maybe-switch" %> -
- <% end %> -
-
diff --git a/app/views/settings/hostings/show.html.erb b/app/views/settings/hostings/show.html.erb index a2af0bed..ba4b7d5d 100644 --- a/app/views/settings/hostings/show.html.erb +++ b/app/views/settings/hostings/show.html.erb @@ -10,7 +10,6 @@ <%= render "settings/hostings/upgrade_settings" %> <%= render "settings/hostings/provider_settings" %> <%= render "settings/hostings/synth_settings" %> - <%= render "settings/hostings/data_enrichment_settings" %>
<% end %> diff --git a/app/views/settings/preferences/_data_enrichment_settings.html.erb b/app/views/settings/preferences/_data_enrichment_settings.html.erb new file mode 100644 index 00000000..73f729fd --- /dev/null +++ b/app/views/settings/preferences/_data_enrichment_settings.html.erb @@ -0,0 +1,24 @@ +<%# locals: (user:) %> + +
+
+
+

<%= t(".title") %>

+

<%= t(".description") %>

+ <% if self_hosted? %> +

<%= t(".self_host_disclaimer") %>

+ <% end %> +
+ + <%= styled_form_with model: user, + data: { controller: "auto-submit-form", "auto-submit-form-trigger-event-value": "blur" } do |form| %> +
+ <%= form.hidden_field :redirect_to, value: "preferences" %> + <%= form.fields_for :family do |family_form| %> + <%= family_form.check_box :data_enrichment_enabled, class: "sr-only peer", "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "input" %> + <%= family_form.label :data_enrichment_enabled, " ".html_safe, class: "maybe-switch" %> + <% end %> +
+ <% end %> +
+
diff --git a/app/views/settings/preferences/show.html.erb b/app/views/settings/preferences/show.html.erb index 225e5d96..8b066503 100644 --- a/app/views/settings/preferences/show.html.erb +++ b/app/views/settings/preferences/show.html.erb @@ -40,6 +40,11 @@ <% end %>
<% end %> + + <%= settings_section title: t(".data"), subtitle: t(".data_subtitle") do %> + <%= render "settings/preferences/data_enrichment_settings", user: @user %> + <% end %> + <%= settings_section title: t(".theme_title"), subtitle: t(".theme_subtitle") do %>
<%= styled_form_with model: @user, class: "flex justify-between items-center" do |form| %> diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index aa8141b8..f784db97 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -26,9 +26,16 @@ en: next: Next previous: Back preferences: + data_enrichment_settings: + description: Let Maybe auto-categorize, name, and add merchant data to your + transactions on each sync. All enrichment is done in English. + self_host_disclaimer: This will incur Synth API credits. + title: Transaction enrichment (English only) show: country: Country currency: Currency + data: Data enrichment + data_subtitle: Enable data enrichment for your accounts date_format: Date format general_subtitle: Configure your preferences general_title: General diff --git a/config/locales/views/settings/hostings/en.yml b/config/locales/views/settings/hostings/en.yml index b3f04fc9..90a89fd7 100644 --- a/config/locales/views/settings/hostings/en.yml +++ b/config/locales/views/settings/hostings/en.yml @@ -2,10 +2,6 @@ en: settings: hostings: - data_enrichment_settings: - description: Enable data enrichment for your account transactions. This will - incur additional Synth credits. - title: Data Enrichment invite_code_settings: description: Every new user that joins your instance of Maybe can only do so via an invite code diff --git a/db/migrate/20241217141716_add_enrichment_setting.rb b/db/migrate/20241217141716_add_enrichment_setting.rb new file mode 100644 index 00000000..8d154887 --- /dev/null +++ b/db/migrate/20241217141716_add_enrichment_setting.rb @@ -0,0 +1,5 @@ +class AddEnrichmentSetting < ActiveRecord::Migration[7.2] + def change + add_column :families, :data_enrichment_enabled, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 5fd3f26d..5ae96e52 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do +ActiveRecord::Schema[7.2].define(version: 2024_12_17_141716) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -222,6 +222,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_12_141453) do t.string "country", default: "US" t.datetime "last_synced_at" t.string "timezone" + t.boolean "data_enrichment_enabled", default: false end create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| From 7be6a372bf8648c1a432c6c7d1738fba368acbd6 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Thu, 19 Dec 2024 10:16:09 -0500 Subject: [PATCH 014/558] Preserve original transaction names when enriching (#1556) * Preserve original transaction name * Remove stale method * Fix tests --- .../concerns/entryable_resource.rb | 2 +- app/models/account.rb | 1 + app/models/account/data_enricher.rb | 2 +- app/models/account/entry.rb | 14 ++---- app/models/account/syncer.rb | 3 +- app/models/account/trade.rb | 5 -- app/models/account/trade_builder.rb | 4 ++ app/models/account/transaction.rb | 4 -- app/models/account/transfer.rb | 4 +- app/models/account/valuation.rb | 4 -- app/models/demo/generator.rb | 1 + app/views/account/trades/_trade.html.erb | 6 +-- .../transactions/_transaction.html.erb | 14 +++--- app/views/account/transactions/show.html.erb | 3 +- app/views/account/valuations/_form.html.erb | 1 + .../account/valuations/_valuation.html.erb | 4 +- config/brakeman.ignore | 48 +++++++++---------- .../20241218132503_add_enriched_name_field.rb | 37 ++++++++++++++ db/schema.rb | 5 +- .../accountable_resource_interface_test.rb | 2 +- test/models/account/entry_test.rb | 6 --- test/system/trades_test.rb | 6 +-- 22 files changed, 100 insertions(+), 76 deletions(-) create mode 100644 db/migrate/20241218132503_add_enriched_name_field.rb diff --git a/app/controllers/concerns/entryable_resource.rb b/app/controllers/concerns/entryable_resource.rb index 84aac1d4..918b32bb 100644 --- a/app/controllers/concerns/entryable_resource.rb +++ b/app/controllers/concerns/entryable_resource.rb @@ -119,7 +119,7 @@ module EntryableResource def entry_params params.require(:account_entry).permit( - :account_id, :name, :date, :amount, :currency, :excluded, :notes, :nature, + :account_id, :name, :enriched_name, :date, :amount, :currency, :excluded, :notes, :nature, entryable_attributes: self.class.permitted_entryable_attributes ) end diff --git a/app/models/account.rb b/app/models/account.rb index 400e8bea..19d883ca 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -151,6 +151,7 @@ class Account < ApplicationRecord else entries.create! \ date: Date.current, + name: "Balance update", amount: balance, currency: currency, entryable: Account::Valuation.new diff --git a/app/models/account/data_enricher.rb b/app/models/account/data_enricher.rb index e0615cc1..0be57dc1 100644 --- a/app/models/account/data_enricher.rb +++ b/app/models/account/data_enricher.rb @@ -50,7 +50,7 @@ class Account::DataEnricher category.save! if category.present? entry.update!( enriched_at: Time.current, - name: entry.enriched_at.nil? && info.name ? info.name : entry.name, + enriched_name: info.name, entryable_attributes: entryable_attributes ) end diff --git a/app/models/account/entry.rb b/app/models/account/entry.rb index 1801fb9e..4d7334fb 100644 --- a/app/models/account/entry.rb +++ b/app/models/account/entry.rb @@ -10,7 +10,7 @@ class Account::Entry < ApplicationRecord delegated_type :entryable, types: Account::Entryable::TYPES, dependent: :destroy accepts_nested_attributes_for :entryable - validates :date, :amount, :currency, presence: true + validates :date, :name, :amount, :currency, presence: true validates :date, uniqueness: { scope: [ :account_id, :entryable_type ] }, if: -> { account_valuation? } validates :date, comparison: { greater_than: -> { min_supported_date } } @@ -47,14 +47,6 @@ class Account::Entry < ApplicationRecord account.sync_later(start_date: sync_start_date) end - def inflow? - amount <= 0 && account_transaction? - end - - def outflow? - amount > 0 && account_transaction? - end - def entryable_name_short entryable_type.demodulize.underscore end @@ -63,6 +55,10 @@ class Account::Entry < ApplicationRecord Account::BalanceTrendCalculator.new(self, entries, balances).trend end + def display_name + enriched_name.presence || name + end + class << self # arbitrary cutoff date to avoid expensive sync operations def min_supported_date diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb index df5a7b03..5b2e4aba 100644 --- a/app/models/account/syncer.rb +++ b/app/models/account/syncer.rb @@ -11,7 +11,8 @@ class Account::Syncer update_account_info(balances, holdings) unless account.plaid_account_id.present? convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency - if account.family.data_enrichment_enabled? + # Enrich if user opted in or if we're syncing transactions from a Plaid account + if account.family.data_enrichment_enabled? || account.plaid_account_id.present? account.enrich_data_later else Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}") diff --git a/app/models/account/trade.rb b/app/models/account/trade.rb index b8ebd7b8..70b0c8f3 100644 --- a/app/models/account/trade.rb +++ b/app/models/account/trade.rb @@ -26,11 +26,6 @@ class Account::Trade < ApplicationRecord qty > 0 end - def name - prefix = sell? ? "Sell " : "Buy " - prefix + "#{qty.abs} shares of #{security.ticker}" - end - def unrealized_gain_loss return nil if sell? current_price = security.current_price diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rb index 191d8100..e62947f7 100644 --- a/app/models/account/trade_builder.rb +++ b/app/models/account/trade_builder.rb @@ -31,7 +31,11 @@ class Account::TradeBuilder end def build_trade + prefix = type == "sell" ? "Sell " : "Buy " + trade_name = prefix + "#{qty.to_i.abs} shares of #{security.ticker}" + account.entries.new( + name: trade_name, date: date, amount: signed_amount, currency: currency, diff --git a/app/models/account/transaction.rb b/app/models/account/transaction.rb index fbf2aa9e..6b8f4995 100644 --- a/app/models/account/transaction.rb +++ b/app/models/account/transaction.rb @@ -48,10 +48,6 @@ class Account::Transaction < ApplicationRecord end end - def name - entry.name || (entry.amount.positive? ? "Expense" : "Income") - end - def eod_balance entry.amount_money end diff --git a/app/models/account/transfer.rb b/app/models/account/transfer.rb index 174576e8..ea908413 100644 --- a/app/models/account/transfer.rb +++ b/app/models/account/transfer.rb @@ -33,11 +33,11 @@ class Account::Transfer < ApplicationRecord end def inflow_transaction - entries.find { |e| e.inflow? } + entries.find { |e| e.amount.negative? } end def outflow_transaction - entries.find { |e| e.outflow? } + entries.find { |e| e.amount.positive? } end def update_entries!(params) diff --git a/app/models/account/valuation.rb b/app/models/account/valuation.rb index 5a4d1b8f..93ebf5ff 100644 --- a/app/models/account/valuation.rb +++ b/app/models/account/valuation.rb @@ -10,8 +10,4 @@ class Account::Valuation < ApplicationRecord false end end - - def name - "Balance update" - end end diff --git a/app/models/demo/generator.rb b/app/models/demo/generator.rb index 29985a36..36e86200 100644 --- a/app/models/demo/generator.rb +++ b/app/models/demo/generator.rb @@ -303,6 +303,7 @@ class Demo::Generator date: date, amount: amount, currency: "USD", + name: "Balance update", entryable: Account::Valuation.new end diff --git a/app/views/account/trades/_trade.html.erb b/app/views/account/trades/_trade.html.erb index 2215cd94..8c5c924e 100644 --- a/app/views/account/trades/_trade.html.erb +++ b/app/views/account/trades/_trade.html.erb @@ -13,14 +13,14 @@
<%= tag.div class: ["flex items-center gap-2"] do %>
- <%= trade.name.first.upcase %> + <%= entry.display_name.first.upcase %>
<% if entry.new_record? %> - <%= content_tag :p, trade.name %> + <%= content_tag :p, entry.display_name %> <% else %> - <%= link_to trade.name, + <%= link_to entry.display_name, account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> diff --git a/app/views/account/transactions/_transaction.html.erb b/app/views/account/transactions/_transaction.html.erb index d63ddf8b..dfe0955a 100644 --- a/app/views/account/transactions/_transaction.html.erb +++ b/app/views/account/transactions/_transaction.html.erb @@ -11,17 +11,17 @@
<%= content_tag :div, class: ["flex items-center gap-2"] do %> - <% if entry.account_transaction.merchant&.icon_url %> - <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %> + <% if transaction.merchant&.icon_url %> + <%= image_tag transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %> <% else %> - <%= render "shared/circle_logo", name: transaction.name, size: "sm" %> + <%= render "shared/circle_logo", name: entry.display_name, size: "sm" %> <% end %>
<% if entry.new_record? %> - <%= content_tag :p, transaction.name %> + <%= content_tag :p, entry.display_name %> <% else %> - <%= link_to transaction.name, + <%= link_to entry.display_name, entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> @@ -41,7 +41,7 @@ <% end %>
- <%= render "account/transfers/account_logos", transfer: entry.transfer, outflow: entry.outflow? %> + <%= render "account/transfers/account_logos", transfer: entry.transfer, outflow: entry.amount.positive? %>
<% else %>
@@ -65,7 +65,7 @@
<%= content_tag :p, format_money(-entry.amount_money), - class: ["text-green-600": entry.inflow?] %> + class: ["text-green-600": entry.amount.negative?] %>
<% if balance_trend %> diff --git a/app/views/account/transactions/show.html.erb b/app/views/account/transactions/show.html.erb index ecce8921..b512b43a 100644 --- a/app/views/account/transactions/show.html.erb +++ b/app/views/account/transactions/show.html.erb @@ -9,7 +9,8 @@ url: account_transaction_path(@entry), class: "space-y-2", data: { controller: "auto-submit-form" } do |f| %> - <%= f.text_field :name, + + <%= f.text_field @entry.enriched_at.present? ? :enriched_name : :name, label: t(".name_label"), "data-auto-submit-form-target": "auto" %> diff --git a/app/views/account/valuations/_form.html.erb b/app/views/account/valuations/_form.html.erb index b56e847b..f3f0aa5f 100644 --- a/app/views/account/valuations/_form.html.erb +++ b/app/views/account/valuations/_form.html.erb @@ -8,6 +8,7 @@ <% end %>
+ <%= form.hidden_field :name, value: "Balance update" %> <%= form.date_field :date, label: true, required: true, value: Date.current, min: Account::Entry.min_supported_date, max: Date.current %> <%= form.money_field :amount, label: t(".amount"), required: true %>
diff --git a/app/views/account/valuations/_valuation.html.erb b/app/views/account/valuations/_valuation.html.erb index 0761fd17..3e34cfba 100644 --- a/app/views/account/valuations/_valuation.html.erb +++ b/app/views/account/valuations/_valuation.html.erb @@ -18,9 +18,9 @@
<% if entry.new_record? %> - <%= content_tag :p, entry.entryable.name %> + <%= content_tag :p, entry.display_name %> <% else %> - <%= link_to entry.entryable.name, + <%= link_to entry.display_name, account_entry_path(entry), data: { turbo_frame: "drawer", turbo_prefetch: false }, class: "hover:underline hover:text-gray-800" %> diff --git a/config/brakeman.ignore b/config/brakeman.ignore index 6ebccac2..ce280c4d 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -23,6 +23,29 @@ ], "note": "" }, + { + "warning_type": "Mass Assignment", + "warning_code": 105, + "fingerprint": "5bfdb129316655dc4e02f3a599156660414a6562212a5f61057d376f6888f078", + "check_name": "PermitAttributes", + "message": "Potentially dangerous key allowed for mass assignment", + "file": "app/controllers/concerns/entryable_resource.rb", + "line": 122, + "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", + "code": "params.require(:account_entry).permit(:account_id, :name, :enriched_name, :date, :amount, :currency, :excluded, :notes, :nature, :entryable_attributes => self.class.permitted_entryable_attributes)", + "render_path": null, + "location": { + "type": "method", + "class": "EntryableResource", + "method": "entry_params" + }, + "user_input": ":account_id", + "confidence": "High", + "cwe_id": [ + 915 + ], + "note": "" + }, { "warning_type": "Mass Assignment", "warning_code": 105, @@ -80,29 +103,6 @@ ], "note": "" }, - { - "warning_type": "Mass Assignment", - "warning_code": 105, - "fingerprint": "f158202dcc66f2273ddea5e5296bad7146a50ca6667f49c77372b5b234542334", - "check_name": "PermitAttributes", - "message": "Potentially dangerous key allowed for mass assignment", - "file": "app/controllers/concerns/entryable_resource.rb", - "line": 122, - "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", - "code": "params.require(:account_entry).permit(:account_id, :name, :date, :amount, :currency, :excluded, :notes, :nature, :entryable_attributes => self.class.permitted_entryable_attributes)", - "render_path": null, - "location": { - "type": "method", - "class": "EntryableResource", - "method": "entry_params" - }, - "user_input": ":account_id", - "confidence": "High", - "cwe_id": [ - 915 - ], - "note": "" - }, { "warning_type": "Dynamic Render Path", "warning_code": 15, @@ -138,6 +138,6 @@ "note": "" } ], - "updated": "2024-11-27 15:33:53 -0500", + "updated": "2024-12-18 17:46:13 -0500", "brakeman_version": "6.2.2" } diff --git a/db/migrate/20241218132503_add_enriched_name_field.rb b/db/migrate/20241218132503_add_enriched_name_field.rb new file mode 100644 index 00000000..08d837c4 --- /dev/null +++ b/db/migrate/20241218132503_add_enriched_name_field.rb @@ -0,0 +1,37 @@ +class AddEnrichedNameField < ActiveRecord::Migration[7.2] + def change + add_column :account_entries, :enriched_name, :string + + reversible do |dir| + dir.up do + execute <<-SQL + UPDATE account_entries ae + SET name = CASE ae.entryable_type + WHEN 'Account::Trade' THEN + CASE + WHEN EXISTS ( + SELECT 1 FROM account_trades t + WHERE t.id = ae.entryable_id AND t.qty < 0 + ) THEN 'Sell trade' + ELSE 'Buy trade' + END + WHEN 'Account::Transaction' THEN + CASE + WHEN ae.amount > 0 THEN 'Expense' + ELSE 'Income' + END + WHEN 'Account::Valuation' THEN 'Balance update' + ELSE 'Unknown entry' + END + WHERE name IS NULL + SQL + + change_column_null :account_entries, :name, false + end + + dir.down do + change_column_null :account_entries, :name, true + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 5ae96e52..7beb5097 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_12_17_141716) do +ActiveRecord::Schema[7.2].define(version: 2024_12_18_132503) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -39,7 +39,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_17_141716) do t.decimal "amount", precision: 19, scale: 4 t.string "currency" t.date "date" - t.string "name" + t.string "name", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false t.uuid "transfer_id" @@ -49,6 +49,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_12_17_141716) do t.boolean "excluded", default: false t.string "plaid_id" t.datetime "enriched_at" + t.string "enriched_name" t.index ["account_id"], name: "index_account_entries_on_account_id" t.index ["import_id"], name: "index_account_entries_on_import_id" t.index ["transfer_id"], name: "index_account_entries_on_transfer_id" diff --git a/test/interfaces/accountable_resource_interface_test.rb b/test/interfaces/accountable_resource_interface_test.rb index c1fe9208..d44baf7e 100644 --- a/test/interfaces/accountable_resource_interface_test.rb +++ b/test/interfaces/accountable_resource_interface_test.rb @@ -76,7 +76,7 @@ module AccountableResourceInterfaceTest end test "updates account balance by editing existing valuation for today" do - @account.entries.create! date: Date.current, amount: 6000, currency: "USD", entryable: Account::Valuation.new + @account.entries.create! date: Date.current, amount: 6000, currency: "USD", name: "Balance update", entryable: Account::Valuation.new assert_no_difference [ "Account::Entry.count", "Account::Valuation.count" ] do patch account_url(@account), params: { diff --git a/test/models/account/entry_test.rb b/test/models/account/entry_test.rb index 9c541b6b..5a9d9ec9 100644 --- a/test/models/account/entry_test.rb +++ b/test/models/account/entry_test.rb @@ -90,10 +90,4 @@ class Account::EntryTest < ActiveSupport::TestCase assert_equal Money.new(-200), family.entries.income_total("USD") end - - # See: https://github.com/maybe-finance/maybe/wiki/vision#signage-of-money - test "transactions with negative amounts are inflows, positive amounts are outflows to an account" do - assert create_transaction(amount: -10).inflow? - assert create_transaction(amount: 10).outflow? - end end diff --git a/test/system/trades_test.rb b/test/system/trades_test.rb index 9e26b708..9ca85c08 100644 --- a/test/system/trades_test.rb +++ b/test/system/trades_test.rb @@ -23,7 +23,7 @@ class TradesTest < ApplicationSystemTestCase end test "can create buy transaction" do - shares_qty = 25.0 + shares_qty = 25 open_new_trade_modal @@ -38,7 +38,7 @@ class TradesTest < ApplicationSystemTestCase visit_account_trades within_trades do - assert_text "Buy 10.0 shares of AAPL" + assert_text "Purchase 10 shares of AAPL" assert_text "Buy #{shares_qty} shares of AAPL" end end @@ -60,7 +60,7 @@ class TradesTest < ApplicationSystemTestCase visit_account_trades within_trades do - assert_text "Sell #{aapl.qty} shares of AAPL" + assert_text "Sell #{aapl.qty.round} shares of AAPL" end end From a4d10097d579cc383057b4d857a724cf22761106 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Fri, 20 Dec 2024 11:24:46 -0500 Subject: [PATCH 015/558] Preserve pagination on entry updates (#1563) * Preserve pagination on entry updates * Test fix --- app/helpers/application_helper.rb | 20 +++++++++++++++++ app/models/concerns/providable.rb | 6 ++--- app/views/account/entries/index.html.erb | 2 +- app/views/accounts/show/_activity.html.erb | 2 +- app/views/application/_pagination.html.erb | 26 ++++++++++++++-------- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 24c962df..85aca138 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -166,4 +166,24 @@ module ApplicationHelper cookies[:admin] == "true" end + + def custom_pagy_url_for(pagy, page, current_path: nil) + if current_path.blank? + pagy_url_for(pagy, page) + else + uri = URI.parse(current_path) + params = URI.decode_www_form(uri.query || "").to_h + + # Delete existing page param if it exists + params.delete("page") + # Add new page param unless it's page 1 + params["page"] = page unless page == 1 + + if params.empty? + uri.path + else + "#{uri.path}?#{URI.encode_www_form(params)}" + end + end + end end diff --git a/app/models/concerns/providable.rb b/app/models/concerns/providable.rb index 4a8de8c0..996efff8 100644 --- a/app/models/concerns/providable.rb +++ b/app/models/concerns/providable.rb @@ -23,10 +23,8 @@ module Providable end def synth_provider - @synth_provider ||= begin - api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"] - api_key.present? ? Provider::Synth.new(api_key) : nil - end + api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"] + api_key.present? ? Provider::Synth.new(api_key) : nil end private diff --git a/app/views/account/entries/index.html.erb b/app/views/account/entries/index.html.erb index c659d97f..31197675 100644 --- a/app/views/account/entries/index.html.erb +++ b/app/views/account/entries/index.html.erb @@ -84,7 +84,7 @@
- <%= render "pagination", pagy: @pagy %> + <%= render "pagination", pagy: @pagy, current_path: account_path(@account, page: params[:page]) %>
<% end %> diff --git a/app/views/accounts/show/_activity.html.erb b/app/views/accounts/show/_activity.html.erb index c041b652..290c5be5 100644 --- a/app/views/accounts/show/_activity.html.erb +++ b/app/views/accounts/show/_activity.html.erb @@ -1,5 +1,5 @@ <%# locals: (account:) %> -<%= turbo_frame_tag dom_id(account, :entries), src: account_entries_path(account_id: account.id) do %> +<%= turbo_frame_tag dom_id(account, :entries), src: account_entries_path(account_id: account.id, page: params[:page]) do %> <%= render "account/entries/loading" %> <% end %> diff --git a/app/views/application/_pagination.html.erb b/app/views/application/_pagination.html.erb index dee086be..f4548512 100644 --- a/app/views/application/_pagination.html.erb +++ b/app/views/application/_pagination.html.erb @@ -1,9 +1,11 @@ -<%# locals: (pagy:) %> +<%# locals: (pagy:, current_path: nil) %> From 77def1db404457c32e46797e68a420cc53819216 Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Fri, 20 Dec 2024 11:37:26 -0500 Subject: [PATCH 016/558] Nested Categories (#1561) * Prepare entry search for nested categories * Subcategory implementation * Remove caching for test stability --- app/controllers/categories_controller.rb | 17 +++- app/controllers/registrations_controller.rb | 1 - app/controllers/transactions_controller.rb | 10 +-- app/models/account/entry.rb | 46 +--------- app/models/account/entry_search.rb | 59 +++++++++++++ app/models/account/trade.rb | 20 +---- app/models/account/transaction.rb | 47 +---------- app/models/account/transaction_search.rb | 42 ++++++++++ app/models/account/valuation.rb | 10 --- app/models/category.rb | 84 +++++++++++++------ app/models/demo/generator.rb | 5 ++ app/views/account/trades/_header.html.erb | 4 +- app/views/categories/_badge.html.erb | 2 +- app/views/categories/_category.html.erb | 6 +- app/views/categories/_form.html.erb | 13 ++- app/views/categories/_menu.html.erb | 2 +- app/views/categories/edit.html.erb | 2 +- app/views/categories/index.html.erb | 28 +++++-- app/views/categories/new.html.erb | 2 +- app/views/category/dropdowns/_row.html.erb | 3 + app/views/category/dropdowns/show.html.erb | 17 +++- config/locales/models/transaction/en.yml | 12 --- config/locales/views/categories/en.yml | 13 ++- config/routes.rb | 2 + .../20241219174803_add_parent_category.rb | 6 ++ db/schema.rb | 6 +- .../controllers/categories_controller_test.rb | 14 +++- .../registrations_controller_test.rb | 9 -- test/fixtures/categories.yml | 7 +- test/models/account/entry_test.rb | 4 - test/models/category_test.rb | 38 +++------ 31 files changed, 297 insertions(+), 234 deletions(-) create mode 100644 app/models/account/entry_search.rb create mode 100644 app/models/account/transaction_search.rb delete mode 100644 config/locales/models/transaction/en.yml create mode 100644 db/migrate/20241219174803_add_parent_category.rb diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index ddc1ecaa..2d1882f4 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -10,6 +10,7 @@ class CategoriesController < ApplicationController def new @category = Current.family.categories.new color: Category::COLORS.sample + @categories = Current.family.categories.alphabetically.where(parent_id: nil).where.not(id: @category.id) end def create @@ -17,19 +18,21 @@ class CategoriesController < ApplicationController if @category.save @transaction.update(category_id: @category.id) if @transaction - redirect_back_or_to transactions_path, notice: t(".success") + + redirect_back_or_to categories_path, notice: t(".success") else - redirect_back_or_to transactions_path, alert: t(".failure", error: @category.errors.full_messages.to_sentence) + render :new, status: :unprocessable_entity end end def edit + @categories = Current.family.categories.alphabetically.where(parent_id: nil).where.not(id: @category.id) end def update @category.update! category_params - redirect_back_or_to transactions_path, notice: t(".success") + redirect_back_or_to categories_path, notice: t(".success") end def destroy @@ -38,6 +41,12 @@ class CategoriesController < ApplicationController redirect_back_or_to categories_path, notice: t(".success") end + def bootstrap + Current.family.categories.bootstrap_defaults + + redirect_back_or_to categories_path, notice: t(".success") + end + private def set_category @category = Current.family.categories.find(params[:id]) @@ -50,6 +59,6 @@ class CategoriesController < ApplicationController end def category_params - params.require(:category).permit(:name, :color) + params.require(:category).permit(:name, :color, :parent_id) end end diff --git a/app/controllers/registrations_controller.rb b/app/controllers/registrations_controller.rb index 128b309c..0d8d6e92 100644 --- a/app/controllers/registrations_controller.rb +++ b/app/controllers/registrations_controller.rb @@ -24,7 +24,6 @@ class RegistrationsController < ApplicationController if @user.save @invitation&.update!(accepted_at: Time.current) - Category.create_default_categories(@user.family) unless @invitation @session = create_session_for(@user) redirect_to root_path, notice: t(".success") else diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index acceab79..664c3080 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -3,13 +3,13 @@ class TransactionsController < ApplicationController def index @q = search_params - result = Current.family.entries.account_transactions.search(@q).reverse_chronological - @pagy, @transaction_entries = pagy(result, limit: params[:per_page] || "50") + search_query = Current.family.transactions.search(@q).includes(:entryable).reverse_chronological + @pagy, @transaction_entries = pagy(search_query, limit: params[:per_page] || "50") @totals = { - count: result.select { |t| t.currency == Current.family.currency }.count, - income: result.income_total(Current.family.currency).abs, - expense: result.expense_total(Current.family.currency) + count: search_query.select { |t| t.currency == Current.family.currency }.count, + income: search_query.income_total(Current.family.currency).abs, + expense: search_query.expense_total(Current.family.currency) } end diff --git a/app/models/account/entry.rb b/app/models/account/entry.rb index 4d7334fb..8d6a8f40 100644 --- a/app/models/account/entry.rb +++ b/app/models/account/entry.rb @@ -60,6 +60,10 @@ class Account::Entry < ApplicationRecord end class << self + def search(params) + Account::EntrySearch.new(params).build_query(all) + end + # arbitrary cutoff date to avoid expensive sync operations def min_supported_date 30.years.ago.to_date @@ -141,49 +145,7 @@ class Account::Entry < ApplicationRecord Money.new(total, currency) end - def search(params) - query = all - query = query.where("account_entries.name ILIKE ?", "%#{sanitize_sql_like(params[:search])}%") if params[:search].present? - query = query.where("account_entries.date >= ?", params[:start_date]) if params[:start_date].present? - query = query.where("account_entries.date <= ?", params[:end_date]) if params[:end_date].present? - - if params[:types].present? - query = query.where(marked_as_transfer: false) unless params[:types].include?("transfer") - - if params[:types].include?("income") && !params[:types].include?("expense") - query = query.where("account_entries.amount < 0") - elsif params[:types].include?("expense") && !params[:types].include?("income") - query = query.where("account_entries.amount >= 0") - end - end - - if params[:amount].present? && params[:amount_operator].present? - case params[:amount_operator] - when "equal" - query = query.where("ABS(ABS(account_entries.amount) - ?) <= 0.01", params[:amount].to_f.abs) - when "less" - query = query.where("ABS(account_entries.amount) < ?", params[:amount].to_f.abs) - when "greater" - query = query.where("ABS(account_entries.amount) > ?", params[:amount].to_f.abs) - end - end - - if params[:accounts].present? || params[:account_ids].present? - query = query.joins(:account) - end - - query = query.where(accounts: { name: params[:accounts] }) if params[:accounts].present? - query = query.where(accounts: { id: params[:account_ids] }) if params[:account_ids].present? - - # Search attributes on each entryable to further refine results - entryable_ids = entryable_search(params) - query = query.where(entryable_id: entryable_ids) unless entryable_ids.nil? - - query - end - private - def entryable_search(params) entryable_ids = [] entryable_search_performed = false diff --git a/app/models/account/entry_search.rb b/app/models/account/entry_search.rb new file mode 100644 index 00000000..c561765b --- /dev/null +++ b/app/models/account/entry_search.rb @@ -0,0 +1,59 @@ +class Account::EntrySearch + include ActiveModel::Model + include ActiveModel::Attributes + + attribute :search, :string + attribute :amount, :string + attribute :amount_operator, :string + attribute :types, :string + attribute :accounts, :string + attribute :account_ids, :string + attribute :start_date, :string + attribute :end_date, :string + + class << self + def from_entryable_search(entryable_search) + new(entryable_search.attributes.slice(*attribute_names)) + end + end + + def build_query(scope) + query = scope + + query = query.where("account_entries.name ILIKE :search OR account_entries.enriched_name ILIKE :search", + search: "%#{ActiveRecord::Base.sanitize_sql_like(search)}%" + ) if search.present? + query = query.where("account_entries.date >= ?", start_date) if start_date.present? + query = query.where("account_entries.date <= ?", end_date) if end_date.present? + + if types.present? + query = query.where(marked_as_transfer: false) unless types.include?("transfer") + + if types.include?("income") && !types.include?("expense") + query = query.where("account_entries.amount < 0") + elsif types.include?("expense") && !types.include?("income") + query = query.where("account_entries.amount >= 0") + end + end + + if amount.present? && amount_operator.present? + case amount_operator + when "equal" + query = query.where("ABS(ABS(account_entries.amount) - ?) <= 0.01", amount.to_f.abs) + when "less" + query = query.where("ABS(account_entries.amount) < ?", amount.to_f.abs) + when "greater" + query = query.where("ABS(account_entries.amount) > ?", amount.to_f.abs) + end + end + + if accounts.present? || account_ids.present? + query = query.joins(:account) + end + + query = query.where(accounts: { name: accounts }) if accounts.present? + query = query.where(accounts: { id: account_ids }) if account_ids.present? + + query + end +end diff --git a/app/models/account/trade.rb b/app/models/account/trade.rb index 70b0c8f3..7d4976ba 100644 --- a/app/models/account/trade.rb +++ b/app/models/account/trade.rb @@ -8,26 +8,8 @@ class Account::Trade < ApplicationRecord validates :qty, presence: true validates :price, :currency, presence: true - class << self - def search(_params) - all - end - - def requires_search?(_params) - false - end - end - - def sell? - qty < 0 - end - - def buy? - qty > 0 - end - def unrealized_gain_loss - return nil if sell? + return nil if qty.negative? current_price = security.current_price return nil if current_price.nil? diff --git a/app/models/account/transaction.rb b/app/models/account/transaction.rb index 6b8f4995..afe5a568 100644 --- a/app/models/account/transaction.rb +++ b/app/models/account/transaction.rb @@ -12,52 +12,7 @@ class Account::Transaction < ApplicationRecord class << self def search(params) - query = all - if params[:categories].present? - if params[:categories].exclude?("Uncategorized") - query = query - .joins(:category) - .where(categories: { name: params[:categories] }) - else - query = query - .left_joins(:category) - .where(categories: { name: params[:categories] }) - .or(query.where(category_id: nil)) - end - end - - query = query.joins(:merchant).where(merchants: { name: params[:merchants] }) if params[:merchants].present? - - if params[:tags].present? - query = query.joins(:tags) - .where(tags: { name: params[:tags] }) - .distinct - end - - query + Account::TransactionSearch.new(params).build_query(all) end - - def requires_search?(params) - searchable_keys.any? { |key| params.key?(key) } - end - - private - - def searchable_keys - %i[categories merchants tags] - end end - - def eod_balance - entry.amount_money - end - - private - def account - entry.account - end - - def daily_transactions - account.entries.account_transactions - end end diff --git a/app/models/account/transaction_search.rb b/app/models/account/transaction_search.rb new file mode 100644 index 00000000..f61fae69 --- /dev/null +++ b/app/models/account/transaction_search.rb @@ -0,0 +1,42 @@ +class Account::TransactionSearch + include ActiveModel::Model + include ActiveModel::Attributes + + attribute :search, :string + attribute :amount, :string + attribute :amount_operator, :string + attribute :types, array: true + attribute :accounts, array: true + attribute :account_ids, array: true + attribute :start_date, :string + attribute :end_date, :string + attribute :categories, array: true + attribute :merchants, array: true + attribute :tags, array: true + + # Returns array of Account::Entry objects to stay consistent with partials, which only deal with Account::Entry + def build_query(scope) + query = scope + + if categories.present? + if categories.exclude?("Uncategorized") + query = query + .joins(:category) + .where(categories: { name: categories }) + else + query = query + .left_joins(:category) + .where(categories: { name: categories }) + .or(query.where(category_id: nil)) + end + end + + query = query.joins(:merchant).where(merchants: { name: merchants }) if merchants.present? + + query = query.joins(:tags).where(tags: { name: tags }) if tags.present? + + entries_scope = Account::Entry.account_transactions.where(entryable_id: query.select(:id)) + + Account::EntrySearch.from_entryable_search(self).build_query(entries_scope) + end +end diff --git a/app/models/account/valuation.rb b/app/models/account/valuation.rb index 93ebf5ff..219ecd90 100644 --- a/app/models/account/valuation.rb +++ b/app/models/account/valuation.rb @@ -1,13 +1,3 @@ class Account::Valuation < ApplicationRecord include Account::Entryable - - class << self - def search(_params) - all - end - - def requires_search?(_params) - false - end - end end diff --git a/app/models/category.rb b/app/models/category.rb index 4a2d6361..6f50070b 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,12 +1,16 @@ class Category < ApplicationRecord has_many :transactions, dependent: :nullify, class_name: "Account::Transaction" has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping" + belongs_to :family + has_many :subcategories, class_name: "Category", foreign_key: :parent_id + belongs_to :parent, class_name: "Category", optional: true + validates :name, :color, :family, presence: true validates :name, uniqueness: { scope: :family_id } - before_update :clear_internal_category, if: :name_changed? + validate :category_level_limit scope :alphabetically, -> { order(:name) } @@ -14,30 +18,55 @@ class Category < ApplicationRecord UNCATEGORIZED_COLOR = "#737373" - DEFAULT_CATEGORIES = [ - { internal_category: "income", color: COLORS[0] }, - { internal_category: "food_and_drink", color: COLORS[1] }, - { internal_category: "entertainment", color: COLORS[2] }, - { internal_category: "personal_care", color: COLORS[3] }, - { internal_category: "general_services", color: COLORS[4] }, - { internal_category: "auto_and_transport", color: COLORS[5] }, - { internal_category: "rent_and_utilities", color: COLORS[6] }, - { internal_category: "home_improvement", color: COLORS[7] } - ] + class Group + attr_reader :category, :subcategories - def self.create_default_categories(family) - if family.categories.size > 0 - raise ArgumentError, "Family already has some categories" + delegate :name, :color, to: :category + + def self.for(categories) + categories.select { |category| category.parent_id.nil? }.map do |category| + new(category, category.subcategories) + end end - family_id = family.id - categories = self::DEFAULT_CATEGORIES.map { |c| { - name: I18n.t("transaction.default_category.#{c[:internal_category]}"), - internal_category: c[:internal_category], - color: c[:color], - family_id: - } } - self.insert_all(categories) + def initialize(category, subcategories = nil) + @category = category + @subcategories = subcategories || [] + end + end + + class << self + def bootstrap_defaults + default_categories.each do |name, color| + find_or_create_by!(name: name) do |category| + category.color = color + end + end + end + + private + def default_categories + [ + [ "Income", "#e99537" ], + [ "Loan Payments", "#6471eb" ], + [ "Bank Fees", "#db5a54" ], + [ "Entertainment", "#df4e92" ], + [ "Food & Drink", "#c44fe9" ], + [ "Groceries", "#eb5429" ], + [ "Dining Out", "#61c9ea" ], + [ "General Merchandise", "#805dee" ], + [ "Clothing & Accessories", "#6ad28a" ], + [ "Electronics", "#e99537" ], + [ "Healthcare", "#4da568" ], + [ "Insurance", "#6471eb" ], + [ "Utilities", "#db5a54" ], + [ "Transportation", "#df4e92" ], + [ "Gas & Fuel", "#c44fe9" ], + [ "Education", "#eb5429" ], + [ "Charitable Donations", "#61c9ea" ], + [ "Subscriptions", "#805dee" ] + ] + end end def replace_and_destroy!(replacement) @@ -47,9 +76,14 @@ class Category < ApplicationRecord end end - private + def subcategory? + parent.present? + end - def clear_internal_category - self.internal_category = nil + private + def category_level_limit + if subcategory? && parent.subcategory? + errors.add(:parent, "can't have more than 2 levels of subcategories") + end end end diff --git a/app/models/demo/generator.rb b/app/models/demo/generator.rb index 36e86200..d26b85c7 100644 --- a/app/models/demo/generator.rb +++ b/app/models/demo/generator.rb @@ -90,6 +90,11 @@ class Demo::Generator categories.each do |category| family.categories.create!(name: category, color: COLORS.sample) end + + food = family.categories.find_by(name: "Food & Drink") + family.categories.create!(name: "Restaurants", parent_category: food) + family.categories.create!(name: "Groceries", parent_category: food) + family.categories.create!(name: "Alcohol & Bars", parent_category: food) end def create_merchants! diff --git a/app/views/account/trades/_header.html.erb b/app/views/account/trades/_header.html.erb index 7ccadfa4..b89028af 100644 --- a/app/views/account/trades/_header.html.erb +++ b/app/views/account/trades/_header.html.erb @@ -34,7 +34,7 @@
<%= trade.security.ticker %>
- <% if trade.buy? %> + <% if trade.qty.positive? %>
<%= t(".purchase_qty_label") %>
<%= trade.qty.abs %>
@@ -53,7 +53,7 @@
<% end %> - <% if trade.buy? && trade.unrealized_gain_loss.present? %> + <% if trade.qty.positive? && trade.unrealized_gain_loss.present? %>
<%= t(".total_return_label") %>
diff --git a/app/views/categories/_badge.html.erb b/app/views/categories/_badge.html.erb index a9752262..1b4c399b 100644 --- a/app/views/categories/_badge.html.erb +++ b/app/views/categories/_badge.html.erb @@ -2,7 +2,7 @@ <% category ||= null_category %>
- " class="flex justify-between items-center p-4 bg-white"> +
<%= "pb-4" unless category.subcategories.any? %> bg-white">
+ <% if category.subcategory? %> + <%= lucide_icon "corner-down-right", class: "shrink-0 w-5 h-5 text-gray-400 ml-2" %> + <% end %> + <%= render partial: "categories/badge", locals: { category: category } %>
diff --git a/app/views/categories/_form.html.erb b/app/views/categories/_form.html.erb index 313e48ba..2bca2191 100644 --- a/app/views/categories/_form.html.erb +++ b/app/views/categories/_form.html.erb @@ -1,9 +1,12 @@ +<%# locals: (category:, categories:) %> +
<%= styled_form_with model: category, class: "space-y-4", data: { turbo_frame: :_top } do |f| %>
<%= render partial: "shared/color_avatar", locals: { name: category.name, color: category.color } %>
+
<% Category::COLORS.each do |color| %> <% end %>
-
- <%= f.text_field :name, placeholder: t(".placeholder"), required: true, autofocus: true, data: { color_avatar_target: "name" } %> + + <% if category.errors.any? %> + <%= render "shared/form_errors", model: category %> + <% end %> + +
+ <%= f.text_field :name, placeholder: t(".placeholder"), required: true, autofocus: true, label: "Name", data: { color_avatar_target: "name" } %> + <%= f.select :parent_id, categories.pluck(:name, :id), { include_blank: "(unassigned)", label: "Parent category (optional)" } %>
diff --git a/app/views/categories/_menu.html.erb b/app/views/categories/_menu.html.erb index 746bcb45..5a4627b3 100644 --- a/app/views/categories/_menu.html.erb +++ b/app/views/categories/_menu.html.erb @@ -5,7 +5,7 @@ <%= render partial: "categories/badge", locals: { category: transaction.category } %>