diff --git a/app/assets/stylesheets/application.tailwind.css b/app/assets/stylesheets/application.tailwind.css index 110f31d0..33711536 100644 --- a/app/assets/stylesheets/application.tailwind.css +++ b/app/assets/stylesheets/application.tailwind.css @@ -35,7 +35,7 @@ } .form-field__submit { - @apply w-full cursor-pointer rounded-lg bg-black p-3 text-center text-white hover:bg-gray-700; + @apply cursor-pointer rounded-lg bg-black p-3 text-center text-white hover:bg-gray-700; } input:checked+label+.toggle-switch-dot { @@ -100,11 +100,15 @@ } .btn { - @apply px-3 py-2 rounded-lg text-sm font-medium; + @apply px-3 py-2 rounded-lg text-sm font-medium cursor-pointer; } .btn--primary { - @apply bg-gray-900 text-white hover:bg-gray-700; + @apply bg-gray-900 text-white hover:bg-gray-700 disabled:bg-gray-50 disabled:hover:bg-gray-50 disabled:text-gray-400; + } + + .btn--secondary { + @apply bg-gray-50 hover:bg-gray-100 text-gray-900; } .btn--outline { diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 11b55ac6..4ecc9b27 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -23,7 +23,10 @@ class AccountsController < ApplicationController end def new - @account = Account.new(accountable: Accountable.from_type(params[:type])&.new) + @account = Account.new( + accountable: Accountable.from_type(params[:type])&.new, + currency: Current.family.currency + ) @account.accountable.address = Address.new if @account.accountable.is_a?(Property) diff --git a/app/controllers/import/cleans_controller.rb b/app/controllers/import/cleans_controller.rb new file mode 100644 index 00000000..a8bda133 --- /dev/null +++ b/app/controllers/import/cleans_controller.rb @@ -0,0 +1,22 @@ +class Import::CleansController < ApplicationController + layout "imports" + + before_action :set_import + + def show + redirect_to import_configuration_path(@import), alert: "Please configure your import before proceeding." unless @import.configured? + + rows = @import.rows.ordered + + if params[:view] == "errors" + rows = rows.reject { |row| row.valid? } + end + + @pagy, @rows = pagy_array(rows, limit: params[:per_page] || "10") + end + + private + def set_import + @import = Current.family.imports.find(params[:import_id]) + end +end diff --git a/app/controllers/import/configurations_controller.rb b/app/controllers/import/configurations_controller.rb new file mode 100644 index 00000000..b399f683 --- /dev/null +++ b/app/controllers/import/configurations_controller.rb @@ -0,0 +1,25 @@ +class Import::ConfigurationsController < ApplicationController + layout "imports" + + before_action :set_import + + def show + end + + def update + @import.update!(import_params) + @import.generate_rows_from_csv + @import.reload.sync_mappings + + redirect_to import_clean_path(@import), notice: "Import configured successfully." + end + + private + def set_import + @import = Current.family.imports.find(params[:import_id]) + end + + def import_params + params.require(:import).permit(:date_col_label, :date_format, :name_col_label, :category_col_label, :tags_col_label, :amount_col_label, :signage_convention, :account_col_label, :notes_col_label, :entity_type_col_label) + end +end diff --git a/app/controllers/import/confirms_controller.rb b/app/controllers/import/confirms_controller.rb new file mode 100644 index 00000000..0c1a8872 --- /dev/null +++ b/app/controllers/import/confirms_controller.rb @@ -0,0 +1,14 @@ +class Import::ConfirmsController < ApplicationController + layout "imports" + + before_action :set_import + + def show + redirect_to import_clean_path(@import), alert: "You have invalid data, please edit until all errors are resolved" unless @import.cleaned? + end + + private + def set_import + @import = Current.family.imports.find(params[:import_id]) + end +end diff --git a/app/controllers/import/mappings_controller.rb b/app/controllers/import/mappings_controller.rb new file mode 100644 index 00000000..098c4010 --- /dev/null +++ b/app/controllers/import/mappings_controller.rb @@ -0,0 +1,43 @@ +class Import::MappingsController < ApplicationController + before_action :set_import + + def update + mapping = @import.mappings.find(params[:id]) + + mapping.update! \ + create_when_empty: create_when_empty, + mappable: mappable, + value: mapping_params[:value] + + redirect_back_or_to import_confirm_path(@import) + end + + private + def mapping_params + params.require(:import_mapping).permit(:type, :key, :mappable_id, :mappable_type, :value) + end + + def set_import + @import = Current.family.imports.find(params[:import_id]) + end + + def mappable + return nil unless mappable_class.present? + + @mappable ||= mappable_class.find_by(id: mapping_params[:mappable_id], family: Current.family) + end + + def create_when_empty + return false unless mapping_class.present? + + mapping_params[:mappable_id] == mapping_class::CREATE_NEW_KEY + end + + def mappable_class + mapping_params[:mappable_type]&.constantize + end + + def mapping_class + mapping_params[:type]&.constantize + end +end diff --git a/app/controllers/import/rows_controller.rb b/app/controllers/import/rows_controller.rb new file mode 100644 index 00000000..b5b9092c --- /dev/null +++ b/app/controllers/import/rows_controller.rb @@ -0,0 +1,24 @@ +class Import::RowsController < ApplicationController + before_action :set_import_row + + def update + @row.assign_attributes(row_params) + @row.save!(validate: false) + @row.sync_mappings + + redirect_to import_row_path(@row.import, @row) + end + + def show + end + + private + def row_params + params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes) + end + + def set_import_row + @import = Current.family.imports.find(params[:import_id]) + @row = @import.rows.find(params[:id]) + end +end diff --git a/app/controllers/import/uploads_controller.rb b/app/controllers/import/uploads_controller.rb new file mode 100644 index 00000000..e89988ac --- /dev/null +++ b/app/controllers/import/uploads_controller.rb @@ -0,0 +1,47 @@ +class Import::UploadsController < ApplicationController + layout "imports" + + before_action :set_import + + def show + end + + def update + if csv_valid?(csv_str) + @import.assign_attributes(raw_file_str: csv_str, col_sep: upload_params[:col_sep]) + @import.save!(validate: false) + + redirect_to import_configuration_path(@import), notice: "CSV uploaded successfully." + else + flash.now[:alert] = "Must be valid CSV with headers and at least one row of data" + + render :show, status: :unprocessable_entity + end + end + + private + def set_import + @import = Current.family.imports.find(params[:import_id]) + end + + def csv_str + @csv_str ||= upload_params[:csv_file]&.read || upload_params[:raw_file_str] + end + + def csv_valid?(str) + require "csv" + + begin + csv = CSV.parse(str || "", headers: true) + return false if csv.headers.empty? + return false if csv.count == 0 + true + rescue CSV::MalformedCSVError + false + end + end + + def upload_params + params.require(:import).permit(:raw_file_str, :csv_file, :col_sep) + end +end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index c2268eec..ef374f6c 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -1,118 +1,44 @@ -require "ostruct" - class ImportsController < ApplicationController - before_action :set_import, except: %i[index new create] + before_action :set_import, only: %i[show publish destroy] + + def publish + @import.publish_later + + redirect_to import_path(@import), notice: "Your import has started in the background." + end def index @imports = Current.family.imports - render layout: "with_sidebar" + + render layout: with_sidebar end def new - account = Current.family.accounts.find_by(id: params[:account_id]) - @import = Import.new account: account - end - - def edit - end - - def update - account = Current.family.accounts.find(params[:import][:account_id]) - @import.update! account: account, col_sep: params[:import][:col_sep] - - redirect_to load_import_path(@import), notice: t(".import_updated") + @pending_import = Current.family.imports.ordered.pending.first end def create - account = Current.family.accounts.find(params[:import][:account_id]) - @import = Import.create! account: account, col_sep: params[:import][:col_sep] + import = Current.family.imports.create! import_params - redirect_to load_import_path(@import), notice: t(".import_created") + redirect_to import_upload_path(import) + end + + def show + redirect_to import_confirm_path(@import), alert: "Please finalize your mappings before proceeding." unless @import.publishable? end def destroy - @import.destroy! - redirect_to imports_url, notice: t(".import_destroyed"), status: :see_other - end + @import.destroy - def load - end - - def upload_csv - begin - @import.raw_file_str = import_params[:raw_file_str].read - rescue NoMethodError - flash.now[:alert] = "Please select a file to upload" - render :load, status: :unprocessable_entity and return - end - if @import.save - redirect_to configure_import_path(@import), notice: t(".import_loaded") - else - flash.now[:alert] = @import.errors.full_messages.to_sentence - render :load, status: :unprocessable_entity - end - end - - def load_csv - if @import.update(import_params) - redirect_to configure_import_path(@import), notice: t(".import_loaded") - else - flash.now[:alert] = @import.errors.full_messages.to_sentence - render :load, status: :unprocessable_entity - end - end - - def configure - unless @import.loaded? - redirect_to load_import_path(@import), alert: t(".invalid_csv") - end - end - - def update_mappings - @import.update! import_params(@import.expected_fields.map(&:key)) - redirect_to clean_import_path(@import), notice: t(".column_mappings_saved") - end - - def clean - unless @import.loaded? - redirect_to load_import_path(@import), alert: t(".invalid_csv") - end - end - - def update_csv - update_params = import_params[:csv_update] - - @import.update_csv! \ - row_idx: update_params[:row_idx], - col_idx: update_params[:col_idx], - value: update_params[:value] - - render :clean - end - - def confirm - unless @import.cleaned? - redirect_to clean_import_path(@import), alert: t(".invalid_data") - end - end - - def publish - if @import.valid? - @import.publish_later - redirect_to imports_path, notice: t(".import_published") - else - flash.now[:error] = t(".invalid_data") - render :confirm, status: :unprocessable_entity - end + redirect_to imports_path, notice: "Your import has been deleted." end private - def set_import @import = Current.family.imports.find(params[:id]) end - def import_params(permitted_mappings = nil) - params.require(:import).permit(:raw_file_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ]) + def import_params + params.require(:import).permit(:type) end end diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index 0b924d36..a202a697 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -1,19 +1,63 @@ module ImportsHelper - def table_corner_class(row_idx, col_idx, rows, cols) - return "rounded-tl-xl" if row_idx == 0 && col_idx == 0 - return "rounded-tr-xl" if row_idx == 0 && col_idx == cols.size - 1 - return "rounded-bl-xl" if row_idx == rows.size - 1 && col_idx == 0 - return "rounded-br-xl" if row_idx == rows.size - 1 && col_idx == cols.size - 1 - "" + def mapping_label(mapping_class) + { + "Import::AccountTypeMapping" => "Account Type", + "Import::AccountMapping" => "Account", + "Import::CategoryMapping" => "Category", + "Import::TagMapping" => "Tag" + }.fetch(mapping_class.name) end - def nav_steps(import = Import.new) - [ - { name: "Select", complete: import.persisted?, path: import.persisted? ? edit_import_path(import) : new_import_path }, - { name: "Import", complete: import.loaded?, path: import.persisted? ? load_import_path(import) : nil }, - { name: "Setup", complete: import.configured?, path: import.persisted? ? configure_import_path(import) : nil }, - { name: "Clean", complete: import.cleaned?, path: import.persisted? ? clean_import_path(import) : nil }, - { name: "Confirm", complete: import.complete?, path: import.persisted? ? confirm_import_path(import) : nil } - ] + def import_col_label(key) + { + date: "Date", + amount: "Amount", + name: "Name", + currency: "Currency", + category: "Category", + tags: "Tags", + account: "Account", + notes: "Notes", + qty: "Quantity", + ticker: "Ticker", + price: "Price", + entity_type: "Type" + }[key] end + + def dry_run_resource(key) + map = { + transactions: DryRunResource.new(label: "Transactions", icon: "credit-card", text_class: "text-cyan-500", bg_class: "bg-cyan-500/5"), + accounts: DryRunResource.new(label: "Accounts", icon: "layers", text_class: "text-orange-500", bg_class: "bg-orange-500/5"), + categories: DryRunResource.new(label: "Categories", icon: "shapes", text_class: "text-blue-500", bg_class: "bg-blue-500/5"), + tags: DryRunResource.new(label: "Tags", icon: "tags", text_class: "text-violet-500", bg_class: "bg-violet-500/5") + } + + map[key] + end + + def permitted_import_configuration_path(import) + if permitted_import_types.include?(import.type.underscore) + "import/configurations/#{import.type.underscore}" + else + raise "Unknown import type: #{import.type}" + end + end + + def cell_class(row, field) + base = "text-sm focus:ring-gray-900 focus:border-gray-900 w-full max-w-full disabled:text-gray-400" + + row.valid? # populate errors + + border = row.errors.key?(field) ? "border-red-500" : "border-transparent" + + [ base, border ].join(" ") + end + + private + def permitted_import_types + %w[transaction_import trade_import account_import mint_import] + end + + DryRunResource = Struct.new(:label, :icon, :text_class, :bg_class, keyword_init: true) end diff --git a/app/helpers/styled_form_builder.rb b/app/helpers/styled_form_builder.rb index 6da18c39..e9e27538 100644 --- a/app/helpers/styled_form_builder.rb +++ b/app/helpers/styled_form_builder.rb @@ -49,7 +49,7 @@ class StyledFormBuilder < ActionView::Helpers::FormBuilder end def submit(value = nil, options = {}) - merged_options = { class: "form-field__submit" }.merge(options) + merged_options = { class: "btn btn--primary w-full" }.merge(options) value, options = nil, value if value.is_a?(Hash) super(value, merged_options) end diff --git a/app/javascript/controllers/import_upload_controller.js b/app/javascript/controllers/import_upload_controller.js deleted file mode 100644 index 224b7cdb..00000000 --- a/app/javascript/controllers/import_upload_controller.js +++ /dev/null @@ -1,98 +0,0 @@ -import { Controller } from "@hotwired/stimulus" - -export default class extends Controller { - static targets = ["input", "preview", "submit", "filename", "filesize"] - static values = { - acceptedTypes: Array, // ["text/csv", "application/csv", ".csv"] - acceptedExtension: String, // "csv" - unacceptableTypeLabel: String, // "Only CSV files are allowed." - }; - - connect() { - this.submitTarget.disabled = true - } - - addFile(event) { - const file = event.target.files[0] - this._fileAdded(file) - } - - dragover(event) { - event.preventDefault() - event.stopPropagation() - event.currentTarget.classList.add("bg-gray-100") - } - - dragleave(event) { - event.preventDefault() - event.stopPropagation() - event.currentTarget.classList.remove("bg-gray-100") - } - - drop(event) { - event.preventDefault() - event.stopPropagation() - event.currentTarget.classList.remove("bg-gray-100") - - const file = event.dataTransfer.files[0] - if (file && this._formatAcceptable(file)) { - this._setFileInput(file); - this._fileAdded(file) - } else { - this.previewTarget.classList.add("text-red-500") - this.previewTarget.textContent = this.unacceptableTypeLabelValue - } - } - - // Private - - _fetchFileSize(size) { - let fileSize = ''; - if (size < 1024 * 1024) { - fileSize = (size / 1024).toFixed(2) + ' KB'; // Convert bytes to KB - } else { - fileSize = (size / (1024 * 1024)).toFixed(2) + ' MB'; // Convert bytes to MB - } - return fileSize; - } - - _fileAdded(file) { - const fileSizeLimit = 5 * 1024 * 1024 // 5MB - - if (file) { - if (file.size > fileSizeLimit) { - this.previewTarget.classList.add("text-red-500") - this.previewTarget.textContent = this.unacceptableTypeLabelValue - return - } - - this.submitTarget.classList.remove([ - "bg-alpha-black-25", - "text-gray", - "cursor-not-allowed", - ]); - this.submitTarget.classList.add( - "bg-gray-900", - "text-white", - "cursor-pointer", - ); - this.submitTarget.disabled = false; - this.previewTarget.innerHTML = document.querySelector("#template-preview").innerHTML; - this.previewTarget.classList.remove("text-red-500") - this.previewTarget.classList.add("text-gray-900") - this.filenameTarget.textContent = file.name; - this.filesizeTarget.textContent = this._fetchFileSize(file.size); - } - } - - _formatAcceptable(file) { - const extension = file.name.split('.').pop().toLowerCase() - return this.acceptedTypesValue.includes(file.type) || extension === this.acceptedExtensionValue - } - - _setFileInput(file) { - const dataTransfer = new DataTransfer(); - dataTransfer.items.add(file); - this.inputTarget.files = dataTransfer.files; - } -} diff --git a/app/models/account.rb b/app/models/account.rb index d29f5479..cf6ff98b 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -5,14 +5,15 @@ class Account < ApplicationRecord belongs_to :family belongs_to :institution, optional: true + belongs_to :import, optional: true + has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping" has_many :entries, dependent: :destroy, class_name: "Account::Entry" has_many :transactions, through: :entries, source: :entryable, source_type: "Account::Transaction" has_many :valuations, through: :entries, source: :entryable, source_type: "Account::Valuation" has_many :trades, through: :entries, source: :entryable, source_type: "Account::Trade" has_many :holdings, dependent: :destroy has_many :balances, dependent: :destroy - has_many :imports, dependent: :destroy has_many :syncs, dependent: :destroy has_many :issues, as: :issuable, dependent: :destroy diff --git a/app/models/account/entry.rb b/app/models/account/entry.rb index 9bedbab0..4c38ffbd 100644 --- a/app/models/account/entry.rb +++ b/app/models/account/entry.rb @@ -5,6 +5,7 @@ class Account::Entry < ApplicationRecord belongs_to :account belongs_to :transfer, optional: true + belongs_to :import, optional: true delegated_type :entryable, types: Account::Entryable::TYPES, dependent: :destroy accepts_nested_attributes_for :entryable @@ -12,7 +13,6 @@ class Account::Entry < ApplicationRecord validates :date, :amount, :currency, presence: true validates :date, uniqueness: { scope: [ :account_id, :entryable_type ] }, if: -> { account_valuation? } validates :date, comparison: { greater_than: -> { min_supported_date } } - validate :trade_valid?, if: -> { account_trade? } scope :chronological, -> { order(:date, :created_at) } scope :reverse_chronological, -> { order(date: :desc, created_at: :desc) } @@ -219,20 +219,4 @@ class Account::Entry < ApplicationRecord previous: previous_entry&.amount_money, favorable_direction: account.favorable_direction end - - def trade_valid? - if account_trade.sell? - current_qty = account.holding_qty(account_trade.security) - - if current_qty < account_trade.qty.abs - errors.add( - :base, - :invalid_sell_quantity, - sell_qty: account_trade.qty.abs, - ticker: account_trade.security.ticker, - current_qty: current_qty - ) - end - end - end end diff --git a/app/models/account_import.rb b/app/models/account_import.rb new file mode 100644 index 00000000..3987a1ff --- /dev/null +++ b/app/models/account_import.rb @@ -0,0 +1,49 @@ +class AccountImport < Import + def import! + transaction do + rows.each do |row| + mapping = mappings.account_types.find_by(key: row.entity_type) + accountable_class = mapping.value.constantize + + account = family.accounts.build( + name: row.name, + balance: row.amount.to_d, + currency: row.currency, + accountable: accountable_class.new, + import: self + ) + + account.save! + end + end + end + + def mapping_steps + [ Import::AccountTypeMapping ] + end + + def required_column_keys + %i[name amount] + end + + def column_keys + %i[entity_type name amount currency] + end + + def dry_run + { + accounts: rows.count + } + end + + def csv_template + template = <<-CSV + Account type*,Name*,Balance*,Currency + Checking,Main Checking Account,1000.00,USD + Savings,Emergency Fund,5000.00,USD + Credit Card,Rewards Card,-500.00,USD + CSV + + CSV.parse(template, headers: true) + end +end diff --git a/app/models/category.rb b/app/models/category.rb index 516b7631..3744295a 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,5 +1,6 @@ class Category < ApplicationRecord has_many :transactions, dependent: :nullify, class_name: "Account::Transaction" + has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping" belongs_to :family validates :name, :color, :family, presence: true diff --git a/app/models/family.rb b/app/models/family.rb index ce875c64..b7ffdbc9 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -3,9 +3,9 @@ class Family < ApplicationRecord has_many :tags, dependent: :destroy has_many :accounts, dependent: :destroy has_many :institutions, dependent: :destroy + has_many :imports, dependent: :destroy has_many :transactions, through: :accounts has_many :entries, through: :accounts - has_many :imports, through: :accounts has_many :categories, dependent: :destroy has_many :merchants, dependent: :destroy has_many :issues, through: :accounts diff --git a/app/models/import.rb b/app/models/import.rb index 2cb1e634..c77ed6c7 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -1,185 +1,137 @@ class Import < ApplicationRecord - belongs_to :account + TYPES = %w[TransactionImport TradeImport AccountImport MintImport].freeze + SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative] - validate :raw_file_must_be_parsable - validates :col_sep, inclusion: { in: Csv::COL_SEP_LIST } - - before_save :initialize_csv, if: :should_initialize_csv? - - enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true - - store_accessor :column_mappings, :define_column_mapping_keys + belongs_to :family scope :ordered, -> { order(created_at: :desc) } - FALLBACK_TRANSACTION_NAME = "Imported transaction" + enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true + + validates :type, inclusion: { in: TYPES } + validates :col_sep, inclusion: { in: [ ",", ";" ] } + validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS } + + has_many :rows, dependent: :destroy + has_many :mappings, dependent: :destroy + has_many :accounts, dependent: :destroy + has_many :entries, dependent: :destroy, class_name: "Account::Entry" def publish_later + raise "Import is not publishable" unless publishable? + + update! status: :importing + ImportJob.perform_later(self) end - def loaded? + def publish + import! + + family.sync + + update! status: :complete + rescue => error + update! status: :failed, error: error.message + end + + def csv_rows + @csv_rows ||= parsed_csv + end + + def csv_headers + parsed_csv.headers + end + + def csv_sample + @csv_sample ||= parsed_csv.first(2) + end + + def dry_run + { + transactions: rows.count, + accounts: Import::AccountMapping.for_import(self).creational.count, + categories: Import::CategoryMapping.for_import(self).creational.count, + tags: Import::TagMapping.for_import(self).creational.count + } + end + + def required_column_keys + [] + end + + def column_keys + raise NotImplementedError, "Subclass must implement column_keys" + end + + def generate_rows_from_csv + rows.destroy_all + + mapped_rows = csv_rows.map do |row| + { + account: row[account_col_label].to_s, + date: row[date_col_label].to_s, + qty: row[qty_col_label].to_s, + ticker: row[ticker_col_label].to_s, + price: row[price_col_label].to_s, + amount: row[amount_col_label].to_s, + currency: (row[currency_col_label] || default_currency).to_s, + name: (row[name_col_label] || default_row_name).to_s, + category: row[category_col_label].to_s, + tags: row[tags_col_label].to_s, + entity_type: row[entity_type_col_label].to_s, + notes: row[notes_col_label].to_s + } + end + + rows.insert_all!(mapped_rows) + end + + def sync_mappings + mapping_steps.each do |mapping| + mapping.sync(self) + end + end + + def mapping_steps + [] + end + + def uploaded? raw_file_str.present? end def configured? - csv.present? + uploaded? && rows.any? end def cleaned? - loaded? && configured? && csv.valid? + configured? && rows.all?(&:valid?) end - def csv - get_normalized_csv_with_validation - end - - def available_headers - get_raw_csv.table.headers - end - - def get_selected_header_for_field(field) - column_mappings&.dig(field.key) || field.key - end - - def update_csv!(row_idx:, col_idx:, value:) - updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value) - update! normalized_csv_str: updated_csv.to_s - end - - # Type-specific methods (potential STI inheritance in future when more import types added) - def publish - update!(status: "importing") - - transaction do - generate_transactions.each do |txn| - txn.save! - end - end - - self.account.sync - - update!(status: "complete") - rescue => e - update!(status: "failed") - Rails.logger.error("Import with id #{id} failed: #{e}") - end - - def dry_run - generate_transactions - end - - def expected_fields - @expected_fields ||= create_expected_fields + def publishable? + cleaned? && mappings.all?(&:valid?) end private - - def get_normalized_csv_with_validation - return nil if normalized_csv_str.nil? - - csv = Import::Csv.new(normalized_csv_str) - - expected_fields.each do |field| - csv.define_validator(field.key, field.validator) if field.validator - end - - csv + def import! + # no-op, subclasses can implement for customization of algorithm end - def get_raw_csv - return nil if raw_file_str.nil? - Import::Csv.new(raw_file_str, col_sep:) + def default_row_name + "Imported item" end - def should_initialize_csv? - raw_file_str_changed? || column_mappings_changed? + def default_currency + family.currency end - def initialize_csv - generated_csv = generate_normalized_csv(raw_file_str) - self.normalized_csv_str = generated_csv.table.to_s - end - - # Uses the user-provided raw CSV + mappings to generate a normalized CSV for the import - def generate_normalized_csv(csv_str) - Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings, col_sep) - end - - def update_csv(row_idx, col_idx, value) - updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value) - update! normalized_csv_str: updated_csv.to_s - end - - def generate_transactions - transaction_entries = [] - category_cache = {} - tag_cache = {} - - csv.table.each do |row| - category_name = row["category"].presence - tag_strings = row["tags"].presence&.split("|") || [] - tags = [] - - tag_strings.each do |tag_string| - tags << tag_cache[tag_string] ||= account.family.tags.find_or_initialize_by(name: tag_string) - end - - category = category_cache[category_name] ||= account.family.categories.find_or_initialize_by(name: category_name) if category_name.present? - - entry = account.entries.build \ - name: row["name"].presence || FALLBACK_TRANSACTION_NAME, - date: Date.iso8601(row["date"]), - currency: account.currency, - amount: BigDecimal(row["amount"]) * -1, - entryable: Account::Transaction.new(category: category, tags: tags) - - transaction_entries << entry - end - - transaction_entries - end - - def create_expected_fields - date_field = Import::Field.new \ - key: "date", - label: "Date", - validator: ->(value) { Import::Field.iso_date_validator(value) } - - name_field = Import::Field.new \ - key: "name", - label: "Name", - is_optional: true - - category_field = Import::Field.new \ - key: "category", - label: "Category", - is_optional: true - - tags_field = Import::Field.new \ - key: "tags", - label: "Tags", - is_optional: true - - amount_field = Import::Field.new \ - key: "amount", - label: "Amount", - validator: ->(value) { Import::Field.bigdecimal_validator(value) } - - [ date_field, name_field, category_field, tags_field, amount_field ] - end - - def define_column_mapping_keys - expected_fields.each do |field| - field.key.to_sym - end - end - - def raw_file_must_be_parsable - begin - CSV.parse(raw_file_str || "", col_sep:) - rescue CSV::MalformedCSVError - errors.add(:raw_file_str, :invalid_csv_format) - end + def parsed_csv + @parsed_csv ||= CSV.parse( + (raw_file_str || "").strip, + headers: true, + col_sep: col_sep, + converters: [ ->(str) { str&.strip } ] + ) end end diff --git a/app/models/import/account_mapping.rb b/app/models/import/account_mapping.rb new file mode 100644 index 00000000..67b11ba5 --- /dev/null +++ b/app/models/import/account_mapping.rb @@ -0,0 +1,45 @@ +class Import::AccountMapping < Import::Mapping + validates :mappable, presence: true, if: -> { key.blank? || !create_when_empty } + + class << self + def mapping_values(import) + import.rows.map(&:account).uniq + end + end + + def selectable_values + family_accounts = import.family.accounts.alphabetically.map { |account| [ account.name, account.id ] } + + unless key.blank? + family_accounts.unshift [ "Add as new account", CREATE_NEW_KEY ] + end + + family_accounts + end + + def requires_selection? + true + end + + def values_count + import.rows.where(account: key).count + end + + def mappable_class + Account + end + + def create_mappable! + return unless creatable? + + account = import.family.accounts.create_or_find_by!(name: key) do |new_account| + new_account.balance = 0 + new_account.import = import + new_account.currency = import.family.currency + new_account.accountable = Depository.new + end + + self.mappable = account + save! + end +end diff --git a/app/models/import/account_type_mapping.rb b/app/models/import/account_type_mapping.rb new file mode 100644 index 00000000..8b60edcd --- /dev/null +++ b/app/models/import/account_type_mapping.rb @@ -0,0 +1,25 @@ +class Import::AccountTypeMapping < Import::Mapping + validates :value, presence: true + + class << self + def mapping_values(import) + import.rows.map(&:entity_type).uniq + end + end + + def selectable_values + Accountable::TYPES.map { |type| [ type.titleize, type ] } + end + + def requires_selection? + true + end + + def values_count + import.rows.where(entity_type: key).count + end + + def create_mappable! + # no-op + end +end diff --git a/app/models/import/category_mapping.rb b/app/models/import/category_mapping.rb new file mode 100644 index 00000000..12302603 --- /dev/null +++ b/app/models/import/category_mapping.rb @@ -0,0 +1,36 @@ +class Import::CategoryMapping < Import::Mapping + class << self + def mapping_values(import) + import.rows.map(&:category).uniq + end + end + + def selectable_values + family_categories = import.family.categories.alphabetically.map { |category| [ category.name, category.id ] } + + unless key.blank? + family_categories.unshift [ "Add as new category", CREATE_NEW_KEY ] + end + + family_categories + end + + def requires_selection? + false + end + + def values_count + import.rows.where(category: key).count + end + + def mappable_class + Category + end + + def create_mappable! + return unless creatable? + + self.mappable = import.family.categories.find_or_create_by!(name: key) + save! + end +end diff --git a/app/models/import/csv.rb b/app/models/import/csv.rb deleted file mode 100644 index 8fe593e4..00000000 --- a/app/models/import/csv.rb +++ /dev/null @@ -1,83 +0,0 @@ -class Import::Csv - DEFAULT_COL_SEP = ",".freeze - COL_SEP_LIST = [ DEFAULT_COL_SEP, ";" ].freeze - - def self.parse_csv(csv_str, col_sep: DEFAULT_COL_SEP) - CSV.parse( - csv_str&.strip || "", - headers: true, - col_sep:, - converters: [ ->(str) { str&.strip } ] - ) - end - - def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP) - raw_csv = self.parse_csv(raw_file_str, col_sep:) - - generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv| - raw_csv.each do |row| - row_values = [] - - fields.each do |field| - # Finds the column header name the user has designated for the expected field - mapped_field_key = field_mappings[field.key] if field_mappings - mapped_header = mapped_field_key || field.key - - row_values << row.fetch(mapped_header, "") - end - - csv << row_values - end - end - - new(generated_csv_str, col_sep:) - end - - attr_reader :csv_str, :col_sep - - def initialize(csv_str, column_validators: nil, col_sep: DEFAULT_COL_SEP) - @csv_str = csv_str - @col_sep = col_sep - @column_validators = column_validators || {} - end - - def table - @table ||= self.class.parse_csv(csv_str, col_sep:) - end - - def update_cell(row_idx, col_idx, value) - copy = table.by_col_or_row - copy[row_idx][col_idx] = value - copy - end - - def valid? - table.each_with_index.all? do |row, row_idx| - row.each_with_index.all? do |cell, col_idx| - cell_valid?(row_idx, col_idx) - end - end - end - - def cell_valid?(row_idx, col_idx) - value = table.dig(row_idx, col_idx) - header = table.headers[col_idx] - validator = get_validator_by_header(header) - validator.call(value) - end - - def define_validator(header_key, validator = nil, &block) - header = table.headers.find { |h| h.strip == header_key } - raise "Cannot define validator for header #{header_key}: header does not exist in CSV" if header.nil? - - column_validators[header] = validator || block - end - - private - - attr_accessor :column_validators - - def get_validator_by_header(header) - column_validators&.dig(header) || ->(_v) { true } - end -end diff --git a/app/models/import/field.rb b/app/models/import/field.rb deleted file mode 100644 index 45f7ee45..00000000 --- a/app/models/import/field.rb +++ /dev/null @@ -1,37 +0,0 @@ -class Import::Field - def self.iso_date_validator(value) - Date.iso8601(value) - true - rescue - false - end - - def self.bigdecimal_validator(value) - BigDecimal(value) - true - rescue - false - end - - attr_reader :key, :label, :validator - - def initialize(key:, label:, is_optional: false, validator: nil) - @key = key.to_s - @label = label - @is_optional = is_optional - @validator = validator - end - - def optional? - @is_optional - end - - def define_validator(validator = nil, &block) - @validator = validator || block - end - - def validate(value) - return true if validator.nil? - validator.call(value) - end -end diff --git a/app/models/import/mapping.rb b/app/models/import/mapping.rb new file mode 100644 index 00000000..a0a4bc8b --- /dev/null +++ b/app/models/import/mapping.rb @@ -0,0 +1,56 @@ +class Import::Mapping < ApplicationRecord + CREATE_NEW_KEY = "internal_new_resource" + + belongs_to :import + belongs_to :mappable, polymorphic: true, optional: true + + validates :key, presence: true, uniqueness: { scope: [ :import_id, :type ] }, allow_blank: true + + scope :for_import, ->(import) { where(import: import) } + scope :creational, -> { where(create_when_empty: true, mappable: nil) } + scope :categories, -> { where(type: "Import::CategoryMapping") } + scope :tags, -> { where(type: "Import::TagMapping") } + scope :accounts, -> { where(type: "Import::AccountMapping") } + scope :account_types, -> { where(type: "Import::AccountTypeMapping") } + + class << self + def mappable_for(key) + find_by(key: key)&.mappable + end + + def sync(import) + unique_values = mapping_values(import).uniq + + unique_values.each do |value| + mapping = find_or_initialize_by(key: value, import: import, create_when_empty: value.present?) + mapping.save(validate: false) if mapping.new_record? + end + + where(import: import).where.not(key: unique_values).destroy_all + end + + def mapping_values(import) + raise NotImplementedError, "Subclass must implement mapping_values" + end + end + + def selectable_values + raise NotImplementedError, "Subclass must implement selectable_values" + end + + def values_count + raise NotImplementedError, "Subclass must implement values_count" + end + + def mappable_class + nil + end + + def creatable? + mappable.nil? && key.present? && create_when_empty + end + + def create_mappable! + raise NotImplementedError, "Subclass must implement create_mappable!" + end +end diff --git a/app/models/import/row.rb b/app/models/import/row.rb new file mode 100644 index 00000000..a4e7a473 --- /dev/null +++ b/app/models/import/row.rb @@ -0,0 +1,70 @@ +class Import::Row < ApplicationRecord + belongs_to :import + + validates :amount, numericality: true, allow_blank: true + validates :currency, presence: true + + validate :date_matches_user_format + validate :required_columns + validate :currency_is_valid + + scope :ordered, -> { order(:id) } + + def tags_list + if tags.blank? + [ "" ] + else + tags.split("|").map(&:strip) + end + end + + def date_iso + Date.strptime(date, import.date_format).iso8601 + end + + def signed_amount + if import.type == "TradeImport" + price.to_d * apply_signage_convention(qty.to_d) + else + apply_signage_convention(amount.to_d) + end + end + + def sync_mappings + Import::CategoryMapping.sync(import) if import.column_keys.include?(:category) + Import::TagMapping.sync(import) if import.column_keys.include?(:tags) + Import::AccountMapping.sync(import) if import.column_keys.include?(:account) + Import::AccountTypeMapping.sync(import) if import.column_keys.include?(:entity_type) + end + + private + def apply_signage_convention(value) + value * (import.signage_convention == "inflows_positive" ? 1 : -1) + end + + def required_columns + import.required_column_keys.each do |required_key| + errors.add(required_key, "is required") if self[required_key].blank? + end + end + + def date_matches_user_format + return if date.blank? + + parsed_date = Date.strptime(date, import.date_format) rescue nil + + if parsed_date.nil? + errors.add(:date, "must exactly match the format: #{import.date_format}") + end + end + + def currency_is_valid + return true if currency.blank? + + begin + Money::Currency.new(currency) + rescue Money::Currency::UnknownCurrencyError + errors.add(:currency, "is not a valid currency code") + end + end +end diff --git a/app/models/import/tag_mapping.rb b/app/models/import/tag_mapping.rb new file mode 100644 index 00000000..899b4dc5 --- /dev/null +++ b/app/models/import/tag_mapping.rb @@ -0,0 +1,36 @@ +class Import::TagMapping < Import::Mapping + class << self + def mapping_values(import) + import.rows.map(&:tags_list).flatten.uniq + end + end + + def selectable_values + family_tags = import.family.tags.alphabetically.map { |tag| [ tag.name, tag.id ] } + + unless key.blank? + family_tags.unshift [ "Add as new tag", CREATE_NEW_KEY ] + end + + family_tags + end + + def requires_selection? + false + end + + def values_count + import.rows.map(&:tags_list).flatten.count { |tag| tag == key } + end + + def mappable_class + Tag + end + + def create_mappable! + return unless creatable? + + self.mappable = import.family.tags.find_or_create_by!(name: key) + save! + end +end diff --git a/app/models/mint_import.rb b/app/models/mint_import.rb new file mode 100644 index 00000000..1d5c1ee6 --- /dev/null +++ b/app/models/mint_import.rb @@ -0,0 +1,94 @@ +class MintImport < Import + after_create :set_mappings + + def generate_rows_from_csv + rows.destroy_all + + mapped_rows = csv_rows.map do |row| + { + account: row[account_col_label].to_s, + date: row[date_col_label].to_s, + amount: signed_csv_amount(row).to_s, + currency: (row[currency_col_label] || default_currency).to_s, + name: (row[name_col_label] || default_row_name).to_s, + category: row[category_col_label].to_s, + tags: row[tags_col_label].to_s, + notes: row[notes_col_label].to_s + } + end + + rows.insert_all!(mapped_rows) + end + + def import! + transaction do + mappings.each(&:create_mappable!) + + rows.each do |row| + account = mappings.accounts.mappable_for(row.account) + category = mappings.categories.mappable_for(row.category) + tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact + + entry = account.entries.build \ + date: row.date_iso, + amount: row.signed_amount, + name: row.name, + currency: row.currency, + entryable: Account::Transaction.new(category: category, tags: tags, notes: row.notes), + import: self + + entry.save! + end + end + end + + def mapping_steps + [ Import::CategoryMapping, Import::TagMapping, Import::AccountMapping ] + end + + def required_column_keys + %i[date amount] + end + + def column_keys + %i[date amount name currency category tags account notes] + end + + def csv_template + template = <<-CSV + Date,Amount,Account Name,Description,Category,Labels,Currency,Notes,Transaction Type + 01/01/2024,-8.55,Checking,Starbucks,Food & Drink,Coffee|Breakfast,USD,Morning coffee,debit + 04/15/2024,2000,Savings,Paycheck,Income,,USD,Bi-weekly salary,credit + CSV + + CSV.parse(template, headers: true) + end + + def signed_csv_amount(csv_row) + amount = csv_row[amount_col_label] + type = csv_row["Transaction Type"] + + if type == "credit" + amount.to_d + else + amount.to_d * -1 + end + end + + private + def set_mappings + self.signage_convention = "inflows_positive" + self.date_col_label = "Date" + self.date_format = "%m/%d/%Y" + self.name_col_label = "Description" + self.amount_col_label = "Amount" + self.currency_col_label = "Currency" + self.account_col_label = "Account Name" + self.category_col_label = "Category" + self.tags_col_label = "Labels" + self.notes_col_label = "Notes" + self.entity_type_col_label = "Transaction Type" + + save! + end +end diff --git a/app/models/tag.rb b/app/models/tag.rb index eab4d866..6b2fb67b 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -2,6 +2,7 @@ class Tag < ApplicationRecord belongs_to :family has_many :taggings, dependent: :destroy has_many :transactions, through: :taggings, source: :taggable, source_type: "Account::Transaction" + has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping" validates :name, presence: true, uniqueness: { scope: :family } diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb new file mode 100644 index 00000000..4ca57ea1 --- /dev/null +++ b/app/models/trade_import.rb @@ -0,0 +1,52 @@ +class TradeImport < Import + def import! + transaction do + mappings.each(&:create_mappable!) + + rows.each do |row| + account = mappings.accounts.mappable_for(row.account) + security = Security.find_or_create_by(ticker: row.ticker) + + entry = account.entries.build \ + date: row.date_iso, + amount: row.signed_amount, + name: row.name, + currency: row.currency, + entryable: Account::Trade.new(security: security, qty: row.qty, currency: row.currency, price: row.price), + import: self + + entry.save! + end + end + end + + def mapping_steps + [ Import::AccountMapping ] + end + + def required_column_keys + %i[date ticker qty price] + end + + def column_keys + %i[date ticker qty price currency account name] + end + + def dry_run + { + transactions: rows.count, + accounts: Import::AccountMapping.for_import(self).creational.count + } + end + + def csv_template + template = <<-CSV + date*,ticker*,qty*,price*,currency,account,name + 05/15/2024,AAPL,10,150.00,USD,Trading Account,Apple Inc. Purchase + 05/16/2024,GOOGL,-5,2500.00,USD,Investment Account,Alphabet Inc. Sale + 05/17/2024,TSLA,2,700.50,USD,Retirement Account,Tesla Inc. Purchase + CSV + + CSV.parse(template, headers: true) + end +end diff --git a/app/models/transaction_import.rb b/app/models/transaction_import.rb new file mode 100644 index 00000000..7e8cc4fb --- /dev/null +++ b/app/models/transaction_import.rb @@ -0,0 +1,46 @@ +class TransactionImport < Import + def import! + transaction do + mappings.each(&:create_mappable!) + + rows.each do |row| + account = mappings.accounts.mappable_for(row.account) + category = mappings.categories.mappable_for(row.category) + tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact + + entry = account.entries.build \ + date: row.date_iso, + amount: row.signed_amount, + name: row.name, + currency: row.currency, + entryable: Account::Transaction.new(category: category, tags: tags, notes: row.notes), + import: self + + entry.save! + end + end + end + + def required_column_keys + %i[date amount] + end + + def column_keys + %i[date amount name currency category tags account notes] + end + + def mapping_steps + [ Import::CategoryMapping, Import::TagMapping, Import::AccountMapping ] + end + + def csv_template + template = <<-CSV + date*,amount*,name,currency,category,tags,account,notes + 05/15/2024,-45.99,Grocery Store,USD,Food,groceries|essentials,Checking Account,Monthly grocery run + 05/16/2024,1500.00,Salary,,Income,,Main Account, + 05/17/2024,-12.50,Coffee Shop,,,coffee,, + CSV + + CSV.parse(template, headers: true) + end +end diff --git a/app/views/import/cleans/show.html.erb b/app/views/import/cleans/show.html.erb new file mode 100644 index 00000000..ef0c230f --- /dev/null +++ b/app/views/import/cleans/show.html.erb @@ -0,0 +1,59 @@ +<%= content_for :header_nav do %> + <%= render "imports/nav", import: @import %> +<% end %> + +<%= content_for :previous_path, import_configuration_path(@import) %> + +
<%= t(".description") %>
+Your data has been cleaned
+You have errors in your data
+We have pre-configured your Mint import for you. Please proceed to the next step.
+<%= t(".description") %>
+CSV <%= mapping_label(mapping_class) %>
+Maybe <%= mapping_label(mapping_class) %>
+Rows
++ <%= t(".#{step_mapping_class.name.demodulize.underscore}_description", import_type: @import.type.underscore.humanize) %> +
+<%= t(".description") %>
+<%= t(".instructions_1") %>
+ +<%= t(".instructions") %>
-- <%= t(".instructions") %> - - <%= link_to "download this template", "/transactions.csv", download: "" %> - -
-<%= t(".message") %>
- <%= link_to new_import_path(enable_type_selector: true), class: "w-fit flex text-white text-sm font-medium items-center gap-1 bg-gray-900 rounded-lg p-2 pr-3", data: { turbo_frame: "modal" } do %> + <%= link_to new_import_path(enable_type_selector: true), class: "btn btn--primary flex items-center gap-2", data: { turbo_frame: "modal" } do %> <%= lucide_icon("plus", class: "w-5 h-5") %> <%= t(".new") %> <% end %> diff --git a/app/views/imports/_failure.html.erb b/app/views/imports/_failure.html.erb new file mode 100644 index 00000000..9dc003d8 --- /dev/null +++ b/app/views/imports/_failure.html.erb @@ -0,0 +1,18 @@ +<%# locals: (import:) %> + +Please check that your file format, for any errors and that all required fields are filled, then come back and try again.
+- <%= t(".label", account: import.account.name) %> -
+<%= t(".completed_on", datetime: import.updated_at.strftime("%Y-%m-%d")) %>
- <% else %> -<%= t(".started_on", datetime: import.created_at.strftime("%Y-%m-%d")) %>
+ <% if import.pending? %> + + <%= t(".in_progress") %> + + <% elsif import.importing? %> + + <%= t(".uploading") %> + + <% elsif import.failed? %> + + <%= t(".failed") %> + + <% elsif import.complete? %> + + <%= t(".complete") %> + <% end %>Your import is in progress. Check the imports menu for status updates or click 'Check Status' to refresh the page for updates. Feel free to continue using the app.
+<%= t(".description") %>
+item
+count
+<%= resource.label %>
+<%= count %>
+Your imported data has been successfully added to the app and is now ready for use.
+<%= t(".description") %>
-<%= t(".clean_description") %>
-<%= t(".configure_description") %>
-<%= t(".confirm_description") %>
-<%= t(".header_text") %>
-<%= t(".description_text") %>
-<%= t(".description") %>
-<%= t(".description") %>
+<%= t(".header_text") %>
-<%= t(".description_text") %>
+<%= notice %>
- <% end %> +<%= content_for :header_nav do %> + <%= render "imports/nav", import: @import %> +<% end %> - <%= render @import %> +<%= content_for :previous_path, import_confirm_path(@import) %> - <%= link_to "Edit this import", edit_import_path(@import), class: "mt-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> -