CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)

* Remove stale 1.0 import logic and model

* Fresh start

* Checkpoint before removing nav

* First working prototype

* Add trade, account, and mint import flows

* Basic working version with tests

* System tests for each import type

* Clean up mappings flow

* Clean up PR, refactor stale code, tests

* Add back row validations

* Row validations

* Fix import job test

* Fix import navigation

* Fix mint import configuration form

* Currency preset for new accounts

app/models/import/account_mapping.rb

@@ -0,0 +1,45 @@
+class Import::AccountMapping < Import::Mapping
+  validates :mappable, presence: true, if: -> { key.blank? || !create_when_empty }
+
+  class << self
+    def mapping_values(import)
+      import.rows.map(&:account).uniq
+    end
+  end
+
+  def selectable_values
+    family_accounts = import.family.accounts.alphabetically.map { |account| [ account.name, account.id ] }
+
+    unless key.blank?
+      family_accounts.unshift [ "Add as new account", CREATE_NEW_KEY ]
+    end
+
+    family_accounts
+  end
+
+  def requires_selection?
+    true
+  end
+
+  def values_count
+    import.rows.where(account: key).count
+  end
+
+  def mappable_class
+    Account
+  end
+
+  def create_mappable!
+    return unless creatable?
+
+    account = import.family.accounts.create_or_find_by!(name: key) do |new_account|
+      new_account.balance = 0
+      new_account.import = import
+      new_account.currency = import.family.currency
+      new_account.accountable = Depository.new
+    end
+
+    self.mappable = account
+    save!
+  end
+end

app/models/import/account_type_mapping.rb

@@ -0,0 +1,25 @@
+class Import::AccountTypeMapping < Import::Mapping
+  validates :value, presence: true
+
+  class << self
+    def mapping_values(import)
+      import.rows.map(&:entity_type).uniq
+    end
+  end
+
+  def selectable_values
+    Accountable::TYPES.map { |type| [ type.titleize, type ] }
+  end
+
+  def requires_selection?
+    true
+  end
+
+  def values_count
+    import.rows.where(entity_type: key).count
+  end
+
+  def create_mappable!
+    # no-op
+  end
+end

app/models/import/category_mapping.rb

@@ -0,0 +1,36 @@
+class Import::CategoryMapping < Import::Mapping
+  class << self
+    def mapping_values(import)
+      import.rows.map(&:category).uniq
+    end
+  end
+
+  def selectable_values
+    family_categories = import.family.categories.alphabetically.map { |category| [ category.name, category.id ] }
+
+    unless key.blank?
+      family_categories.unshift [ "Add as new category", CREATE_NEW_KEY ]
+    end
+
+    family_categories
+  end
+
+  def requires_selection?
+    false
+  end
+
+  def values_count
+    import.rows.where(category: key).count
+  end
+
+  def mappable_class
+    Category
+  end
+
+  def create_mappable!
+    return unless creatable?
+
+    self.mappable = import.family.categories.find_or_create_by!(name: key)
+    save!
+  end
+end

app/models/import/csv.rb

@@ -1,83 +0,0 @@
-class Import::Csv
-  DEFAULT_COL_SEP = ",".freeze
-  COL_SEP_LIST = [ DEFAULT_COL_SEP, ";" ].freeze
-
-  def self.parse_csv(csv_str, col_sep: DEFAULT_COL_SEP)
-    CSV.parse(
-      csv_str&.strip || "",
-      headers: true,
-      col_sep:,
-      converters: [ ->(str) { str&.strip } ]
-    )
-  end
-
-  def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
-    raw_csv = self.parse_csv(raw_file_str, col_sep:)
-
-    generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|
-      raw_csv.each do |row|
-        row_values = []
-
-        fields.each do |field|
-          # Finds the column header name the user has designated for the expected field
-          mapped_field_key = field_mappings[field.key] if field_mappings
-          mapped_header = mapped_field_key || field.key
-
-          row_values << row.fetch(mapped_header, "")
-        end
-
-        csv << row_values
-      end
-    end
-
-    new(generated_csv_str, col_sep:)
-  end
-
-  attr_reader :csv_str, :col_sep
-
-  def initialize(csv_str, column_validators: nil, col_sep: DEFAULT_COL_SEP)
-    @csv_str = csv_str
-    @col_sep = col_sep
-    @column_validators = column_validators || {}
-  end
-
-  def table
-    @table ||= self.class.parse_csv(csv_str, col_sep:)
-  end
-
-  def update_cell(row_idx, col_idx, value)
-    copy = table.by_col_or_row
-    copy[row_idx][col_idx] = value
-    copy
-  end
-
-  def valid?
-    table.each_with_index.all? do |row, row_idx|
-      row.each_with_index.all? do |cell, col_idx|
-        cell_valid?(row_idx, col_idx)
-      end
-    end
-  end
-
-  def cell_valid?(row_idx, col_idx)
-    value = table.dig(row_idx, col_idx)
-    header = table.headers[col_idx]
-    validator = get_validator_by_header(header)
-    validator.call(value)
-  end
-
-  def define_validator(header_key, validator = nil, &block)
-    header = table.headers.find { |h| h.strip == header_key }
-    raise "Cannot define validator for header #{header_key}: header does not exist in CSV" if header.nil?
-
-    column_validators[header] = validator || block
-  end
-
-  private
-
-    attr_accessor :column_validators
-
-    def get_validator_by_header(header)
-      column_validators&.dig(header) || ->(_v) { true }
-    end
-end

app/models/import/field.rb

@@ -1,37 +0,0 @@
-class Import::Field
-  def self.iso_date_validator(value)
-    Date.iso8601(value)
-    true
-  rescue
-    false
-  end
-
-  def self.bigdecimal_validator(value)
-    BigDecimal(value)
-    true
-  rescue
-    false
-  end
-
-  attr_reader :key, :label, :validator
-
-  def initialize(key:, label:, is_optional: false, validator: nil)
-    @key = key.to_s
-    @label = label
-    @is_optional = is_optional
-    @validator = validator
-  end
-
-  def optional?
-    @is_optional
-  end
-
-  def define_validator(validator = nil, &block)
-    @validator = validator || block
-  end
-
-  def validate(value)
-    return true if validator.nil?
-    validator.call(value)
-  end
-end

app/models/import/mapping.rb

@@ -0,0 +1,56 @@
+class Import::Mapping < ApplicationRecord
+  CREATE_NEW_KEY = "internal_new_resource"
+
+  belongs_to :import
+  belongs_to :mappable, polymorphic: true, optional: true
+
+  validates :key, presence: true, uniqueness: { scope: [ :import_id, :type ] }, allow_blank: true
+
+  scope :for_import, ->(import) { where(import: import) }
+  scope :creational, -> { where(create_when_empty: true, mappable: nil) }
+  scope :categories, -> { where(type: "Import::CategoryMapping") }
+  scope :tags, -> { where(type: "Import::TagMapping") }
+  scope :accounts, -> { where(type: "Import::AccountMapping") }
+  scope :account_types, -> { where(type: "Import::AccountTypeMapping") }
+
+  class << self
+    def mappable_for(key)
+      find_by(key: key)&.mappable
+    end
+
+    def sync(import)
+      unique_values = mapping_values(import).uniq
+
+      unique_values.each do |value|
+        mapping = find_or_initialize_by(key: value, import: import, create_when_empty: value.present?)
+        mapping.save(validate: false) if mapping.new_record?
+      end
+
+      where(import: import).where.not(key: unique_values).destroy_all
+    end
+
+    def mapping_values(import)
+      raise NotImplementedError, "Subclass must implement mapping_values"
+    end
+  end
+
+  def selectable_values
+    raise NotImplementedError, "Subclass must implement selectable_values"
+  end
+
+  def values_count
+    raise NotImplementedError, "Subclass must implement values_count"
+  end
+
+  def mappable_class
+    nil
+  end
+
+  def creatable?
+    mappable.nil? && key.present? && create_when_empty
+  end
+
+  def create_mappable!
+    raise NotImplementedError, "Subclass must implement create_mappable!"
+  end
+end

app/models/import/row.rb

@@ -0,0 +1,70 @@
+class Import::Row < ApplicationRecord
+  belongs_to :import
+
+  validates :amount, numericality: true, allow_blank: true
+  validates :currency, presence: true
+
+  validate :date_matches_user_format
+  validate :required_columns
+  validate :currency_is_valid
+
+  scope :ordered, -> { order(:id) }
+
+  def tags_list
+    if tags.blank?
+      [ "" ]
+    else
+      tags.split("|").map(&:strip)
+    end
+  end
+
+  def date_iso
+    Date.strptime(date, import.date_format).iso8601
+  end
+
+  def signed_amount
+    if import.type == "TradeImport"
+      price.to_d * apply_signage_convention(qty.to_d)
+    else
+      apply_signage_convention(amount.to_d)
+    end
+  end
+
+  def sync_mappings
+    Import::CategoryMapping.sync(import) if import.column_keys.include?(:category)
+    Import::TagMapping.sync(import) if import.column_keys.include?(:tags)
+    Import::AccountMapping.sync(import) if import.column_keys.include?(:account)
+    Import::AccountTypeMapping.sync(import) if import.column_keys.include?(:entity_type)
+  end
+
+  private
+    def apply_signage_convention(value)
+      value * (import.signage_convention == "inflows_positive" ? 1 : -1)
+    end
+
+    def required_columns
+      import.required_column_keys.each do |required_key|
+        errors.add(required_key, "is required") if self[required_key].blank?
+      end
+    end
+
+    def date_matches_user_format
+      return if date.blank?
+
+      parsed_date = Date.strptime(date, import.date_format) rescue nil
+
+      if parsed_date.nil?
+        errors.add(:date, "must exactly match the format: #{import.date_format}")
+      end
+    end
+
+    def currency_is_valid
+      return true if currency.blank?
+
+      begin
+        Money::Currency.new(currency)
+      rescue Money::Currency::UnknownCurrencyError
+        errors.add(:currency, "is not a valid currency code")
+      end
+    end
+end

app/models/import/tag_mapping.rb

@@ -0,0 +1,36 @@
+class Import::TagMapping < Import::Mapping
+  class << self
+    def mapping_values(import)
+      import.rows.map(&:tags_list).flatten.uniq
+    end
+  end
+
+  def selectable_values
+    family_tags = import.family.tags.alphabetically.map { |tag| [ tag.name, tag.id ] }
+
+    unless key.blank?
+      family_tags.unshift [ "Add as new tag", CREATE_NEW_KEY ]
+    end
+
+    family_tags
+  end
+
+  def requires_selection?
+    false
+  end
+
+  def values_count
+    import.rows.map(&:tags_list).flatten.count { |tag| tag == key }
+  end
+
+  def mappable_class
+    Tag
+  end
+
+  def create_mappable!
+    return unless creatable?
+
+    self.mappable = import.family.tags.find_or_create_by!(name: key)
+    save!
+  end
+end