CSV Transaction Imports (#708)

Introduces a basic CSV import module for bulk-importing account transactions.

Changes include:

- User can load a CSV
- User can configure the column mappings for a CSV
- Imported CSV shows invalid cells
- User can clean up their data directly in the UI
- User can see a preview of the import rows and confirm import
- Layout refactor + Import nav stepper
- System test stability improvements

app/models/account.rb

@@ -9,6 +9,7 @@ class Account < ApplicationRecord
   has_many :balances, dependent: :destroy
   has_many :valuations, dependent: :destroy
   has_many :transactions, dependent: :destroy
+  has_many :imports, dependent: :destroy
 
   monetize :balance
 

app/models/family.rb

@@ -2,6 +2,7 @@ class Family < ApplicationRecord
   has_many :users, dependent: :destroy
   has_many :accounts, dependent: :destroy
   has_many :transactions, through: :accounts
+  has_many :imports, through: :accounts
   has_many :transaction_categories, dependent: :destroy, class_name: "Transaction::Category"
   has_many :transaction_merchants, dependent: :destroy, class_name: "Transaction::Merchant"
 

app/models/import.rb

@@ -0,0 +1,161 @@
+class Import < ApplicationRecord
+  belongs_to :account
+
+  validate :raw_csv_must_be_parsable
+
+  before_save :initialize_csv, if: :should_initialize_csv?
+
+  enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true
+
+  store_accessor :column_mappings, :define_column_mapping_keys
+
+  scope :ordered, -> { order(created_at: :desc) }
+
+  def publish_later
+    ImportJob.perform_later(self)
+  end
+
+  def loaded?
+    raw_csv_str.present?
+  end
+
+  def configured?
+    csv.present?
+  end
+
+  def cleaned?
+    loaded? && configured? && csv.valid?
+  end
+
+  def csv
+    get_normalized_csv_with_validation
+  end
+
+  def available_headers
+    get_raw_csv.table.headers
+  end
+
+  def get_selected_header_for_field(field)
+    column_mappings&.dig(field) || field.key
+  end
+
+  def update_csv!(row_idx:, col_idx:, value:)
+    updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value)
+    update! normalized_csv_str: updated_csv.to_s
+  end
+
+  # Type-specific methods (potential STI inheritance in future when more import types added)
+  def publish
+    update!(status: "importing")
+
+    transaction do
+      generate_transactions.each do |txn|
+        txn.save!
+      end
+    end
+
+    update!(status: "complete")
+  rescue => e
+    update!(status: "failed")
+    Rails.logger.error("Import with id #{id} failed: #{e}")
+  end
+
+  def dry_run
+    generate_transactions
+  end
+
+  def expected_fields
+    @expected_fields ||= create_expected_fields
+  end
+
+  private
+
+    def get_normalized_csv_with_validation
+      return nil if normalized_csv_str.nil?
+
+      csv = Import::Csv.new(normalized_csv_str)
+
+      expected_fields.each do |field|
+        csv.define_validator(field.key, field.validator) if field.validator
+      end
+
+      csv
+    end
+
+    def get_raw_csv
+      return nil if raw_csv_str.nil?
+      Import::Csv.new(raw_csv_str)
+    end
+
+    def should_initialize_csv?
+      raw_csv_str_changed? || column_mappings_changed?
+    end
+
+    def initialize_csv
+      generated_csv = generate_normalized_csv(raw_csv_str)
+      self.normalized_csv_str = generated_csv.table.to_s
+    end
+
+    # Uses the user-provided raw CSV + mappings to generate a normalized CSV for the import
+    def generate_normalized_csv(csv_str)
+      Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings)
+    end
+
+    def update_csv(row_idx, col_idx, value)
+      updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value)
+      update! normalized_csv_str: updated_csv.to_s
+    end
+
+    def generate_transactions
+      transactions = []
+
+      csv.table.each do |row|
+        category = account.family.transaction_categories.find_or_initialize_by(name: row["category"])
+        txn = account.transactions.build \
+          name: row["name"] || "Imported transaction",
+          date: Date.iso8601(row["date"]),
+          category: category,
+          amount: BigDecimal(row["amount"]) * -1 # User inputs amounts with opposite signage of our internal representation
+
+        transactions << txn
+      end
+
+      transactions
+    end
+
+    def create_expected_fields
+      date_field = Import::Field.new \
+        key: "date",
+        label: "Date",
+        validator: ->(value) { Import::Field.iso_date_validator(value) }
+
+      name_field = Import::Field.new \
+        key: "name",
+        label: "Name"
+
+      category_field = Import::Field.new \
+        key: "category",
+        label: "Category"
+
+      amount_field = Import::Field.new \
+        key: "amount",
+        label: "Amount",
+        validator: ->(value) { Import::Field.bigdecimal_validator(value) }
+
+      [ date_field, name_field, category_field, amount_field ]
+    end
+
+    def define_column_mapping_keys
+      expected_fields.each do |field|
+        field.key.to_sym
+      end
+    end
+
+    def raw_csv_must_be_parsable
+      begin
+        CSV.parse(raw_csv_str || "")
+      rescue CSV::MalformedCSVError
+        errors.add(:raw_csv_str, "is not a valid CSV format")
+      end
+    end
+end

app/models/import/csv.rb

@@ -0,0 +1,74 @@
+class Import::Csv
+  def self.parse_csv(csv_str)
+    CSV.parse((csv_str || "").strip, headers: true, converters: [ ->(str) { str.strip } ])
+  end
+
+  def self.create_with_field_mappings(raw_csv_str, fields, field_mappings)
+    raw_csv = self.parse_csv(raw_csv_str)
+
+    generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true do |csv|
+      raw_csv.each do |row|
+        row_values = []
+
+        fields.each do |field|
+          # Finds the column header name the user has designated for the expected field
+          mapped_field_key = field_mappings[field.key] if field_mappings
+          mapped_header = mapped_field_key || field.key
+
+          row_values << row.fetch(mapped_header, "")
+        end
+
+        csv << row_values
+      end
+    end
+
+    new(generated_csv_str)
+  end
+
+  attr_reader :csv_str
+
+  def initialize(csv_str, column_validators: nil)
+    @csv_str = csv_str
+    @column_validators = column_validators || {}
+  end
+
+  def table
+    @table ||= self.class.parse_csv(csv_str)
+  end
+
+  def update_cell(row_idx, col_idx, value)
+    copy = table.by_col_or_row
+    copy[row_idx][col_idx] = value
+    copy
+  end
+
+  def valid?
+    table.each_with_index.all? do |row, row_idx|
+      row.each_with_index.all? do |cell, col_idx|
+        cell_valid?(row_idx, col_idx)
+      end
+    end
+  end
+
+  def cell_valid?(row_idx, col_idx)
+    value = table.dig(row_idx, col_idx)
+    header = table.headers[col_idx]
+    validator = get_validator_by_header(header)
+    validator.call(value)
+  end
+
+  def define_validator(header_key, validator = nil, &block)
+    header = table.headers.find { |h| h.strip == header_key }
+    raise "Cannot define validator for header #{header_key}: header does not exist in CSV" if header.nil?
+
+    column_validators[header] = validator || block
+  end
+
+  private
+
+    attr_accessor :column_validators
+
+    def get_validator_by_header(header)
+      column_validators&.dig(header) || ->(_v) { true }
+    end
+end

app/models/import/field.rb

@@ -0,0 +1,32 @@
+class Import::Field
+  def self.iso_date_validator(value)
+    Date.iso8601(value)
+    true
+  rescue
+    false
+  end
+
+  def self.bigdecimal_validator(value)
+    BigDecimal(value)
+    true
+  rescue
+    false
+  end
+
+  attr_reader :key, :label, :validator
+
+  def initialize(key:, label:, validator: nil)
+    @key = key.to_s
+    @label = label
+    @validator = validator
+  end
+
+  def define_validator(validator = nil, &block)
+    @validator = validator || block
+  end
+
+  def validate(value)
+    return true if validator.nil?
+    validator.call(value)
+  end
+end