CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)

* Remove stale 1.0 import logic and model

* Fresh start

* Checkpoint before removing nav

* First working prototype

* Add trade, account, and mint import flows

* Basic working version with tests

* System tests for each import type

* Clean up mappings flow

* Clean up PR, refactor stale code, tests

* Add back row validations

* Row validations

* Fix import job test

* Fix import navigation

* Fix mint import configuration form

* Currency preset for new accounts

app/controllers/accounts_controller.rb

@@ -23,7 +23,10 @@ class AccountsController < ApplicationController
   end
 
   def new
-    @account = Account.new(accountable: Accountable.from_type(params[:type])&.new)
+    @account = Account.new(
+      accountable: Accountable.from_type(params[:type])&.new,
+      currency: Current.family.currency
+    )
 
     @account.accountable.address = Address.new if @account.accountable.is_a?(Property)
 

app/controllers/import/cleans_controller.rb

@@ -0,0 +1,22 @@
+class Import::CleansController < ApplicationController
+  layout "imports"
+
+  before_action :set_import
+
+  def show
+    redirect_to import_configuration_path(@import), alert: "Please configure your import before proceeding." unless @import.configured?
+
+    rows = @import.rows.ordered
+
+    if params[:view] == "errors"
+      rows = rows.reject { |row| row.valid? }
+    end
+
+    @pagy, @rows = pagy_array(rows, limit: params[:per_page] || "10")
+  end
+
+  private
+    def set_import
+      @import = Current.family.imports.find(params[:import_id])
+    end
+end

app/controllers/import/configurations_controller.rb

@@ -0,0 +1,25 @@
+class Import::ConfigurationsController < ApplicationController
+  layout "imports"
+
+  before_action :set_import
+
+  def show
+  end
+
+  def update
+    @import.update!(import_params)
+    @import.generate_rows_from_csv
+    @import.reload.sync_mappings
+
+    redirect_to import_clean_path(@import), notice: "Import configured successfully."
+  end
+
+  private
+    def set_import
+      @import = Current.family.imports.find(params[:import_id])
+    end
+
+    def import_params
+      params.require(:import).permit(:date_col_label, :date_format, :name_col_label, :category_col_label, :tags_col_label, :amount_col_label, :signage_convention, :account_col_label, :notes_col_label, :entity_type_col_label)
+    end
+end

app/controllers/import/confirms_controller.rb

@@ -0,0 +1,14 @@
+class Import::ConfirmsController < ApplicationController
+  layout "imports"
+
+  before_action :set_import
+
+  def show
+    redirect_to import_clean_path(@import), alert: "You have invalid data, please edit until all errors are resolved" unless @import.cleaned?
+  end
+
+  private
+    def set_import
+      @import = Current.family.imports.find(params[:import_id])
+    end
+end

app/controllers/import/mappings_controller.rb

@@ -0,0 +1,43 @@
+class Import::MappingsController < ApplicationController
+  before_action :set_import
+
+  def update
+    mapping = @import.mappings.find(params[:id])
+
+    mapping.update! \
+      create_when_empty: create_when_empty,
+      mappable: mappable,
+      value: mapping_params[:value]
+
+    redirect_back_or_to import_confirm_path(@import)
+  end
+
+  private
+    def mapping_params
+      params.require(:import_mapping).permit(:type, :key, :mappable_id, :mappable_type, :value)
+    end
+
+    def set_import
+      @import = Current.family.imports.find(params[:import_id])
+    end
+
+    def mappable
+      return nil unless mappable_class.present?
+
+      @mappable ||= mappable_class.find_by(id: mapping_params[:mappable_id], family: Current.family)
+    end
+
+    def create_when_empty
+      return false unless mapping_class.present?
+
+      mapping_params[:mappable_id] == mapping_class::CREATE_NEW_KEY
+    end
+
+    def mappable_class
+      mapping_params[:mappable_type]&.constantize
+    end
+
+    def mapping_class
+      mapping_params[:type]&.constantize
+    end
+end

app/controllers/import/rows_controller.rb

@@ -0,0 +1,24 @@
+class Import::RowsController < ApplicationController
+  before_action :set_import_row
+
+  def update
+    @row.assign_attributes(row_params)
+    @row.save!(validate: false)
+    @row.sync_mappings
+
+    redirect_to import_row_path(@row.import, @row)
+  end
+
+  def show
+  end
+
+  private
+    def row_params
+      params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes)
+    end
+
+    def set_import_row
+      @import = Current.family.imports.find(params[:import_id])
+      @row = @import.rows.find(params[:id])
+    end
+end

app/controllers/import/uploads_controller.rb

@@ -0,0 +1,47 @@
+class Import::UploadsController < ApplicationController
+  layout "imports"
+
+  before_action :set_import
+
+  def show
+  end
+
+  def update
+    if csv_valid?(csv_str)
+      @import.assign_attributes(raw_file_str: csv_str, col_sep: upload_params[:col_sep])
+      @import.save!(validate: false)
+
+      redirect_to import_configuration_path(@import), notice: "CSV uploaded successfully."
+    else
+      flash.now[:alert] = "Must be valid CSV with headers and at least one row of data"
+
+      render :show, status: :unprocessable_entity
+    end
+  end
+
+  private
+    def set_import
+      @import = Current.family.imports.find(params[:import_id])
+    end
+
+    def csv_str
+      @csv_str ||= upload_params[:csv_file]&.read || upload_params[:raw_file_str]
+    end
+
+    def csv_valid?(str)
+      require "csv"
+
+      begin
+        csv = CSV.parse(str || "", headers: true)
+        return false if csv.headers.empty?
+        return false if csv.count == 0
+        true
+      rescue CSV::MalformedCSVError
+        false
+      end
+    end
+
+    def upload_params
+      params.require(:import).permit(:raw_file_str, :csv_file, :col_sep)
+    end
+end

app/controllers/imports_controller.rb

@@ -1,118 +1,44 @@
-require "ostruct"
-
 class ImportsController < ApplicationController
-  before_action :set_import, except: %i[index new create]
+  before_action :set_import, only: %i[show publish destroy]
+
+  def publish
+    @import.publish_later
+
+    redirect_to import_path(@import), notice: "Your import has started in the background."
+  end
 
   def index
     @imports = Current.family.imports
-    render layout: "with_sidebar"
+
+    render layout: with_sidebar
   end
 
   def new
-    account = Current.family.accounts.find_by(id: params[:account_id])
-    @import = Import.new account: account
-  end
-
-  def edit
-  end
-
-  def update
-    account = Current.family.accounts.find(params[:import][:account_id])
-    @import.update! account: account, col_sep: params[:import][:col_sep]
-
-    redirect_to load_import_path(@import), notice: t(".import_updated")
+    @pending_import = Current.family.imports.ordered.pending.first
   end
 
   def create
-    account = Current.family.accounts.find(params[:import][:account_id])
-    @import = Import.create! account: account, col_sep: params[:import][:col_sep]
+    import = Current.family.imports.create! import_params
 
-    redirect_to load_import_path(@import), notice: t(".import_created")
+    redirect_to import_upload_path(import)
+  end
+
+  def show
+    redirect_to import_confirm_path(@import), alert: "Please finalize your mappings before proceeding." unless @import.publishable?
   end
 
   def destroy
-    @import.destroy!
-    redirect_to imports_url, notice: t(".import_destroyed"), status: :see_other
-  end
+    @import.destroy
 
-  def load
-  end
-
-  def upload_csv
-    begin
-      @import.raw_file_str = import_params[:raw_file_str].read
-    rescue NoMethodError
-      flash.now[:alert] = "Please select a file to upload"
-      render :load, status: :unprocessable_entity and return
-    end
-    if @import.save
-      redirect_to configure_import_path(@import), notice: t(".import_loaded")
-    else
-      flash.now[:alert] = @import.errors.full_messages.to_sentence
-      render :load, status: :unprocessable_entity
-    end
-  end
-
-  def load_csv
-    if @import.update(import_params)
-      redirect_to configure_import_path(@import), notice: t(".import_loaded")
-    else
-      flash.now[:alert] = @import.errors.full_messages.to_sentence
-      render :load, status: :unprocessable_entity
-    end
-  end
-
-  def configure
-    unless @import.loaded?
-      redirect_to load_import_path(@import), alert: t(".invalid_csv")
-    end
-  end
-
-  def update_mappings
-    @import.update! import_params(@import.expected_fields.map(&:key))
-    redirect_to clean_import_path(@import), notice: t(".column_mappings_saved")
-  end
-
-  def clean
-    unless @import.loaded?
-      redirect_to load_import_path(@import), alert: t(".invalid_csv")
-    end
-  end
-
-  def update_csv
-    update_params = import_params[:csv_update]
-
-    @import.update_csv! \
-      row_idx: update_params[:row_idx],
-      col_idx: update_params[:col_idx],
-      value: update_params[:value]
-
-    render :clean
-  end
-
-  def confirm
-    unless @import.cleaned?
-      redirect_to clean_import_path(@import), alert: t(".invalid_data")
-    end
-  end
-
-  def publish
-    if @import.valid?
-      @import.publish_later
-      redirect_to imports_path, notice: t(".import_published")
-    else
-      flash.now[:error] = t(".invalid_data")
-      render :confirm, status: :unprocessable_entity
-    end
+    redirect_to imports_path, notice: "Your import has been deleted."
   end
 
   private
-
     def set_import
       @import = Current.family.imports.find(params[:id])
     end
 
-    def import_params(permitted_mappings = nil)
-      params.require(:import).permit(:raw_file_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
+    def import_params
+      params.require(:import).permit(:type)
     end
 end