CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)

* Remove stale 1.0 import logic and model

* Fresh start

* Checkpoint before removing nav

* First working prototype

* Add trade, account, and mint import flows

* Basic working version with tests

* System tests for each import type

* Clean up mappings flow

* Clean up PR, refactor stale code, tests

* Add back row validations

* Row validations

* Fix import job test

* Fix import navigation

* Fix mint import configuration form

* Currency preset for new accounts

test/controllers/import/cleans_controller_test.rb

@@ -0,0 +1,25 @@
+require "test_helper"
+
+class Import::CleansControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+  end
+
+  test "shows if configured" do
+    import = imports(:transaction)
+
+    TransactionImport.any_instance.stubs(:configured?).returns(true)
+
+    get import_clean_path(import)
+    assert_response :success
+  end
+
+  test "redirects if not configured" do
+    import = imports(:transaction)
+
+    TransactionImport.any_instance.stubs(:configured?).returns(false)
+
+    get import_clean_path(import)
+    assert_redirected_to import_configuration_path(import)
+  end
+end

test/controllers/import/configurations_controller_test.rb

@@ -0,0 +1,33 @@
+require "test_helper"
+
+class Import::ConfigurationsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+    @import = imports(:transaction)
+  end
+
+  test "show" do
+    get import_configuration_url(@import)
+    assert_response :success
+  end
+
+  test "updating a valid configuration regenerates rows" do
+    TransactionImport.any_instance.expects(:generate_rows_from_csv).once
+
+    patch import_configuration_url(@import), params: {
+      import: {
+        date_col_label: "Date",
+        date_format: "%Y-%m-%d",
+        name_col_label: "Name",
+        category_col_label: "Category",
+        tags_col_label: "Tags",
+        amount_col_label: "Amount",
+        signage_convention: "inflows_positive",
+        account_col_label: "Account"
+      }
+    }
+
+    assert_redirected_to import_clean_url(@import)
+    assert_equal "Import configured successfully.", flash[:notice]
+  end
+end

test/controllers/import/confirms_controller_test.rb

@@ -0,0 +1,26 @@
+require "test_helper"
+
+class Import::ConfirmsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+  end
+
+  test "shows if cleaned" do
+    import = imports(:transaction)
+
+    TransactionImport.any_instance.stubs(:cleaned?).returns(true)
+
+    get import_confirm_path(import)
+    assert_response :success
+  end
+
+  test "redirects if not cleaned" do
+    import = imports(:transaction)
+
+    TransactionImport.any_instance.stubs(:cleaned?).returns(false)
+
+    get import_confirm_path(import)
+    assert_redirected_to import_clean_path(import)
+    assert_equal "You have invalid data, please edit until all errors are resolved", flash[:alert]
+  end
+end

test/controllers/import/mappings_controller_test.rb

@@ -0,0 +1,29 @@
+require "test_helper"
+
+class Import::MappingsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+
+    @import = imports(:transaction)
+  end
+
+  test "updates mapping" do
+    mapping = import_mappings(:one)
+    new_category = categories(:income)
+
+    patch import_mapping_path(@import, mapping), params: {
+      import_mapping: {
+        mappable_type: "Category",
+        mappable_id: new_category.id,
+        key: "Food"
+      }
+    }
+
+    mapping.reload
+
+    assert_equal new_category, mapping.mappable
+    assert_equal "Food", mapping.key
+
+    assert_redirected_to import_confirm_path(@import)
+  end
+end

test/controllers/import/rows_controller_test.rb

@@ -0,0 +1,79 @@
+require "test_helper"
+
+class Import::RowsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+
+    @import = imports(:transaction)
+    @row = import_rows(:one)
+  end
+
+  test "show transaction row" do
+    get import_row_path(@import, @row)
+
+    assert_row_fields(@row, [ :date, :name, :amount, :currency, :category, :tags, :account, :notes ])
+
+    assert_response :success
+  end
+
+  test "show trade row" do
+    import = @user.family.imports.create!(type: "TradeImport")
+    row = import.rows.create!(date: "01/01/2024", currency: "USD", qty: 10, price: 100, ticker: "AAPL")
+
+    get import_row_path(import, row)
+
+    assert_row_fields(row, [ :date, :ticker, :qty, :price, :currency, :account, :name ])
+
+    assert_response :success
+  end
+
+  test "show account row" do
+    import = @user.family.imports.create!(type: "AccountImport")
+    row = import.rows.create!(name: "Test Account", amount: 10000, currency: "USD")
+
+    get import_row_path(import, row)
+
+    assert_row_fields(row, [ :entity_type, :name, :amount, :currency ])
+
+    assert_response :success
+  end
+
+  test "show mint row" do
+    import = @user.family.imports.create!(type: "MintImport")
+    row = import.rows.create!(date: "01/01/2024", amount: 100, currency: "USD")
+
+    get import_row_path(import, row)
+
+    assert_row_fields(row, [ :date, :name, :amount, :currency, :category, :tags, :account, :notes ])
+
+    assert_response :success
+  end
+
+  test "update" do
+    patch import_row_path(@import, @row), params: {
+      import_row: {
+        account: "Checking Account",
+        date: "2024-01-01",
+        qty: nil,
+        ticker: nil,
+        price: nil,
+        amount: 100,
+        currency: "USD",
+        name: "Test",
+        category: "Food",
+        tags: "grocery, dinner",
+        entity_type: nil,
+        notes: "Weekly shopping"
+      }
+    }
+
+    assert_redirected_to import_row_path(@import, @row)
+  end
+
+  private
+    def assert_row_fields(row, fields)
+      fields.each do |field|
+        assert_select "turbo-frame##{dom_id(row, field)}"
+      end
+    end
+end

test/controllers/import/uploads_controller_test.rb

@@ -0,0 +1,46 @@
+require "test_helper"
+
+class Import::UploadsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    sign_in @user = users(:family_admin)
+    @import = imports(:transaction)
+  end
+
+  test "show" do
+    get import_upload_url(@import)
+    assert_response :success
+  end
+
+  test "uploads valid csv by copy and pasting" do
+    patch import_upload_url(@import), params: {
+      import: {
+        raw_file_str: file_fixture("imports/valid.csv").read
+      }
+    }
+
+    assert_redirected_to import_configuration_url(@import)
+    assert_equal "CSV uploaded successfully.", flash[:notice]
+  end
+
+  test "uploads valid csv by file" do
+    patch import_upload_url(@import), params: {
+      import: {
+        csv_file: file_fixture_upload("imports/valid.csv")
+      }
+    }
+
+    assert_redirected_to import_configuration_url(@import)
+    assert_equal "CSV uploaded successfully.", flash[:notice]
+  end
+
+  test "invalid csv cannot be uploaded" do
+    patch import_upload_url(@import), params: {
+      import: {
+        csv_file: file_fixture_upload("imports/invalid.csv")
+      }
+    }
+
+    assert_response :unprocessable_entity
+    assert_equal "Must be valid CSV with headers and at least one row of data", flash[:alert]
+  end
+end

test/controllers/imports_controller_test.rb

@@ -1,20 +1,13 @@
 require "test_helper"
 
 class ImportsControllerTest < ActionDispatch::IntegrationTest
-  include ImportTestHelper
-
   setup do
     sign_in @user = users(:family_admin)
-    @empty_import = imports(:empty_import)
-
-    @loaded_import = @empty_import.dup
-    @loaded_import.update! raw_file_str: valid_csv_str
-
-    @completed_import = imports(:completed_import)
   end
 
-  test "should get index" do
+  test "gets index" do
     get imports_url
+
     assert_response :success
 
     @user.family.imports.ordered.each do |import|
@@ -22,152 +15,44 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
     end
   end
 
-  test "should get new" do
+  test "gets new" do
     get new_import_url
+
     assert_response :success
+
+    assert_select "turbo-frame#modal"
   end
 
-  test "should create import" do
-    assert_difference("Import.count") do
-      post imports_url, params: { import: { account_id: @user.family.accounts.first.id, col_sep: "," } }
-    end
-
-    assert_redirected_to load_import_path(Import.ordered.first)
-  end
-
-  test "should get edit" do
-    get edit_import_url(@empty_import)
-    assert_response :success
-  end
-
-  test "should update import" do
-    patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id, col_sep: "," } }
-    assert_redirected_to load_import_path(@empty_import)
-  end
-
-  test "should destroy import" do
-    assert_difference("Import.count", -1) do
-      delete import_url(@empty_import)
-    end
-
-    assert_redirected_to imports_url
-  end
-
-  test "should get load" do
-    get load_import_url(@empty_import)
-    assert_response :success
-  end
-
-  test "should save raw CSV if valid" do
-    patch load_import_url(@empty_import), params: { import: { raw_file_str: valid_csv_str } }
-
-    assert_redirected_to configure_import_path(@empty_import)
-    assert_equal "Import CSV loaded", flash[:notice]
-  end
-
-  test "should upload CSV file if valid" do
-    Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
-      CSV.open(temp, "wb", headers: true) do |csv|
-        valid_csv_str.split("\n").each { |row| csv << row.split(",") }
-      end
-
-      patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
-      assert_redirected_to configure_import_path(@empty_import)
-      assert_equal "CSV File loaded", flash[:notice]
-    end
-  end
-
-  test "should flash error message if invalid CSV input" do
-    patch load_import_url(@empty_import), params: { import: { raw_file_str: malformed_csv_str } }
-
-    assert_response :unprocessable_entity
-    assert_equal "Raw file str is not a valid CSV format", flash[:alert]
-  end
-
-  test "should flash error message if invalid CSV file upload" do
-    Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
-      temp.write(malformed_csv_str)
-      temp.rewind
-
-      patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
-      assert_response :unprocessable_entity
-      assert_equal "Raw file str is not a valid CSV format", flash[:alert]
-    end
-  end
-
-  test "should flash error message if no fileprovided for upload" do
-    patch upload_import_url(@empty_import), params: { import: { raw_file_str: nil } }
-    assert_response :unprocessable_entity
-    assert_equal "Please select a file to upload", flash[:alert]
-  end
-
-  test "should get configure" do
-    get configure_import_url(@loaded_import)
-    assert_response :success
-  end
-
-  test "should redirect back to load step with an alert message if not loaded" do
-    get configure_import_url(@empty_import)
-    assert_equal "Please load a CSV first", flash[:alert]
-    assert_redirected_to load_import_path(@empty_import)
-  end
-
-  test "should update mappings" do
-    patch configure_import_url(@loaded_import), params: {
-      import: {
-        column_mappings: {
-          date: "date",
-          name: "name",
-          category: "category",
-          amount: "amount"
+  test "creates import" do
+    assert_difference "Import.count", 1 do
+      post imports_url, params: {
+        import: {
+          type: "TransactionImport"
         }
       }
-    }
+    end
 
-    assert_redirected_to clean_import_path(@loaded_import)
-    assert_equal "Column mappings saved", flash[:notice]
+    assert_redirected_to import_upload_url(Import.all.ordered.first)
   end
 
-  test "can update a cell" do
-    assert_equal @loaded_import.csv.table[0][1], "Starbucks drink"
+  test "publishes import" do
+    import = imports(:transaction)
 
-    patch clean_import_url(@loaded_import), params: {
-      import: {
-        csv_update: {
-          row_idx: 0,
-          col_idx: 1,
-          value: "new_merchant"
-        }
-      }
-    }
+    TransactionImport.any_instance.expects(:publish_later).once
 
-    assert_response :success
+    post publish_import_url(import)
 
-    @loaded_import.reload
-    assert_equal "new_merchant", @loaded_import.csv.table[0][1]
+    assert_equal "Your import has started in the background.", flash[:notice]
+    assert_redirected_to import_path(import)
   end
 
-  test "should get clean" do
-    get clean_import_url(@loaded_import)
-    assert_response :success
-  end
+  test "destroys import" do
+    import = imports(:transaction)
 
-  test "should get confirm if all values are valid" do
-    get confirm_import_url(@loaded_import)
-    assert_response :success
-  end
+    assert_difference "Import.count", -1 do
+      delete import_url(import)
+    end
 
-  test "should redirect back to clean if data is invalid" do
-    @empty_import.update! raw_file_str: valid_csv_with_invalid_values
-
-    get confirm_import_url(@empty_import)
-    assert_equal "You have invalid data, please fix before continuing", flash[:alert]
-    assert_redirected_to clean_import_path(@empty_import)
-  end
-
-  test "should confirm import" do
-    patch confirm_import_url(@loaded_import)
     assert_redirected_to imports_path
-    assert_equal "Import has started in the background", flash[:notice]
   end
 end