CSV Transaction Imports (#708)

Introduces a basic CSV import module for bulk-importing account transactions.

Changes include:

- User can load a CSV
- User can configure the column mappings for a CSV
- Imported CSV shows invalid cells
- User can clean up their data directly in the UI
- User can see a preview of the import rows and confirm import
- Layout refactor + Import nav stepper
- System test stability improvements

test/application_system_test_case.rb

@@ -1,21 +1,27 @@
 require "test_helper"
 
 class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
-  driven_by :selenium, using: :headless_chrome, screen_size: [ 1400, 1400 ]
+  driven_by :selenium, using: ENV["CI"].present? ? :headless_chrome : :chrome, screen_size: [ 1400, 1400 ]
 
   private
 
-  def sign_in(user)
-    visit new_session_path
-    within "form" do
-      fill_in "Email", with: user.email
-      fill_in "Password", with: "password"
-      click_button "Log in"
-    end
-  end
+    def sign_in(user)
+      visit new_session_path
+      within "form" do
+        fill_in "Email", with: user.email
+        fill_in "Password", with: "password"
+        click_on "Log in"
+      end
 
-  def sign_out
-    find("#user-menu").click
-    click_button "Logout"
-  end
+      # Trigger Capybara's wait mechanism to avoid timing issues with logins
+      find("h1", text: "Dashboard")
+    end
+
+    def sign_out
+      find("#user-menu").click
+      click_button "Logout"
+
+      # Trigger Capybara's wait mechanism to avoid timing issues with logout
+      find("h2", text: "Sign in to your account")
+    end
 end

test/controllers/imports_controller_test.rb

@@ -0,0 +1,141 @@
+require "test_helper"
+
+class ImportsControllerTest < ActionDispatch::IntegrationTest
+  include ImportTestHelper
+
+  setup do
+    sign_in @user = users(:family_admin)
+    @empty_import = imports(:empty_import)
+    @loaded_import = imports(:loaded_import)
+    @completed_import = imports(:completed_import)
+  end
+
+  test "should get index" do
+    get imports_url
+    assert_response :success
+
+    @user.family.imports.ordered.each do |import|
+      assert_select "#" + dom_id(import), count: 1
+    end
+  end
+
+  test "should get new" do
+    get new_import_url
+    assert_response :success
+  end
+
+  test "should create import" do
+    assert_difference("Import.count") do
+      post imports_url, params: { import: { account_id: @user.family.accounts.first.id } }
+    end
+
+    assert_redirected_to load_import_path(Import.ordered.first)
+  end
+
+  test "should get edit" do
+    get edit_import_url(@empty_import)
+    assert_response :success
+  end
+
+  test "should update import" do
+    patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id } }
+    assert_redirected_to load_import_path(@empty_import)
+  end
+
+  test "should destroy import" do
+    assert_difference("Import.count", -1) do
+      delete import_url(@empty_import)
+    end
+
+    assert_redirected_to imports_url
+  end
+
+  test "should get load" do
+    get load_import_url(@empty_import)
+    assert_response :success
+  end
+
+  test "should save raw CSV if valid" do
+    patch load_import_url(@empty_import), params: { import: { raw_csv_str: valid_csv_str } }
+
+    assert_redirected_to configure_import_path(@empty_import)
+    assert_equal "Import CSV loaded", flash[:notice]
+  end
+
+  test "should flash error message if invalid CSV input" do
+    patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }
+
+    assert_response :unprocessable_entity
+    assert_equal "Raw csv str is not a valid CSV format", flash[:error]
+  end
+
+  test "should get configure" do
+    get configure_import_url(@loaded_import)
+    assert_response :success
+  end
+
+  test "should redirect back to load step with an alert message if not loaded" do
+    get configure_import_url(@empty_import)
+    assert_equal "Please load a CSV first", flash[:alert]
+    assert_redirected_to load_import_path(@empty_import)
+  end
+
+  test "should update mappings" do
+    patch configure_import_url(@loaded_import), params: {
+      import: {
+        column_mappings: {
+          date: "date",
+          name: "name",
+          category: "category",
+          amount: "amount"
+        }
+      }
+    }
+
+    assert_redirected_to clean_import_path(@loaded_import)
+    assert_equal "Column mappings saved", flash[:notice]
+  end
+
+  test "can update a cell" do
+    assert_equal @loaded_import.csv.table[0][1], "Starbucks drink"
+
+    patch clean_import_url(@loaded_import), params: {
+      import: {
+        csv_update: {
+          row_idx: 0,
+          col_idx: 1,
+          value: "new_merchant"
+        }
+      }
+    }
+
+    assert_response :success
+
+    @loaded_import.reload
+    assert_equal "new_merchant", @loaded_import.csv.table[0][1]
+  end
+
+  test "should get clean" do
+    get clean_import_url(@loaded_import)
+    assert_response :success
+  end
+
+  test "should get confirm if all values are valid" do
+    get confirm_import_url(@loaded_import)
+    assert_response :success
+  end
+
+  test "should redirect back to clean if data is invalid" do
+    @empty_import.update! raw_csv_str: valid_csv_with_invalid_values
+
+    get confirm_import_url(@empty_import)
+    assert_equal "You have invalid data, please fix before continuing", flash[:alert]
+    assert_redirected_to clean_import_path(@empty_import)
+  end
+
+  test "should confirm import" do
+    patch confirm_import_url(@loaded_import)
+    assert_redirected_to imports_path
+    assert_equal "Import has started in the background", flash[:notice]
+  end
+end

test/fixtures/imports.yml

@@ -0,0 +1,32 @@
+empty_import:
+  account: checking
+  created_at: <%= 1.minute.ago %>
+
+loaded_import:
+  account: checking
+  raw_csv_str: |
+    date,name,category,amount
+    2024-01-01,Starbucks drink,Food,20
+    2024-01-02,Amazon stuff,Shopping,200
+  normalized_csv_str: |
+    date,name,category,amount
+    2024-01-01,Starbucks drink,Food,20
+    2024-01-02,Amazon stuff,Shopping,200
+  created_at: <%= 2.days.ago %>
+
+completed_import:
+  account: checking
+  column_mappings:
+    date: date
+    name: name
+    category: category
+    amount: amount
+  raw_csv_str: |
+    date,name,category,amount
+    2024-01-01,Starbucks drink,Food,20
+  normalized_csv_str: |
+    date,name,category,amount
+    2024-01-01,Starbucks drink,Food,20
+  created_at: <%= 2.days.ago %>
+
+

test/jobs/import_job_test.rb

@@ -0,0 +1,18 @@
+require "test_helper"
+
+class ImportJobTest < ActiveJob::TestCase
+  include ImportTestHelper
+
+  test "import is published" do
+    import = imports(:empty_import)
+    import.update! raw_csv_str: valid_csv_str
+
+    assert import.pending?
+
+    perform_enqueued_jobs do
+      ImportJob.perform_later(import)
+    end
+
+    assert import.reload.complete?
+  end
+end

test/models/import/csv_test.rb

@@ -0,0 +1,87 @@
+require "test_helper"
+
+class Import::CsvTest < ActiveSupport::TestCase
+  include ImportTestHelper
+
+  setup do
+    @csv = Import::Csv.new(valid_csv_str)
+  end
+
+  test "cannot define validator for non-existent header" do
+    assert_raises do
+      @csv.define_validator "invalid", method(:validate_iso_date)
+    end
+  end
+
+  test "csv with no validators is valid" do
+    assert @csv.cell_valid?(0, 0)
+    assert @csv.valid?
+  end
+
+  test "valid csv values" do
+    @csv.define_validator "date", method(:validate_iso_date)
+
+    assert_equal "2024-01-01", @csv.table[0][0]
+    assert @csv.cell_valid?(0, 0)
+    assert @csv.valid?
+  end
+
+  test "invalid csv values" do
+    invalid_csv = Import::Csv.new valid_csv_with_invalid_values
+
+    invalid_csv.define_validator "date", method(:validate_iso_date)
+
+    assert_equal "invalid_date", invalid_csv.table[0][0]
+    assert_not invalid_csv.cell_valid?(0, 0)
+    assert_not invalid_csv.valid?
+  end
+
+  test "updating a cell returns a copy of the original csv" do
+    original_date = "2024-01-01"
+    new_date = "2024-01-01"
+
+    assert_equal original_date, @csv.table[0][0]
+    updated = @csv.update_cell(0, 0, new_date)
+
+    assert_equal original_date, @csv.table[0][0]
+    assert_equal new_date, updated[0][0]
+  end
+
+  test "can create CSV with expected columns and field mappings with validators" do
+    date_field = Import::Field.new \
+      key: "date",
+      label: "Date",
+      validator: method(:validate_iso_date)
+
+    name_field = Import::Field.new \
+      key: "name",
+      label: "Name"
+
+    fields = [ date_field, name_field ]
+
+    raw_csv_str = <<-ROWS
+      date,Custom Field Header,extra_field
+      invalid_date_value,Starbucks drink,Food
+      2024-01-02,Amazon stuff,Shopping
+    ROWS
+
+    mappings = {
+      "name" => "Custom Field Header"
+    }
+
+    csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings)
+
+    assert_equal %w[ date name ], csv.table.headers
+    assert_equal 2, csv.table.size
+    assert_equal "Amazon stuff", csv.table[1][1]
+  end
+
+  private
+
+    def validate_iso_date(value)
+      Date.iso8601(value)
+      true
+    rescue
+      false
+    end
+end

test/models/import/field_test.rb

@@ -0,0 +1,28 @@
+require "test_helper"
+
+class Import::FieldTest < ActiveSupport::TestCase
+  test "key is always a string" do
+    field1 = Import::Field.new label: "Test", key: "test"
+    field2 = Import::Field.new label: "Test2", key: :test2
+
+    assert_equal "test", field1.key
+    assert_equal "test2", field2.key
+  end
+
+  test "can set and override a validator for a field" do
+    field = Import::Field.new \
+      label: "Test",
+      key: "Test",
+      validator: ->(val) { val == 42 }
+
+    assert field.validate(42)
+    assert_not field.validate(41)
+
+    field.define_validator do |value|
+      value == 100
+    end
+
+    assert field.validate(100)
+    assert_not field.validate(42)
+  end
+end

test/models/import_test.rb

@@ -0,0 +1,61 @@
+require "test_helper"
+
+class ImportTest < ActiveSupport::TestCase
+  include ImportTestHelper, ActiveJob::TestHelper
+
+  setup do
+    @empty_import = imports(:empty_import)
+    @loaded_import = imports(:loaded_import)
+  end
+
+  test "raw csv input must conform to csv spec" do
+    @empty_import.raw_csv_str = malformed_csv_str
+    assert_not @empty_import.valid?
+
+    @empty_import.raw_csv_str = valid_csv_str
+    assert @empty_import.valid?
+  end
+
+  test "can update csv value without affecting raw input" do
+    assert_equal "Starbucks drink", @loaded_import.csv.table[0][1]
+
+    prior_raw_csv_str_value = @loaded_import.raw_csv_str
+    prior_normalized_csv_str_value = @loaded_import.normalized_csv_str
+
+    @loaded_import.update_csv! \
+      row_idx: 0,
+      col_idx: 1,
+      value: "new_category"
+
+    assert_equal "new_category", @loaded_import.csv.table[0][1]
+    assert_equal prior_raw_csv_str_value, @loaded_import.raw_csv_str
+    assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str
+  end
+
+  test "publishes later" do
+    assert_enqueued_with(job: ImportJob) do
+      @loaded_import.publish_later
+    end
+  end
+
+  test "publishes a valid import" do
+    assert_difference "Transaction.count", 2 do
+      @loaded_import.publish
+    end
+
+    @loaded_import.reload
+
+    assert @loaded_import.complete?
+  end
+
+  test "failed publish results in error status" do
+    @empty_import.update! raw_csv_str: valid_csv_with_invalid_values
+
+    assert_difference "Transaction.count", 0 do
+      @empty_import.publish
+    end
+
+    @empty_import.reload
+    assert @empty_import.failed?
+  end
+end

test/support/import_test_helper.rb

@@ -0,0 +1,24 @@
+module ImportTestHelper
+  def valid_csv_str
+    <<-ROWS
+      date,name,category,amount
+      2024-01-01,Starbucks drink,Food,20
+      2024-01-02,Amazon stuff,Shopping,200
+    ROWS
+  end
+
+  def valid_csv_with_invalid_values
+    <<-ROWS
+      date,name,category,amount
+      invalid_date,Starbucks drink,Food,invalid_amount
+    ROWS
+  end
+
+  def malformed_csv_str
+    <<-ROWS
+      name,age
+      "John Doe,23
+      "Jane Doe",25
+    ROWS
+  end
+end

test/system/imports_test.rb

@@ -0,0 +1,113 @@
+require "application_system_test_case"
+
+class ImportsTest < ApplicationSystemTestCase
+  include ImportTestHelper
+
+  setup do
+    sign_in @user = users(:family_admin)
+
+    @imports = @user.family.imports.ordered.to_a
+  end
+
+  test "can trigger new import from settings" do
+    trigger_import_from_settings
+    verify_import_modal
+  end
+
+  test "can resume existing import from settings" do
+    visit imports_url
+
+    within "#" + dom_id(@imports.first) do
+      click_button
+      click_link "Edit"
+    end
+
+    assert_current_path edit_import_path(@imports.first)
+  end
+
+  test "can resume latest import" do
+    trigger_import_from_transactions
+    verify_import_modal
+
+    click_link "Resume latest import"
+
+    assert_current_path edit_import_path(@imports.first)
+  end
+
+  test "can perform basic CSV import" do
+    trigger_import_from_settings
+    verify_import_modal
+
+    within "#modal" do
+      click_link "New import from CSV"
+    end
+
+    # 1) Create import step
+    assert_selector "h1", text: "New import"
+
+    within "form" do
+      select "Checking Account", from: "import_account_id"
+    end
+
+    click_button "Next"
+
+    # 2) Load Step
+    assert_selector "h1", text: "Load import"
+
+    within "form" do
+      fill_in "import_raw_csv_str", with: <<-ROWS
+        date,Custom Name Column,category,amount
+        invalid_date,Starbucks drink,Food,-20.50
+        2024-01-01,Amazon purchase,Shopping,-89.50
+      ROWS
+    end
+
+    click_button "Next"
+
+    # 3) Configure step
+    assert_selector "h1", text: "Configure import"
+
+    within "form" do
+      select "Custom Name Column", from: "import_column_mappings_name"
+    end
+
+    click_button "Next"
+
+    # 4) Clean step
+    assert_selector "h1", text: "Clean import"
+
+    # We have an invalid value, so user cannot click next yet
+    assert_no_text "Next"
+
+    # Replace invalid date with valid date
+    fill_in "cell-0-0", with: "2024-01-02"
+
+    # Trigger blur event so value saves
+    find("body").click
+
+    click_link "Next"
+
+    # 5) Confirm step
+    assert_selector "h1", text: "Confirm import"
+    click_button "Import 2 transactions"
+    assert_selector "h1", text: "Imports"
+  end
+
+  private
+
+    def trigger_import_from_settings
+      visit imports_url
+      click_link "New import"
+    end
+
+    def trigger_import_from_transactions
+      visit transactions_url
+      click_link "Import"
+    end
+
+    def verify_import_modal
+      within "#modal" do
+        assert_text "Import transactions"
+      end
+    end
+end

test/system/settings_test.rb

@@ -14,6 +14,7 @@ class SettingsTest < ApplicationSystemTestCase
       [ "Categories", "Categories", transaction_categories_path ],
       [ "Merchants", "Merchants", transaction_merchants_path ],
       [ "Rules", "Rules", transaction_rules_path ],
+      [ "Imports", "Imports", imports_path ],
       [ "What's New", "What's New", changelog_path ],
       [ "Feedback", "Feedback", feedback_path ],
       [ "Invite friends", "Invite friends", invites_path ]
@@ -27,6 +28,7 @@ class SettingsTest < ApplicationSystemTestCase
   end
 
   private
+
     def open_settings_from_sidebar
       find("#user-menu").click
       click_link "Settings"