1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-07-25 08:09:38 +02:00

CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)

* Remove stale 1.0 import logic and model

* Fresh start

* Checkpoint before removing nav

* First working prototype

* Add trade, account, and mint import flows

* Basic working version with tests

* System tests for each import type

* Clean up mappings flow

* Clean up PR, refactor stale code, tests

* Add back row validations

* Row validations

* Fix import job test

* Fix import navigation

* Fix mint import configuration form

* Currency preset for new accounts
This commit is contained in:
Zach Gollwitzer 2024-10-01 10:47:59 -04:00 committed by GitHub
parent 23786b444a
commit 398b246965
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
103 changed files with 2420 additions and 1689 deletions

View file

@ -0,0 +1,25 @@
require "test_helper"
class Import::CleansControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
end
test "shows if configured" do
import = imports(:transaction)
TransactionImport.any_instance.stubs(:configured?).returns(true)
get import_clean_path(import)
assert_response :success
end
test "redirects if not configured" do
import = imports(:transaction)
TransactionImport.any_instance.stubs(:configured?).returns(false)
get import_clean_path(import)
assert_redirected_to import_configuration_path(import)
end
end

View file

@ -0,0 +1,33 @@
require "test_helper"
class Import::ConfigurationsControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
@import = imports(:transaction)
end
test "show" do
get import_configuration_url(@import)
assert_response :success
end
test "updating a valid configuration regenerates rows" do
TransactionImport.any_instance.expects(:generate_rows_from_csv).once
patch import_configuration_url(@import), params: {
import: {
date_col_label: "Date",
date_format: "%Y-%m-%d",
name_col_label: "Name",
category_col_label: "Category",
tags_col_label: "Tags",
amount_col_label: "Amount",
signage_convention: "inflows_positive",
account_col_label: "Account"
}
}
assert_redirected_to import_clean_url(@import)
assert_equal "Import configured successfully.", flash[:notice]
end
end

View file

@ -0,0 +1,26 @@
require "test_helper"
class Import::ConfirmsControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
end
test "shows if cleaned" do
import = imports(:transaction)
TransactionImport.any_instance.stubs(:cleaned?).returns(true)
get import_confirm_path(import)
assert_response :success
end
test "redirects if not cleaned" do
import = imports(:transaction)
TransactionImport.any_instance.stubs(:cleaned?).returns(false)
get import_confirm_path(import)
assert_redirected_to import_clean_path(import)
assert_equal "You have invalid data, please edit until all errors are resolved", flash[:alert]
end
end

View file

@ -0,0 +1,29 @@
require "test_helper"
class Import::MappingsControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
@import = imports(:transaction)
end
test "updates mapping" do
mapping = import_mappings(:one)
new_category = categories(:income)
patch import_mapping_path(@import, mapping), params: {
import_mapping: {
mappable_type: "Category",
mappable_id: new_category.id,
key: "Food"
}
}
mapping.reload
assert_equal new_category, mapping.mappable
assert_equal "Food", mapping.key
assert_redirected_to import_confirm_path(@import)
end
end

View file

@ -0,0 +1,79 @@
require "test_helper"
class Import::RowsControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
@import = imports(:transaction)
@row = import_rows(:one)
end
test "show transaction row" do
get import_row_path(@import, @row)
assert_row_fields(@row, [ :date, :name, :amount, :currency, :category, :tags, :account, :notes ])
assert_response :success
end
test "show trade row" do
import = @user.family.imports.create!(type: "TradeImport")
row = import.rows.create!(date: "01/01/2024", currency: "USD", qty: 10, price: 100, ticker: "AAPL")
get import_row_path(import, row)
assert_row_fields(row, [ :date, :ticker, :qty, :price, :currency, :account, :name ])
assert_response :success
end
test "show account row" do
import = @user.family.imports.create!(type: "AccountImport")
row = import.rows.create!(name: "Test Account", amount: 10000, currency: "USD")
get import_row_path(import, row)
assert_row_fields(row, [ :entity_type, :name, :amount, :currency ])
assert_response :success
end
test "show mint row" do
import = @user.family.imports.create!(type: "MintImport")
row = import.rows.create!(date: "01/01/2024", amount: 100, currency: "USD")
get import_row_path(import, row)
assert_row_fields(row, [ :date, :name, :amount, :currency, :category, :tags, :account, :notes ])
assert_response :success
end
test "update" do
patch import_row_path(@import, @row), params: {
import_row: {
account: "Checking Account",
date: "2024-01-01",
qty: nil,
ticker: nil,
price: nil,
amount: 100,
currency: "USD",
name: "Test",
category: "Food",
tags: "grocery, dinner",
entity_type: nil,
notes: "Weekly shopping"
}
}
assert_redirected_to import_row_path(@import, @row)
end
private
def assert_row_fields(row, fields)
fields.each do |field|
assert_select "turbo-frame##{dom_id(row, field)}"
end
end
end

View file

@ -0,0 +1,46 @@
require "test_helper"
class Import::UploadsControllerTest < ActionDispatch::IntegrationTest
setup do
sign_in @user = users(:family_admin)
@import = imports(:transaction)
end
test "show" do
get import_upload_url(@import)
assert_response :success
end
test "uploads valid csv by copy and pasting" do
patch import_upload_url(@import), params: {
import: {
raw_file_str: file_fixture("imports/valid.csv").read
}
}
assert_redirected_to import_configuration_url(@import)
assert_equal "CSV uploaded successfully.", flash[:notice]
end
test "uploads valid csv by file" do
patch import_upload_url(@import), params: {
import: {
csv_file: file_fixture_upload("imports/valid.csv")
}
}
assert_redirected_to import_configuration_url(@import)
assert_equal "CSV uploaded successfully.", flash[:notice]
end
test "invalid csv cannot be uploaded" do
patch import_upload_url(@import), params: {
import: {
csv_file: file_fixture_upload("imports/invalid.csv")
}
}
assert_response :unprocessable_entity
assert_equal "Must be valid CSV with headers and at least one row of data", flash[:alert]
end
end

View file

@ -1,20 +1,13 @@
require "test_helper"
class ImportsControllerTest < ActionDispatch::IntegrationTest
include ImportTestHelper
setup do
sign_in @user = users(:family_admin)
@empty_import = imports(:empty_import)
@loaded_import = @empty_import.dup
@loaded_import.update! raw_file_str: valid_csv_str
@completed_import = imports(:completed_import)
end
test "should get index" do
test "gets index" do
get imports_url
assert_response :success
@user.family.imports.ordered.each do |import|
@ -22,152 +15,44 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
end
end
test "should get new" do
test "gets new" do
get new_import_url
assert_response :success
assert_select "turbo-frame#modal"
end
test "should create import" do
assert_difference("Import.count") do
post imports_url, params: { import: { account_id: @user.family.accounts.first.id, col_sep: "," } }
end
assert_redirected_to load_import_path(Import.ordered.first)
end
test "should get edit" do
get edit_import_url(@empty_import)
assert_response :success
end
test "should update import" do
patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id, col_sep: "," } }
assert_redirected_to load_import_path(@empty_import)
end
test "should destroy import" do
assert_difference("Import.count", -1) do
delete import_url(@empty_import)
end
assert_redirected_to imports_url
end
test "should get load" do
get load_import_url(@empty_import)
assert_response :success
end
test "should save raw CSV if valid" do
patch load_import_url(@empty_import), params: { import: { raw_file_str: valid_csv_str } }
assert_redirected_to configure_import_path(@empty_import)
assert_equal "Import CSV loaded", flash[:notice]
end
test "should upload CSV file if valid" do
Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
CSV.open(temp, "wb", headers: true) do |csv|
valid_csv_str.split("\n").each { |row| csv << row.split(",") }
end
patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
assert_redirected_to configure_import_path(@empty_import)
assert_equal "CSV File loaded", flash[:notice]
end
end
test "should flash error message if invalid CSV input" do
patch load_import_url(@empty_import), params: { import: { raw_file_str: malformed_csv_str } }
assert_response :unprocessable_entity
assert_equal "Raw file str is not a valid CSV format", flash[:alert]
end
test "should flash error message if invalid CSV file upload" do
Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
temp.write(malformed_csv_str)
temp.rewind
patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
assert_response :unprocessable_entity
assert_equal "Raw file str is not a valid CSV format", flash[:alert]
end
end
test "should flash error message if no fileprovided for upload" do
patch upload_import_url(@empty_import), params: { import: { raw_file_str: nil } }
assert_response :unprocessable_entity
assert_equal "Please select a file to upload", flash[:alert]
end
test "should get configure" do
get configure_import_url(@loaded_import)
assert_response :success
end
test "should redirect back to load step with an alert message if not loaded" do
get configure_import_url(@empty_import)
assert_equal "Please load a CSV first", flash[:alert]
assert_redirected_to load_import_path(@empty_import)
end
test "should update mappings" do
patch configure_import_url(@loaded_import), params: {
import: {
column_mappings: {
date: "date",
name: "name",
category: "category",
amount: "amount"
test "creates import" do
assert_difference "Import.count", 1 do
post imports_url, params: {
import: {
type: "TransactionImport"
}
}
}
end
assert_redirected_to clean_import_path(@loaded_import)
assert_equal "Column mappings saved", flash[:notice]
assert_redirected_to import_upload_url(Import.all.ordered.first)
end
test "can update a cell" do
assert_equal @loaded_import.csv.table[0][1], "Starbucks drink"
test "publishes import" do
import = imports(:transaction)
patch clean_import_url(@loaded_import), params: {
import: {
csv_update: {
row_idx: 0,
col_idx: 1,
value: "new_merchant"
}
}
}
TransactionImport.any_instance.expects(:publish_later).once
assert_response :success
post publish_import_url(import)
@loaded_import.reload
assert_equal "new_merchant", @loaded_import.csv.table[0][1]
assert_equal "Your import has started in the background.", flash[:notice]
assert_redirected_to import_path(import)
end
test "should get clean" do
get clean_import_url(@loaded_import)
assert_response :success
end
test "destroys import" do
import = imports(:transaction)
test "should get confirm if all values are valid" do
get confirm_import_url(@loaded_import)
assert_response :success
end
assert_difference "Import.count", -1 do
delete import_url(import)
end
test "should redirect back to clean if data is invalid" do
@empty_import.update! raw_file_str: valid_csv_with_invalid_values
get confirm_import_url(@empty_import)
assert_equal "You have invalid data, please fix before continuing", flash[:alert]
assert_redirected_to clean_import_path(@empty_import)
end
test "should confirm import" do
patch confirm_import_url(@loaded_import)
assert_redirected_to imports_path
assert_equal "Import has started in the background", flash[:notice]
end
end

View file

@ -0,0 +1,5 @@
type,name,amount,currency
Checking,Main Checking Account,5000.00,USD
Savings,Emergency Fund,10000.00,USD
Credit Card,Rewards Credit Card,-1500.00,USD
Investment,Retirement Portfolio,75000.00,USD
1 type name amount currency
2 Checking Main Checking Account 5000.00 USD
3 Savings Emergency Fund 10000.00 USD
4 Credit Card Rewards Credit Card -1500.00 USD
5 Investment Retirement Portfolio 75000.00 USD

View file

@ -0,0 +1,3 @@
name,age
"John Doe,23
"Jane Doe",25
Can't render this file because it contains an unexpected character in line 3 and column 1.

11
test/fixtures/files/imports/mint.csv vendored Normal file
View file

@ -0,0 +1,11 @@
Date,Description,Original Description,Amount,Transaction Type,Category,Account Name,Labels,Notes
05/01/2023,Grocery Store,SAFEWAY #1234,78.32,debit,Groceries,Checking Account,,
05/02/2023,Gas Station,SHELL OIL 57442893,-45.67,credit,Gas & Fuel,Credit Card,,
05/03/2023,Monthly Rent,AUTOPAY MORTGAGE,1500.00,debit,Mortgage & Rent,Checking Account,,
05/04/2023,Paycheck,ACME CORP PAYROLL,-2500.00,credit,Paycheck,Checking Account,Income,
05/05/2023,Restaurant,CHIPOTLE MEX GRILL,15.75,debit,Restaurants,Credit Card,,
05/06/2023,Online Shopping,AMAZON.COM,32.99,debit,Shopping,Credit Card,,
05/07/2023,Utility Bill,CITY POWER & LIGHT,89.50,debit,Utilities,Checking Account,,
05/08/2023,Coffee Shop,STARBUCKS,4.25,debit,Coffee Shops,Credit Card,,
05/09/2023,Gym Membership,FITNESS WORLD,49.99,debit,Gym,Checking Account,Health,Monthly membership
05/10/2023,Movie Theater,AMC THEATERS #123,24.50,debit,Movies & DVDs,Credit Card,Entertainment,
1 Date Description Original Description Amount Transaction Type Category Account Name Labels Notes
2 05/01/2023 Grocery Store SAFEWAY #1234 78.32 debit Groceries Checking Account
3 05/02/2023 Gas Station SHELL OIL 57442893 -45.67 credit Gas & Fuel Credit Card
4 05/03/2023 Monthly Rent AUTOPAY MORTGAGE 1500.00 debit Mortgage & Rent Checking Account
5 05/04/2023 Paycheck ACME CORP PAYROLL -2500.00 credit Paycheck Checking Account Income
6 05/05/2023 Restaurant CHIPOTLE MEX GRILL 15.75 debit Restaurants Credit Card
7 05/06/2023 Online Shopping AMAZON.COM 32.99 debit Shopping Credit Card
8 05/07/2023 Utility Bill CITY POWER & LIGHT 89.50 debit Utilities Checking Account
9 05/08/2023 Coffee Shop STARBUCKS 4.25 debit Coffee Shops Credit Card
10 05/09/2023 Gym Membership FITNESS WORLD 49.99 debit Gym Checking Account Health Monthly membership
11 05/10/2023 Movie Theater AMC THEATERS #123 24.50 debit Movies & DVDs Credit Card Entertainment

11
test/fixtures/files/imports/trades.csv vendored Normal file
View file

@ -0,0 +1,11 @@
date,ticker,qty,price,amount,account,name
2023-01-15,AAPL,10,150.25,1502.50,Brokerage Account,Buy Apple Inc
2023-02-03,GOOGL,5,2100.75,10503.75,Retirement Account,Buy Alphabet Inc
2023-03-10,MSFT,15,245.50,3682.50,Brokerage Account,Buy Microsoft Corp
2023-04-05,AMZN,8,3200.00,25600.00,Brokerage Account,Buy Amazon.com Inc
2023-05-20,TSLA,20,180.75,3615.00,Retirement Account,Buy Tesla Inc
2023-06-15,AAPL,-5,170.50,-852.50,Brokerage Account,Sell Apple Inc
2023-07-02,GOOGL,-2,2250.00,-4500.00,Retirement Account,Sell Alphabet Inc
2023-08-18,NVDA,12,450.25,5403.00,Brokerage Account,Buy NVIDIA Corp
2023-09-07,MSFT,-7,300.75,-2105.25,Brokerage Account,Sell Microsoft Corp
2023-10-01,META,25,310.50,7762.50,Retirement Account,Buy Meta Platforms Inc
1 date ticker qty price amount account name
2 2023-01-15 AAPL 10 150.25 1502.50 Brokerage Account Buy Apple Inc
3 2023-02-03 GOOGL 5 2100.75 10503.75 Retirement Account Buy Alphabet Inc
4 2023-03-10 MSFT 15 245.50 3682.50 Brokerage Account Buy Microsoft Corp
5 2023-04-05 AMZN 8 3200.00 25600.00 Brokerage Account Buy Amazon.com Inc
6 2023-05-20 TSLA 20 180.75 3615.00 Retirement Account Buy Tesla Inc
7 2023-06-15 AAPL -5 170.50 -852.50 Brokerage Account Sell Apple Inc
8 2023-07-02 GOOGL -2 2250.00 -4500.00 Retirement Account Sell Alphabet Inc
9 2023-08-18 NVDA 12 450.25 5403.00 Brokerage Account Buy NVIDIA Corp
10 2023-09-07 MSFT -7 300.75 -2105.25 Brokerage Account Sell Microsoft Corp
11 2023-10-01 META 25 310.50 7762.50 Retirement Account Buy Meta Platforms Inc

View file

@ -0,0 +1,10 @@
Date,Name,Amount,Category,Tags,Account,Notes
2023-05-01,Grocery Store,-89.75,Food,Groceries,Checking Account,Weekly grocery shopping
2023-05-03,Electric Company,-120.50,Utilities,Bills|Home,Credit Card,Monthly electricity bill
2023-05-05,Coffee Shop,-4.25,Food,Coffee|Work,Debit Card,Morning coffee
2023-05-07,Gas Station,-45.00,Transportation,Car|Fuel,Credit Card,Fill up car tank
2023-05-10,Online Retailer,-79.99,Shopping,Clothing,Credit Card,New shoes purchase
2023-05-12,Restaurant,-65.30,Food,Dining Out|Date Night,Checking Account,Dinner with partner
2023-05-15,Mobile Phone Provider,-55.00,Utilities,Bills|Communication,Debit Card,Monthly phone bill
2023-05-18,Movie Theater,-24.00,Entertainment,Movies,Credit Card,Weekend movie night
2023-05-20,Pharmacy,-32.50,Health,Medicine,Debit Card,Prescription refill
1 Date Name Amount Category Tags Account Notes
2 2023-05-01 Grocery Store -89.75 Food Groceries Checking Account Weekly grocery shopping
3 2023-05-03 Electric Company -120.50 Utilities Bills|Home Credit Card Monthly electricity bill
4 2023-05-05 Coffee Shop -4.25 Food Coffee|Work Debit Card Morning coffee
5 2023-05-07 Gas Station -45.00 Transportation Car|Fuel Credit Card Fill up car tank
6 2023-05-10 Online Retailer -79.99 Shopping Clothing Credit Card New shoes purchase
7 2023-05-12 Restaurant -65.30 Food Dining Out|Date Night Checking Account Dinner with partner
8 2023-05-15 Mobile Phone Provider -55.00 Utilities Bills|Communication Debit Card Monthly phone bill
9 2023-05-18 Movie Theater -24.00 Entertainment Movies Credit Card Weekend movie night
10 2023-05-20 Pharmacy -32.50 Health Medicine Debit Card Prescription refill

6
test/fixtures/import/mappings.yml vendored Normal file
View file

@ -0,0 +1,6 @@
one:
import: transaction
key: Food
type: Import::CategoryMapping
mappable: food_and_drink
mappable_type: Category

5
test/fixtures/import/rows.yml vendored Normal file
View file

@ -0,0 +1,5 @@
one:
import: transaction
date: 01/01/2024
amount: 100
currency: USD

View file

@ -1,20 +1,3 @@
empty_import:
account: depository
created_at: <%= 1.minute.ago %>
completed_import:
account: depository
column_mappings:
date: date
name: name
category: category
amount: amount
raw_file_str: |
date,name,category,tags,amount
2024-01-01,Starbucks drink,Food & Drink,Test Tag,-20
normalized_csv_str: |
date,name,category,tags,amount
2024-01-01,Starbucks drink,Food & Drink,Test Tag,-20
created_at: <%= 2.days.ago %>
transaction:
family: dylan_family
type: TransactionImport

View file

@ -0,0 +1,57 @@
require "test_helper"
module ImportInterfaceTest
extend ActiveSupport::Testing::Declarative
test "import interface" do
assert_respond_to @subject, :publish
assert_respond_to @subject, :publish_later
assert_respond_to @subject, :generate_rows_from_csv
assert_respond_to @subject, :csv_rows
assert_respond_to @subject, :csv_headers
assert_respond_to @subject, :csv_sample
assert_respond_to @subject, :uploaded?
assert_respond_to @subject, :configured?
assert_respond_to @subject, :cleaned?
assert_respond_to @subject, :publishable?
assert_respond_to @subject, :importing?
assert_respond_to @subject, :complete?
assert_respond_to @subject, :failed?
end
test "publishes later" do
import = imports(:transaction)
import.stubs(:publishable?).returns(true)
assert_enqueued_with job: ImportJob, args: [ import ] do
import.publish_later
end
assert_equal "importing", import.reload.status
end
test "raises if not publishable" do
import = imports(:transaction)
import.stubs(:publishable?).returns(false)
assert_raises(RuntimeError, "Import is not publishable") do
import.publish_later
end
end
test "handles publish errors" do
import = imports(:transaction)
import.stubs(:publishable?).returns(true)
import.stubs(:import!).raises(StandardError, "Failed to publish")
assert_nil import.error
import.publish
assert_equal "Failed to publish", import.error
assert_equal "failed", import.status
end
end

View file

@ -1,19 +1,10 @@
require "test_helper"
class ImportJobTest < ActiveJob::TestCase
include ImportTestHelper
test "import is published" do
import = imports(:empty_import)
import.update! raw_file_str: valid_csv_str
import = imports(:transaction)
import.expects(:publish).once
assert import.pending?
perform_enqueued_jobs do
ImportJob.perform_later(import)
end
assert import.reload.complete?
assert import.account.balances.present?
ImportJob.perform_now(import)
end
end

View file

@ -99,20 +99,4 @@ class Account::EntryTest < ActiveSupport::TestCase
assert create_transaction(amount: -10).inflow?
assert create_transaction(amount: 10).outflow?
end
test "cannot sell more shares of stock than owned" do
account = families(:empty).accounts.create! name: "Test", balance: 0, currency: "USD", accountable: Investment.new
security = securities(:aapl)
error = assert_raises ActiveRecord::RecordInvalid do
account.entries.create! \
date: Date.current,
amount: 100,
currency: "USD",
name: "Sell 10 shares of AMZN",
entryable: Account::Trade.new(qty: -10, price: 200, security: security)
end
assert_match /cannot sell 10.0 shares of AAPL because you only own 0.0 shares/, error.message
end
end

View file

@ -1,129 +0,0 @@
require "test_helper"
class Import::CsvTest < ActiveSupport::TestCase
include ImportTestHelper
setup do
@csv = Import::Csv.new(valid_csv_str)
end
test "cannot define validator for non-existent header" do
assert_raises do
@csv.define_validator "invalid", method(:validate_iso_date)
end
end
test "csv with no validators is valid" do
assert @csv.cell_valid?(0, 0)
assert @csv.valid?
end
test "valid csv values" do
@csv.define_validator "date", method(:validate_iso_date)
assert_equal "2024-01-01", @csv.table[0][0]
assert @csv.cell_valid?(0, 0)
assert @csv.valid?
end
test "invalid csv values" do
invalid_csv = Import::Csv.new valid_csv_with_invalid_values
invalid_csv.define_validator "date", method(:validate_iso_date)
assert_equal "invalid_date", invalid_csv.table[0][0]
assert_not invalid_csv.cell_valid?(0, 0)
assert_not invalid_csv.valid?
end
test "CSV with semicolon column separator" do
csv = Import::Csv.new(valid_csv_str_with_semicolon_separator, col_sep: ";")
assert_equal %w[date name category tags amount], csv.table.headers
assert_equal 4, csv.table.size
assert_equal "Paycheck", csv.table[3][1]
end
test "csv with additional columns and empty values" do
csv = Import::Csv.new valid_csv_with_missing_data
assert csv.valid?
end
test "updating a cell returns a copy of the original csv" do
original_date = "2024-01-01"
new_date = "2024-01-01"
assert_equal original_date, @csv.table[0][0]
updated = @csv.update_cell(0, 0, new_date)
assert_equal original_date, @csv.table[0][0]
assert_equal new_date, updated[0][0]
end
test "can create CSV with expected columns and field mappings with validators" do
date_field = Import::Field.new \
key: "date",
label: "Date",
validator: method(:validate_iso_date)
name_field = Import::Field.new \
key: "name",
label: "Name"
fields = [ date_field, name_field ]
raw_file_str = <<-ROWS
date,Custom Field Header,extra_field
invalid_date_value,Starbucks drink,Food
2024-01-02,Amazon stuff,Shopping
ROWS
mappings = {
"name" => "Custom Field Header"
}
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings)
assert_equal %w[date name], csv.table.headers
assert_equal 2, csv.table.size
assert_equal "Amazon stuff", csv.table[1][1]
end
test "can create CSV with expected columns, field mappings with validators and semicolon column separator" do
date_field = Import::Field.new \
key: "date",
label: "Date",
validator: method(:validate_iso_date)
name_field = Import::Field.new \
key: "name",
label: "Name"
fields = [ date_field, name_field ]
raw_file_str = <<-ROWS
date;Custom Field Header;extra_field
invalid_date_value;Starbucks drink;Food
2024-01-02;Amazon stuff;Shopping
ROWS
mappings = {
"name" => "Custom Field Header"
}
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings, ";")
assert_equal %w[date name], csv.table.headers
assert_equal 2, csv.table.size
assert_equal "Amazon stuff", csv.table[1][1]
end
private
def validate_iso_date(value)
Date.iso8601(value)
true
rescue
false
end
end

View file

@ -1,28 +0,0 @@
require "test_helper"
class Import::FieldTest < ActiveSupport::TestCase
test "key is always a string" do
field1 = Import::Field.new label: "Test", key: "test"
field2 = Import::Field.new label: "Test2", key: :test2
assert_equal "test", field1.key
assert_equal "test2", field2.key
end
test "can set and override a validator for a field" do
field = Import::Field.new \
label: "Test",
key: "Test",
validator: ->(val) { val == 42 }
assert field.validate(42)
assert_not field.validate(41)
field.define_validator do |value|
value == 100
end
assert field.validate(100)
assert_not field.validate(42)
end
end

View file

@ -0,0 +1,7 @@
require "test_helper"
class Import::MappingTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end

View file

@ -0,0 +1,7 @@
require "test_helper"
class Import::RowTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end

View file

@ -1,115 +0,0 @@
require "test_helper"
class ImportTest < ActiveSupport::TestCase
include ImportTestHelper, ActiveJob::TestHelper
setup do
@empty_import = imports(:empty_import)
@loaded_import = @empty_import.dup
@loaded_import.update! raw_file_str: valid_csv_str
end
test "validates the correct col_sep" do
assert_equal ",", @empty_import.col_sep
assert @empty_import.valid?
@empty_import.col_sep = "invalid"
assert @empty_import.invalid?
@empty_import.col_sep = ","
assert @empty_import.valid?
@empty_import.col_sep = ";"
assert @empty_import.valid?
end
test "raw csv input must conform to csv spec" do
@empty_import.raw_file_str = malformed_csv_str
assert_not @empty_import.valid?
@empty_import.raw_file_str = valid_csv_str
assert @empty_import.valid?
end
test "can update csv value without affecting raw input" do
assert_equal "Starbucks drink", @loaded_import.csv.table[0][1]
prior_raw_file_str_value = @loaded_import.raw_file_str
prior_normalized_csv_str_value = @loaded_import.normalized_csv_str
@loaded_import.update_csv! \
row_idx: 0,
col_idx: 1,
value: "new_category"
assert_equal "new_category", @loaded_import.csv.table[0][1]
assert_equal prior_raw_file_str_value, @loaded_import.raw_file_str
assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str
end
test "publishes later" do
assert_enqueued_with(job: ImportJob) do
@loaded_import.publish_later
end
end
test "publishes a valid import" do
# Import has 3 unique categories: "Food & Drink", "Income", and "Shopping" (x2)
# Fixtures already define "Food & Drink" and "Income", so these should not be created
# "Shopping" is a new category, but should only be created 1x during import
assert_difference \
-> { Account::Transaction.count } => 4,
-> { Account::Entry.count } => 4,
-> { Category.count } => 1,
-> { Tagging.count } => 4,
-> { Tag.count } => 2 do
@loaded_import.publish
end
@loaded_import.reload
assert @loaded_import.complete?
end
test "publishes a valid import with missing data" do
@empty_import.update! raw_file_str: valid_csv_with_missing_data
assert_difference -> { Category.count } => 1,
-> { Account::Transaction.count } => 2,
-> { Account::Entry.count } => 2 do
@empty_import.publish
end
assert_not_nil Account::Entry.find_sole_by(name: Import::FALLBACK_TRANSACTION_NAME)
@empty_import.reload
assert @empty_import.complete?
end
test "failed publish results in error status" do
@empty_import.update! raw_file_str: valid_csv_with_invalid_values
assert_difference "Account::Transaction.count", 0 do
@empty_import.publish
end
@empty_import.reload
assert @empty_import.failed?
end
test "can create transactions from csv with custom column separator" do
loaded_import = @empty_import.dup
loaded_import.update! raw_file_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
transactions = loaded_import.dry_run
assert_equal 4, transactions.count
data = transactions.first.as_json(only: [ :name, :amount, :date ])
assert_equal data, { "amount" => "8.55", "date" => "2024-01-01", "name" => "Starbucks drink" }
assert_equal valid_csv_str, loaded_import.normalized_csv_str
end
end

View file

@ -0,0 +1,66 @@
require "test_helper"
class TransactionImportTest < ActiveSupport::TestCase
include ActiveJob::TestHelper, ImportInterfaceTest
setup do
@subject = @import = imports(:transaction)
end
test "uploaded? if raw_file_str is present" do
@import.expects(:raw_file_str).returns("test").once
assert @import.uploaded?
end
test "configured? if uploaded and rows are generated" do
@import.expects(:uploaded?).returns(true).once
assert @import.configured?
end
test "cleaned? if rows are generated and valid" do
@import.expects(:configured?).returns(true).once
assert @import.cleaned?
end
test "publishable? if cleaned and mappings are valid" do
@import.expects(:cleaned?).returns(true).once
assert @import.publishable?
end
test "imports transactions, categories, tags, and accounts" do
import = <<-CSV
date,name,amount,category,tags,account,notes
01/01/2024,Txn1,100,TestCategory1,TestTag1,TestAccount1,notes1
01/02/2024,Txn2,200,TestCategory2,TestTag1|TestTag2,TestAccount2,notes2
01/03/2024,Txn3,300,,,,notes3
CSV
@import.update!(raw_file_str: import)
@import.generate_rows_from_csv
@import.mappings.create! key: "TestCategory1", create_when_empty: true, type: "Import::CategoryMapping"
@import.mappings.create! key: "TestCategory2", mappable: categories(:food_and_drink), type: "Import::CategoryMapping"
@import.mappings.create! key: "", create_when_empty: false, mappable: nil, type: "Import::CategoryMapping" # Leaves uncategorized
@import.mappings.create! key: "TestTag1", create_when_empty: true, type: "Import::TagMapping"
@import.mappings.create! key: "TestTag2", mappable: tags(:one), type: "Import::TagMapping"
@import.mappings.create! key: "", create_when_empty: false, mappable: nil, type: "Import::TagMapping" # Leaves untagged
@import.mappings.create! key: "TestAccount1", create_when_empty: true, type: "Import::AccountMapping"
@import.mappings.create! key: "TestAccount2", mappable: accounts(:depository), type: "Import::AccountMapping"
@import.mappings.create! key: "", mappable: accounts(:depository), type: "Import::AccountMapping"
@import.reload
assert_difference -> { Account::Entry.count } => 3,
-> { Account::Transaction.count } => 3,
-> { Tag.count } => 1,
-> { Category.count } => 1,
-> { Account.count } => 1 do
@import.publish
end
assert_equal "complete", @import.status
end
end

View file

@ -1,44 +0,0 @@
module ImportTestHelper
def valid_csv_str
<<~ROWS
date,name,category,tags,amount
2024-01-01,Starbucks drink,Food & Drink,Tag1|Tag2,-8.55
2024-01-01,Etsy,Shopping,Tag1,-80.98
2024-01-02,Amazon stuff,Shopping,Tag2,-200
2024-01-03,Paycheck,Income,,1000
ROWS
end
def valid_csv_str_with_semicolon_separator
<<~ROWS
date;name;category;tags;amount
2024-01-01;Starbucks drink;Food & Drink;Tag1|Tag2;-8.55
2024-01-01;Etsy;Shopping;Tag1;-80.98
2024-01-02;Amazon stuff;Shopping;Tag2;-200
2024-01-03;Paycheck;Income;;1000
ROWS
end
def valid_csv_with_invalid_values
<<~ROWS
date,name,category,tags,amount
invalid_date,Starbucks drink,Food,,invalid_amount
ROWS
end
def valid_csv_with_missing_data
<<~ROWS
date,name,category,"optional id",amount
2024-01-01,Drink,Food,1234,-200
2024-01-02,,,,-100
ROWS
end
def malformed_csv_str
<<~ROWS
name,age
"John Doe,23
"Jane Doe",25
ROWS
end
end

View file

@ -1,170 +1,157 @@
require "application_system_test_case"
class ImportsTest < ApplicationSystemTestCase
include ImportTestHelper
include ActiveJob::TestHelper
setup do
sign_in @user = users(:family_admin)
@imports = @user.family.imports.ordered.to_a
end
test "can trigger new import from settings" do
trigger_import_from_settings
verify_import_modal
test "transaction import" do
visit new_import_path
click_on "Import transactions"
fill_in "import[raw_file_str]", with: file_fixture("imports/transactions.csv").read
find('input[type="submit"][value="Upload CSV"]').click
select "Date", from: "Date"
select "YYYY-MM-DD", from: "Date format"
select "Amount", from: "Amount"
select "Account", from: "Account (optional)"
select "Name", from: "Name (optional)"
select "Category", from: "Category (optional)"
select "Tags", from: "Tags (optional)"
select "Notes", from: "Notes (optional)"
click_on "Apply configuration"
click_on "Next step"
assert_selector "h1", text: "Assign your categories"
click_on "Next"
assert_selector "h1", text: "Assign your tags"
click_on "Next"
assert_selector "h1", text: "Assign your accounts"
click_on "Next"
click_on "Publish import"
assert_text "Import in progress"
perform_enqueued_jobs
click_on "Check status"
assert_text "Import successful"
click_on "Back to dashboard"
end
test "can resume existing import from settings" do
visit imports_url
test "trade import" do
visit new_import_path
within "#" + dom_id(@imports.first) do
click_button
click_link "Edit"
end
click_on "Import investments"
assert_current_path edit_import_path(@imports.first)
fill_in "import[raw_file_str]", with: file_fixture("imports/trades.csv").read
find('input[type="submit"][value="Upload CSV"]').click
select "YYYY-MM-DD", from: "Date format"
click_on "Apply configuration"
click_on "Next step"
assert_selector "h1", text: "Assign your accounts"
click_on "Next"
click_on "Publish import"
assert_text "Import in progress"
perform_enqueued_jobs
click_on "Check status"
assert_text "Import successful"
click_on "Back to dashboard"
end
test "can resume latest import" do
trigger_import_from_transactions
verify_import_modal
test "account import" do
visit new_import_path
click_link "Resume latest import"
click_on "Import accounts"
assert_current_path edit_import_path(@imports.first)
end
fill_in "import[raw_file_str]", with: file_fixture("imports/accounts.csv").read
test "can perform basic CSV import" do
trigger_import_from_settings
verify_import_modal
find('input[type="submit"][value="Upload CSV"]').click
within "#modal" do
click_link "New import from CSV"
end
click_on "Apply configuration"
# 1) Create import step
assert_selector "h1", text: "New import"
click_on "Next step"
within "form" do
select "Checking Account", from: "import_account_id"
end
assert_selector "h1", text: "Assign your account types"
click_button "Next"
click_button "Copy & Paste"
# 2) Load Step
assert_selector "h1", text: "Load import"
within "form" do
fill_in "import_raw_file_str", with: <<-ROWS
date,Custom Name Column,category,amount
invalid_date,Starbucks drink,Food,-20.50
2024-01-01,Amazon purchase,Shopping,-89.50
ROWS
end
click_button "Next"
# 3) Configure step
assert_selector "h1", text: "Configure import"
within "form" do
select "Custom Name Column", from: "import_column_mappings_name"
end
click_button "Next"
# 4) Clean step
assert_selector "h1", text: "Clean import"
# We have an invalid value, so user cannot click next yet
assert_no_text "Next"
# Replace invalid date with valid date
fill_in "cell-0-0", with: "2024-01-02"
# Trigger blur event so value saves
find("body").click
click_link "Next"
# 5) Confirm step
assert_selector "h1", text: "Confirm import"
assert_selector "#new_account_entry", count: 2
click_button "Import 2 transactions"
assert_selector "h1", text: "Imports"
end
test "can perform import by CSV upload" do
trigger_import_from_settings
verify_import_modal
within "#modal" do
click_link "New import from CSV"
end
# 1) Create import step
assert_selector "h1", text: "New import"
within "form" do
select "Checking Account", from: "import_account_id"
end
click_button "Next"
click_button "Upload CSV"
find(".raw-file-drop-box").drop File.join(file_fixture_path, "transactions.csv")
assert_selector "div.csv-preview", text: "transactions.csv"
click_button "Next"
# 3) Configure step
assert_selector "h1", text: "Configure import"
within "form" do
select "Custom Name Column", from: "import_column_mappings_name"
end
click_button "Next"
# 4) Clean step
assert_selector "h1", text: "Clean import"
# We have an invalid value, so user cannot click next yet
assert_no_text "Next"
# Replace invalid date with valid date
fill_in "cell-0-0", with: "2024-01-02"
# Trigger blur event so value saves
find("body").click
click_link "Next"
# 5) Confirm step
assert_selector "h1", text: "Confirm import"
assert_selector "#new_account_entry", count: 2
click_button "Import 2 transactions"
assert_selector "h1", text: "Imports"
end
private
def trigger_import_from_settings
visit imports_url
click_link "New import"
end
def trigger_import_from_transactions
visit transactions_url
click_link "Import"
end
def verify_import_modal
within "#modal" do
assert_text "Import transactions"
all("form").each do |form|
within(form) do
select = form.find("select")
select "Depository", from: select["id"]
sleep 1
end
end
click_on "Next"
click_on "Publish import"
assert_text "Import in progress"
perform_enqueued_jobs
click_on "Check status"
assert_text "Import successful"
click_on "Back to dashboard"
end
test "mint import" do
visit new_import_path
click_on "Import from Mint"
fill_in "import[raw_file_str]", with: file_fixture("imports/mint.csv").read
find('input[type="submit"][value="Upload CSV"]').click
click_on "Apply configuration"
click_on "Next step"
assert_selector "h1", text: "Assign your categories"
click_on "Next"
assert_selector "h1", text: "Assign your tags"
click_on "Next"
assert_selector "h1", text: "Assign your accounts"
click_on "Next"
click_on "Publish import"
assert_text "Import in progress"
perform_enqueued_jobs
click_on "Check status"
assert_text "Import successful"
click_on "Back to dashboard"
end
end