mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-07-24 23:59:40 +02:00
CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)
* Remove stale 1.0 import logic and model * Fresh start * Checkpoint before removing nav * First working prototype * Add trade, account, and mint import flows * Basic working version with tests * System tests for each import type * Clean up mappings flow * Clean up PR, refactor stale code, tests * Add back row validations * Row validations * Fix import job test * Fix import navigation * Fix mint import configuration form * Currency preset for new accounts
This commit is contained in:
Zach Gollwitzer
GitHub
parent
23786b444a
commit
398b246965
103 changed files with 2420 additions and 1689 deletions
|
|
@ -99,20 +99,4 @@ class Account::EntryTest < ActiveSupport::TestCase
|
|||
assert create_transaction(amount: -10).inflow?
|
||||
assert create_transaction(amount: 10).outflow?
|
||||
end
|
||||
|
||||
test "cannot sell more shares of stock than owned" do
|
||||
account = families(:empty).accounts.create! name: "Test", balance: 0, currency: "USD", accountable: Investment.new
|
||||
security = securities(:aapl)
|
||||
|
||||
error = assert_raises ActiveRecord::RecordInvalid do
|
||||
account.entries.create! \
|
||||
date: Date.current,
|
||||
amount: 100,
|
||||
currency: "USD",
|
||||
name: "Sell 10 shares of AMZN",
|
||||
entryable: Account::Trade.new(qty: -10, price: 200, security: security)
|
||||
end
|
||||
|
||||
assert_match /cannot sell 10.0 shares of AAPL because you only own 0.0 shares/, error.message
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,129 +0,0 @@
|
|||
require "test_helper"
|
||||
|
||||
class Import::CsvTest < ActiveSupport::TestCase
|
||||
include ImportTestHelper
|
||||
|
||||
setup do
|
||||
@csv = Import::Csv.new(valid_csv_str)
|
||||
end
|
||||
|
||||
test "cannot define validator for non-existent header" do
|
||||
assert_raises do
|
||||
@csv.define_validator "invalid", method(:validate_iso_date)
|
||||
end
|
||||
end
|
||||
|
||||
test "csv with no validators is valid" do
|
||||
assert @csv.cell_valid?(0, 0)
|
||||
assert @csv.valid?
|
||||
end
|
||||
|
||||
test "valid csv values" do
|
||||
@csv.define_validator "date", method(:validate_iso_date)
|
||||
|
||||
assert_equal "2024-01-01", @csv.table[0][0]
|
||||
assert @csv.cell_valid?(0, 0)
|
||||
assert @csv.valid?
|
||||
end
|
||||
|
||||
test "invalid csv values" do
|
||||
invalid_csv = Import::Csv.new valid_csv_with_invalid_values
|
||||
|
||||
invalid_csv.define_validator "date", method(:validate_iso_date)
|
||||
|
||||
assert_equal "invalid_date", invalid_csv.table[0][0]
|
||||
assert_not invalid_csv.cell_valid?(0, 0)
|
||||
assert_not invalid_csv.valid?
|
||||
end
|
||||
|
||||
test "CSV with semicolon column separator" do
|
||||
csv = Import::Csv.new(valid_csv_str_with_semicolon_separator, col_sep: ";")
|
||||
|
||||
assert_equal %w[date name category tags amount], csv.table.headers
|
||||
assert_equal 4, csv.table.size
|
||||
assert_equal "Paycheck", csv.table[3][1]
|
||||
end
|
||||
|
||||
test "csv with additional columns and empty values" do
|
||||
csv = Import::Csv.new valid_csv_with_missing_data
|
||||
assert csv.valid?
|
||||
end
|
||||
|
||||
test "updating a cell returns a copy of the original csv" do
|
||||
original_date = "2024-01-01"
|
||||
new_date = "2024-01-01"
|
||||
|
||||
assert_equal original_date, @csv.table[0][0]
|
||||
updated = @csv.update_cell(0, 0, new_date)
|
||||
|
||||
assert_equal original_date, @csv.table[0][0]
|
||||
assert_equal new_date, updated[0][0]
|
||||
end
|
||||
|
||||
test "can create CSV with expected columns and field mappings with validators" do
|
||||
date_field = Import::Field.new \
|
||||
key: "date",
|
||||
label: "Date",
|
||||
validator: method(:validate_iso_date)
|
||||
|
||||
name_field = Import::Field.new \
|
||||
key: "name",
|
||||
label: "Name"
|
||||
|
||||
fields = [ date_field, name_field ]
|
||||
|
||||
raw_file_str = <<-ROWS
|
||||
date,Custom Field Header,extra_field
|
||||
invalid_date_value,Starbucks drink,Food
|
||||
2024-01-02,Amazon stuff,Shopping
|
||||
ROWS
|
||||
|
||||
mappings = {
|
||||
"name" => "Custom Field Header"
|
||||
}
|
||||
|
||||
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings)
|
||||
|
||||
assert_equal %w[date name], csv.table.headers
|
||||
assert_equal 2, csv.table.size
|
||||
assert_equal "Amazon stuff", csv.table[1][1]
|
||||
end
|
||||
|
||||
test "can create CSV with expected columns, field mappings with validators and semicolon column separator" do
|
||||
date_field = Import::Field.new \
|
||||
key: "date",
|
||||
label: "Date",
|
||||
validator: method(:validate_iso_date)
|
||||
|
||||
name_field = Import::Field.new \
|
||||
key: "name",
|
||||
label: "Name"
|
||||
|
||||
fields = [ date_field, name_field ]
|
||||
|
||||
raw_file_str = <<-ROWS
|
||||
date;Custom Field Header;extra_field
|
||||
invalid_date_value;Starbucks drink;Food
|
||||
2024-01-02;Amazon stuff;Shopping
|
||||
ROWS
|
||||
|
||||
mappings = {
|
||||
"name" => "Custom Field Header"
|
||||
}
|
||||
|
||||
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings, ";")
|
||||
|
||||
assert_equal %w[date name], csv.table.headers
|
||||
assert_equal 2, csv.table.size
|
||||
assert_equal "Amazon stuff", csv.table[1][1]
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def validate_iso_date(value)
|
||||
Date.iso8601(value)
|
||||
true
|
||||
rescue
|
||||
false
|
||||
end
|
||||
end
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
require "test_helper"
|
||||
|
||||
class Import::FieldTest < ActiveSupport::TestCase
|
||||
test "key is always a string" do
|
||||
field1 = Import::Field.new label: "Test", key: "test"
|
||||
field2 = Import::Field.new label: "Test2", key: :test2
|
||||
|
||||
assert_equal "test", field1.key
|
||||
assert_equal "test2", field2.key
|
||||
end
|
||||
|
||||
test "can set and override a validator for a field" do
|
||||
field = Import::Field.new \
|
||||
label: "Test",
|
||||
key: "Test",
|
||||
validator: ->(val) { val == 42 }
|
||||
|
||||
assert field.validate(42)
|
||||
assert_not field.validate(41)
|
||||
|
||||
field.define_validator do |value|
|
||||
value == 100
|
||||
end
|
||||
|
||||
assert field.validate(100)
|
||||
assert_not field.validate(42)
|
||||
end
|
||||
end
|
||||
7
test/models/import/mapping_test.rb
Normal file
7
test/models/import/mapping_test.rb
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
require "test_helper"
|
||||
|
||||
class Import::MappingTest < ActiveSupport::TestCase
|
||||
# test "the truth" do
|
||||
# assert true
|
||||
# end
|
||||
end
|
||||
7
test/models/import/row_test.rb
Normal file
7
test/models/import/row_test.rb
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
require "test_helper"
|
||||
|
||||
class Import::RowTest < ActiveSupport::TestCase
|
||||
# test "the truth" do
|
||||
# assert true
|
||||
# end
|
||||
end
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
require "test_helper"
|
||||
|
||||
class ImportTest < ActiveSupport::TestCase
|
||||
include ImportTestHelper, ActiveJob::TestHelper
|
||||
|
||||
setup do
|
||||
@empty_import = imports(:empty_import)
|
||||
|
||||
@loaded_import = @empty_import.dup
|
||||
@loaded_import.update! raw_file_str: valid_csv_str
|
||||
end
|
||||
|
||||
test "validates the correct col_sep" do
|
||||
assert_equal ",", @empty_import.col_sep
|
||||
|
||||
assert @empty_import.valid?
|
||||
|
||||
@empty_import.col_sep = "invalid"
|
||||
assert @empty_import.invalid?
|
||||
|
||||
@empty_import.col_sep = ","
|
||||
assert @empty_import.valid?
|
||||
|
||||
@empty_import.col_sep = ";"
|
||||
assert @empty_import.valid?
|
||||
end
|
||||
|
||||
test "raw csv input must conform to csv spec" do
|
||||
@empty_import.raw_file_str = malformed_csv_str
|
||||
assert_not @empty_import.valid?
|
||||
|
||||
@empty_import.raw_file_str = valid_csv_str
|
||||
assert @empty_import.valid?
|
||||
end
|
||||
|
||||
test "can update csv value without affecting raw input" do
|
||||
assert_equal "Starbucks drink", @loaded_import.csv.table[0][1]
|
||||
|
||||
prior_raw_file_str_value = @loaded_import.raw_file_str
|
||||
prior_normalized_csv_str_value = @loaded_import.normalized_csv_str
|
||||
|
||||
@loaded_import.update_csv! \
|
||||
row_idx: 0,
|
||||
col_idx: 1,
|
||||
value: "new_category"
|
||||
|
||||
assert_equal "new_category", @loaded_import.csv.table[0][1]
|
||||
assert_equal prior_raw_file_str_value, @loaded_import.raw_file_str
|
||||
assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str
|
||||
end
|
||||
|
||||
test "publishes later" do
|
||||
assert_enqueued_with(job: ImportJob) do
|
||||
@loaded_import.publish_later
|
||||
end
|
||||
end
|
||||
|
||||
test "publishes a valid import" do
|
||||
# Import has 3 unique categories: "Food & Drink", "Income", and "Shopping" (x2)
|
||||
# Fixtures already define "Food & Drink" and "Income", so these should not be created
|
||||
# "Shopping" is a new category, but should only be created 1x during import
|
||||
assert_difference \
|
||||
-> { Account::Transaction.count } => 4,
|
||||
-> { Account::Entry.count } => 4,
|
||||
-> { Category.count } => 1,
|
||||
-> { Tagging.count } => 4,
|
||||
-> { Tag.count } => 2 do
|
||||
@loaded_import.publish
|
||||
end
|
||||
|
||||
@loaded_import.reload
|
||||
|
||||
assert @loaded_import.complete?
|
||||
end
|
||||
|
||||
test "publishes a valid import with missing data" do
|
||||
@empty_import.update! raw_file_str: valid_csv_with_missing_data
|
||||
assert_difference -> { Category.count } => 1,
|
||||
-> { Account::Transaction.count } => 2,
|
||||
-> { Account::Entry.count } => 2 do
|
||||
@empty_import.publish
|
||||
end
|
||||
|
||||
assert_not_nil Account::Entry.find_sole_by(name: Import::FALLBACK_TRANSACTION_NAME)
|
||||
|
||||
@empty_import.reload
|
||||
|
||||
assert @empty_import.complete?
|
||||
end
|
||||
|
||||
test "failed publish results in error status" do
|
||||
@empty_import.update! raw_file_str: valid_csv_with_invalid_values
|
||||
|
||||
assert_difference "Account::Transaction.count", 0 do
|
||||
@empty_import.publish
|
||||
end
|
||||
|
||||
@empty_import.reload
|
||||
assert @empty_import.failed?
|
||||
end
|
||||
|
||||
test "can create transactions from csv with custom column separator" do
|
||||
loaded_import = @empty_import.dup
|
||||
|
||||
loaded_import.update! raw_file_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
|
||||
transactions = loaded_import.dry_run
|
||||
|
||||
assert_equal 4, transactions.count
|
||||
|
||||
data = transactions.first.as_json(only: [ :name, :amount, :date ])
|
||||
assert_equal data, { "amount" => "8.55", "date" => "2024-01-01", "name" => "Starbucks drink" }
|
||||
|
||||
assert_equal valid_csv_str, loaded_import.normalized_csv_str
|
||||
end
|
||||
end
|
||||
66
test/models/transaction_import_test.rb
Normal file
66
test/models/transaction_import_test.rb
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
require "test_helper"
|
||||
|
||||
class TransactionImportTest < ActiveSupport::TestCase
|
||||
include ActiveJob::TestHelper, ImportInterfaceTest
|
||||
|
||||
setup do
|
||||
@subject = @import = imports(:transaction)
|
||||
end
|
||||
|
||||
test "uploaded? if raw_file_str is present" do
|
||||
@import.expects(:raw_file_str).returns("test").once
|
||||
assert @import.uploaded?
|
||||
end
|
||||
|
||||
test "configured? if uploaded and rows are generated" do
|
||||
@import.expects(:uploaded?).returns(true).once
|
||||
assert @import.configured?
|
||||
end
|
||||
|
||||
test "cleaned? if rows are generated and valid" do
|
||||
@import.expects(:configured?).returns(true).once
|
||||
assert @import.cleaned?
|
||||
end
|
||||
|
||||
test "publishable? if cleaned and mappings are valid" do
|
||||
@import.expects(:cleaned?).returns(true).once
|
||||
assert @import.publishable?
|
||||
end
|
||||
|
||||
test "imports transactions, categories, tags, and accounts" do
|
||||
import = <<-CSV
|
||||
date,name,amount,category,tags,account,notes
|
||||
01/01/2024,Txn1,100,TestCategory1,TestTag1,TestAccount1,notes1
|
||||
01/02/2024,Txn2,200,TestCategory2,TestTag1|TestTag2,TestAccount2,notes2
|
||||
01/03/2024,Txn3,300,,,,notes3
|
||||
CSV
|
||||
|
||||
@import.update!(raw_file_str: import)
|
||||
|
||||
@import.generate_rows_from_csv
|
||||
|
||||
@import.mappings.create! key: "TestCategory1", create_when_empty: true, type: "Import::CategoryMapping"
|
||||
@import.mappings.create! key: "TestCategory2", mappable: categories(:food_and_drink), type: "Import::CategoryMapping"
|
||||
@import.mappings.create! key: "", create_when_empty: false, mappable: nil, type: "Import::CategoryMapping" # Leaves uncategorized
|
||||
|
||||
@import.mappings.create! key: "TestTag1", create_when_empty: true, type: "Import::TagMapping"
|
||||
@import.mappings.create! key: "TestTag2", mappable: tags(:one), type: "Import::TagMapping"
|
||||
@import.mappings.create! key: "", create_when_empty: false, mappable: nil, type: "Import::TagMapping" # Leaves untagged
|
||||
|
||||
@import.mappings.create! key: "TestAccount1", create_when_empty: true, type: "Import::AccountMapping"
|
||||
@import.mappings.create! key: "TestAccount2", mappable: accounts(:depository), type: "Import::AccountMapping"
|
||||
@import.mappings.create! key: "", mappable: accounts(:depository), type: "Import::AccountMapping"
|
||||
|
||||
@import.reload
|
||||
|
||||
assert_difference -> { Account::Entry.count } => 3,
|
||||
-> { Account::Transaction.count } => 3,
|
||||
-> { Tag.count } => 1,
|
||||
-> { Category.count } => 1,
|
||||
-> { Account.count } => 1 do
|
||||
@import.publish
|
||||
end
|
||||
|
||||
assert_equal "complete", @import.status
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue