mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-08-02 20:15:22 +02:00
CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)
* Remove stale 1.0 import logic and model * Fresh start * Checkpoint before removing nav * First working prototype * Add trade, account, and mint import flows * Basic working version with tests * System tests for each import type * Clean up mappings flow * Clean up PR, refactor stale code, tests * Add back row validations * Row validations * Fix import job test * Fix import navigation * Fix mint import configuration form * Currency preset for new accounts
This commit is contained in:
parent
23786b444a
commit
398b246965
103 changed files with 2420 additions and 1689 deletions
|
@ -5,14 +5,15 @@ class Account < ApplicationRecord
|
|||
|
||||
belongs_to :family
|
||||
belongs_to :institution, optional: true
|
||||
belongs_to :import, optional: true
|
||||
|
||||
has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping"
|
||||
has_many :entries, dependent: :destroy, class_name: "Account::Entry"
|
||||
has_many :transactions, through: :entries, source: :entryable, source_type: "Account::Transaction"
|
||||
has_many :valuations, through: :entries, source: :entryable, source_type: "Account::Valuation"
|
||||
has_many :trades, through: :entries, source: :entryable, source_type: "Account::Trade"
|
||||
has_many :holdings, dependent: :destroy
|
||||
has_many :balances, dependent: :destroy
|
||||
has_many :imports, dependent: :destroy
|
||||
has_many :syncs, dependent: :destroy
|
||||
has_many :issues, as: :issuable, dependent: :destroy
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ class Account::Entry < ApplicationRecord
|
|||
|
||||
belongs_to :account
|
||||
belongs_to :transfer, optional: true
|
||||
belongs_to :import, optional: true
|
||||
|
||||
delegated_type :entryable, types: Account::Entryable::TYPES, dependent: :destroy
|
||||
accepts_nested_attributes_for :entryable
|
||||
|
@ -12,7 +13,6 @@ class Account::Entry < ApplicationRecord
|
|||
validates :date, :amount, :currency, presence: true
|
||||
validates :date, uniqueness: { scope: [ :account_id, :entryable_type ] }, if: -> { account_valuation? }
|
||||
validates :date, comparison: { greater_than: -> { min_supported_date } }
|
||||
validate :trade_valid?, if: -> { account_trade? }
|
||||
|
||||
scope :chronological, -> { order(:date, :created_at) }
|
||||
scope :reverse_chronological, -> { order(date: :desc, created_at: :desc) }
|
||||
|
@ -219,20 +219,4 @@ class Account::Entry < ApplicationRecord
|
|||
previous: previous_entry&.amount_money,
|
||||
favorable_direction: account.favorable_direction
|
||||
end
|
||||
|
||||
def trade_valid?
|
||||
if account_trade.sell?
|
||||
current_qty = account.holding_qty(account_trade.security)
|
||||
|
||||
if current_qty < account_trade.qty.abs
|
||||
errors.add(
|
||||
:base,
|
||||
:invalid_sell_quantity,
|
||||
sell_qty: account_trade.qty.abs,
|
||||
ticker: account_trade.security.ticker,
|
||||
current_qty: current_qty
|
||||
)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
49
app/models/account_import.rb
Normal file
49
app/models/account_import.rb
Normal file
|
@ -0,0 +1,49 @@
|
|||
class AccountImport < Import
|
||||
def import!
|
||||
transaction do
|
||||
rows.each do |row|
|
||||
mapping = mappings.account_types.find_by(key: row.entity_type)
|
||||
accountable_class = mapping.value.constantize
|
||||
|
||||
account = family.accounts.build(
|
||||
name: row.name,
|
||||
balance: row.amount.to_d,
|
||||
currency: row.currency,
|
||||
accountable: accountable_class.new,
|
||||
import: self
|
||||
)
|
||||
|
||||
account.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def mapping_steps
|
||||
[ Import::AccountTypeMapping ]
|
||||
end
|
||||
|
||||
def required_column_keys
|
||||
%i[name amount]
|
||||
end
|
||||
|
||||
def column_keys
|
||||
%i[entity_type name amount currency]
|
||||
end
|
||||
|
||||
def dry_run
|
||||
{
|
||||
accounts: rows.count
|
||||
}
|
||||
end
|
||||
|
||||
def csv_template
|
||||
template = <<-CSV
|
||||
Account type*,Name*,Balance*,Currency
|
||||
Checking,Main Checking Account,1000.00,USD
|
||||
Savings,Emergency Fund,5000.00,USD
|
||||
Credit Card,Rewards Card,-500.00,USD
|
||||
CSV
|
||||
|
||||
CSV.parse(template, headers: true)
|
||||
end
|
||||
end
|
|
@ -1,5 +1,6 @@
|
|||
class Category < ApplicationRecord
|
||||
has_many :transactions, dependent: :nullify, class_name: "Account::Transaction"
|
||||
has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping"
|
||||
belongs_to :family
|
||||
|
||||
validates :name, :color, :family, presence: true
|
||||
|
|
|
@ -3,9 +3,9 @@ class Family < ApplicationRecord
|
|||
has_many :tags, dependent: :destroy
|
||||
has_many :accounts, dependent: :destroy
|
||||
has_many :institutions, dependent: :destroy
|
||||
has_many :imports, dependent: :destroy
|
||||
has_many :transactions, through: :accounts
|
||||
has_many :entries, through: :accounts
|
||||
has_many :imports, through: :accounts
|
||||
has_many :categories, dependent: :destroy
|
||||
has_many :merchants, dependent: :destroy
|
||||
has_many :issues, through: :accounts
|
||||
|
|
|
@ -1,185 +1,137 @@
|
|||
class Import < ApplicationRecord
|
||||
belongs_to :account
|
||||
TYPES = %w[TransactionImport TradeImport AccountImport MintImport].freeze
|
||||
SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]
|
||||
|
||||
validate :raw_file_must_be_parsable
|
||||
validates :col_sep, inclusion: { in: Csv::COL_SEP_LIST }
|
||||
|
||||
before_save :initialize_csv, if: :should_initialize_csv?
|
||||
|
||||
enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true
|
||||
|
||||
store_accessor :column_mappings, :define_column_mapping_keys
|
||||
belongs_to :family
|
||||
|
||||
scope :ordered, -> { order(created_at: :desc) }
|
||||
|
||||
FALLBACK_TRANSACTION_NAME = "Imported transaction"
|
||||
enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true
|
||||
|
||||
validates :type, inclusion: { in: TYPES }
|
||||
validates :col_sep, inclusion: { in: [ ",", ";" ] }
|
||||
validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }
|
||||
|
||||
has_many :rows, dependent: :destroy
|
||||
has_many :mappings, dependent: :destroy
|
||||
has_many :accounts, dependent: :destroy
|
||||
has_many :entries, dependent: :destroy, class_name: "Account::Entry"
|
||||
|
||||
def publish_later
|
||||
raise "Import is not publishable" unless publishable?
|
||||
|
||||
update! status: :importing
|
||||
|
||||
ImportJob.perform_later(self)
|
||||
end
|
||||
|
||||
def loaded?
|
||||
def publish
|
||||
import!
|
||||
|
||||
family.sync
|
||||
|
||||
update! status: :complete
|
||||
rescue => error
|
||||
update! status: :failed, error: error.message
|
||||
end
|
||||
|
||||
def csv_rows
|
||||
@csv_rows ||= parsed_csv
|
||||
end
|
||||
|
||||
def csv_headers
|
||||
parsed_csv.headers
|
||||
end
|
||||
|
||||
def csv_sample
|
||||
@csv_sample ||= parsed_csv.first(2)
|
||||
end
|
||||
|
||||
def dry_run
|
||||
{
|
||||
transactions: rows.count,
|
||||
accounts: Import::AccountMapping.for_import(self).creational.count,
|
||||
categories: Import::CategoryMapping.for_import(self).creational.count,
|
||||
tags: Import::TagMapping.for_import(self).creational.count
|
||||
}
|
||||
end
|
||||
|
||||
def required_column_keys
|
||||
[]
|
||||
end
|
||||
|
||||
def column_keys
|
||||
raise NotImplementedError, "Subclass must implement column_keys"
|
||||
end
|
||||
|
||||
def generate_rows_from_csv
|
||||
rows.destroy_all
|
||||
|
||||
mapped_rows = csv_rows.map do |row|
|
||||
{
|
||||
account: row[account_col_label].to_s,
|
||||
date: row[date_col_label].to_s,
|
||||
qty: row[qty_col_label].to_s,
|
||||
ticker: row[ticker_col_label].to_s,
|
||||
price: row[price_col_label].to_s,
|
||||
amount: row[amount_col_label].to_s,
|
||||
currency: (row[currency_col_label] || default_currency).to_s,
|
||||
name: (row[name_col_label] || default_row_name).to_s,
|
||||
category: row[category_col_label].to_s,
|
||||
tags: row[tags_col_label].to_s,
|
||||
entity_type: row[entity_type_col_label].to_s,
|
||||
notes: row[notes_col_label].to_s
|
||||
}
|
||||
end
|
||||
|
||||
rows.insert_all!(mapped_rows)
|
||||
end
|
||||
|
||||
def sync_mappings
|
||||
mapping_steps.each do |mapping|
|
||||
mapping.sync(self)
|
||||
end
|
||||
end
|
||||
|
||||
def mapping_steps
|
||||
[]
|
||||
end
|
||||
|
||||
def uploaded?
|
||||
raw_file_str.present?
|
||||
end
|
||||
|
||||
def configured?
|
||||
csv.present?
|
||||
uploaded? && rows.any?
|
||||
end
|
||||
|
||||
def cleaned?
|
||||
loaded? && configured? && csv.valid?
|
||||
configured? && rows.all?(&:valid?)
|
||||
end
|
||||
|
||||
def csv
|
||||
get_normalized_csv_with_validation
|
||||
end
|
||||
|
||||
def available_headers
|
||||
get_raw_csv.table.headers
|
||||
end
|
||||
|
||||
def get_selected_header_for_field(field)
|
||||
column_mappings&.dig(field.key) || field.key
|
||||
end
|
||||
|
||||
def update_csv!(row_idx:, col_idx:, value:)
|
||||
updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value)
|
||||
update! normalized_csv_str: updated_csv.to_s
|
||||
end
|
||||
|
||||
# Type-specific methods (potential STI inheritance in future when more import types added)
|
||||
def publish
|
||||
update!(status: "importing")
|
||||
|
||||
transaction do
|
||||
generate_transactions.each do |txn|
|
||||
txn.save!
|
||||
end
|
||||
end
|
||||
|
||||
self.account.sync
|
||||
|
||||
update!(status: "complete")
|
||||
rescue => e
|
||||
update!(status: "failed")
|
||||
Rails.logger.error("Import with id #{id} failed: #{e}")
|
||||
end
|
||||
|
||||
def dry_run
|
||||
generate_transactions
|
||||
end
|
||||
|
||||
def expected_fields
|
||||
@expected_fields ||= create_expected_fields
|
||||
def publishable?
|
||||
cleaned? && mappings.all?(&:valid?)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def get_normalized_csv_with_validation
|
||||
return nil if normalized_csv_str.nil?
|
||||
|
||||
csv = Import::Csv.new(normalized_csv_str)
|
||||
|
||||
expected_fields.each do |field|
|
||||
csv.define_validator(field.key, field.validator) if field.validator
|
||||
end
|
||||
|
||||
csv
|
||||
def import!
|
||||
# no-op, subclasses can implement for customization of algorithm
|
||||
end
|
||||
|
||||
def get_raw_csv
|
||||
return nil if raw_file_str.nil?
|
||||
Import::Csv.new(raw_file_str, col_sep:)
|
||||
def default_row_name
|
||||
"Imported item"
|
||||
end
|
||||
|
||||
def should_initialize_csv?
|
||||
raw_file_str_changed? || column_mappings_changed?
|
||||
def default_currency
|
||||
family.currency
|
||||
end
|
||||
|
||||
def initialize_csv
|
||||
generated_csv = generate_normalized_csv(raw_file_str)
|
||||
self.normalized_csv_str = generated_csv.table.to_s
|
||||
end
|
||||
|
||||
# Uses the user-provided raw CSV + mappings to generate a normalized CSV for the import
|
||||
def generate_normalized_csv(csv_str)
|
||||
Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings, col_sep)
|
||||
end
|
||||
|
||||
def update_csv(row_idx, col_idx, value)
|
||||
updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value)
|
||||
update! normalized_csv_str: updated_csv.to_s
|
||||
end
|
||||
|
||||
def generate_transactions
|
||||
transaction_entries = []
|
||||
category_cache = {}
|
||||
tag_cache = {}
|
||||
|
||||
csv.table.each do |row|
|
||||
category_name = row["category"].presence
|
||||
tag_strings = row["tags"].presence&.split("|") || []
|
||||
tags = []
|
||||
|
||||
tag_strings.each do |tag_string|
|
||||
tags << tag_cache[tag_string] ||= account.family.tags.find_or_initialize_by(name: tag_string)
|
||||
end
|
||||
|
||||
category = category_cache[category_name] ||= account.family.categories.find_or_initialize_by(name: category_name) if category_name.present?
|
||||
|
||||
entry = account.entries.build \
|
||||
name: row["name"].presence || FALLBACK_TRANSACTION_NAME,
|
||||
date: Date.iso8601(row["date"]),
|
||||
currency: account.currency,
|
||||
amount: BigDecimal(row["amount"]) * -1,
|
||||
entryable: Account::Transaction.new(category: category, tags: tags)
|
||||
|
||||
transaction_entries << entry
|
||||
end
|
||||
|
||||
transaction_entries
|
||||
end
|
||||
|
||||
def create_expected_fields
|
||||
date_field = Import::Field.new \
|
||||
key: "date",
|
||||
label: "Date",
|
||||
validator: ->(value) { Import::Field.iso_date_validator(value) }
|
||||
|
||||
name_field = Import::Field.new \
|
||||
key: "name",
|
||||
label: "Name",
|
||||
is_optional: true
|
||||
|
||||
category_field = Import::Field.new \
|
||||
key: "category",
|
||||
label: "Category",
|
||||
is_optional: true
|
||||
|
||||
tags_field = Import::Field.new \
|
||||
key: "tags",
|
||||
label: "Tags",
|
||||
is_optional: true
|
||||
|
||||
amount_field = Import::Field.new \
|
||||
key: "amount",
|
||||
label: "Amount",
|
||||
validator: ->(value) { Import::Field.bigdecimal_validator(value) }
|
||||
|
||||
[ date_field, name_field, category_field, tags_field, amount_field ]
|
||||
end
|
||||
|
||||
def define_column_mapping_keys
|
||||
expected_fields.each do |field|
|
||||
field.key.to_sym
|
||||
end
|
||||
end
|
||||
|
||||
def raw_file_must_be_parsable
|
||||
begin
|
||||
CSV.parse(raw_file_str || "", col_sep:)
|
||||
rescue CSV::MalformedCSVError
|
||||
errors.add(:raw_file_str, :invalid_csv_format)
|
||||
end
|
||||
def parsed_csv
|
||||
@parsed_csv ||= CSV.parse(
|
||||
(raw_file_str || "").strip,
|
||||
headers: true,
|
||||
col_sep: col_sep,
|
||||
converters: [ ->(str) { str&.strip } ]
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
45
app/models/import/account_mapping.rb
Normal file
45
app/models/import/account_mapping.rb
Normal file
|
@ -0,0 +1,45 @@
|
|||
class Import::AccountMapping < Import::Mapping
|
||||
validates :mappable, presence: true, if: -> { key.blank? || !create_when_empty }
|
||||
|
||||
class << self
|
||||
def mapping_values(import)
|
||||
import.rows.map(&:account).uniq
|
||||
end
|
||||
end
|
||||
|
||||
def selectable_values
|
||||
family_accounts = import.family.accounts.alphabetically.map { |account| [ account.name, account.id ] }
|
||||
|
||||
unless key.blank?
|
||||
family_accounts.unshift [ "Add as new account", CREATE_NEW_KEY ]
|
||||
end
|
||||
|
||||
family_accounts
|
||||
end
|
||||
|
||||
def requires_selection?
|
||||
true
|
||||
end
|
||||
|
||||
def values_count
|
||||
import.rows.where(account: key).count
|
||||
end
|
||||
|
||||
def mappable_class
|
||||
Account
|
||||
end
|
||||
|
||||
def create_mappable!
|
||||
return unless creatable?
|
||||
|
||||
account = import.family.accounts.create_or_find_by!(name: key) do |new_account|
|
||||
new_account.balance = 0
|
||||
new_account.import = import
|
||||
new_account.currency = import.family.currency
|
||||
new_account.accountable = Depository.new
|
||||
end
|
||||
|
||||
self.mappable = account
|
||||
save!
|
||||
end
|
||||
end
|
25
app/models/import/account_type_mapping.rb
Normal file
25
app/models/import/account_type_mapping.rb
Normal file
|
@ -0,0 +1,25 @@
|
|||
class Import::AccountTypeMapping < Import::Mapping
|
||||
validates :value, presence: true
|
||||
|
||||
class << self
|
||||
def mapping_values(import)
|
||||
import.rows.map(&:entity_type).uniq
|
||||
end
|
||||
end
|
||||
|
||||
def selectable_values
|
||||
Accountable::TYPES.map { |type| [ type.titleize, type ] }
|
||||
end
|
||||
|
||||
def requires_selection?
|
||||
true
|
||||
end
|
||||
|
||||
def values_count
|
||||
import.rows.where(entity_type: key).count
|
||||
end
|
||||
|
||||
def create_mappable!
|
||||
# no-op
|
||||
end
|
||||
end
|
36
app/models/import/category_mapping.rb
Normal file
36
app/models/import/category_mapping.rb
Normal file
|
@ -0,0 +1,36 @@
|
|||
class Import::CategoryMapping < Import::Mapping
|
||||
class << self
|
||||
def mapping_values(import)
|
||||
import.rows.map(&:category).uniq
|
||||
end
|
||||
end
|
||||
|
||||
def selectable_values
|
||||
family_categories = import.family.categories.alphabetically.map { |category| [ category.name, category.id ] }
|
||||
|
||||
unless key.blank?
|
||||
family_categories.unshift [ "Add as new category", CREATE_NEW_KEY ]
|
||||
end
|
||||
|
||||
family_categories
|
||||
end
|
||||
|
||||
def requires_selection?
|
||||
false
|
||||
end
|
||||
|
||||
def values_count
|
||||
import.rows.where(category: key).count
|
||||
end
|
||||
|
||||
def mappable_class
|
||||
Category
|
||||
end
|
||||
|
||||
def create_mappable!
|
||||
return unless creatable?
|
||||
|
||||
self.mappable = import.family.categories.find_or_create_by!(name: key)
|
||||
save!
|
||||
end
|
||||
end
|
|
@ -1,83 +0,0 @@
|
|||
class Import::Csv
|
||||
DEFAULT_COL_SEP = ",".freeze
|
||||
COL_SEP_LIST = [ DEFAULT_COL_SEP, ";" ].freeze
|
||||
|
||||
def self.parse_csv(csv_str, col_sep: DEFAULT_COL_SEP)
|
||||
CSV.parse(
|
||||
csv_str&.strip || "",
|
||||
headers: true,
|
||||
col_sep:,
|
||||
converters: [ ->(str) { str&.strip } ]
|
||||
)
|
||||
end
|
||||
|
||||
def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
|
||||
raw_csv = self.parse_csv(raw_file_str, col_sep:)
|
||||
|
||||
generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|
|
||||
raw_csv.each do |row|
|
||||
row_values = []
|
||||
|
||||
fields.each do |field|
|
||||
# Finds the column header name the user has designated for the expected field
|
||||
mapped_field_key = field_mappings[field.key] if field_mappings
|
||||
mapped_header = mapped_field_key || field.key
|
||||
|
||||
row_values << row.fetch(mapped_header, "")
|
||||
end
|
||||
|
||||
csv << row_values
|
||||
end
|
||||
end
|
||||
|
||||
new(generated_csv_str, col_sep:)
|
||||
end
|
||||
|
||||
attr_reader :csv_str, :col_sep
|
||||
|
||||
def initialize(csv_str, column_validators: nil, col_sep: DEFAULT_COL_SEP)
|
||||
@csv_str = csv_str
|
||||
@col_sep = col_sep
|
||||
@column_validators = column_validators || {}
|
||||
end
|
||||
|
||||
def table
|
||||
@table ||= self.class.parse_csv(csv_str, col_sep:)
|
||||
end
|
||||
|
||||
def update_cell(row_idx, col_idx, value)
|
||||
copy = table.by_col_or_row
|
||||
copy[row_idx][col_idx] = value
|
||||
copy
|
||||
end
|
||||
|
||||
def valid?
|
||||
table.each_with_index.all? do |row, row_idx|
|
||||
row.each_with_index.all? do |cell, col_idx|
|
||||
cell_valid?(row_idx, col_idx)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def cell_valid?(row_idx, col_idx)
|
||||
value = table.dig(row_idx, col_idx)
|
||||
header = table.headers[col_idx]
|
||||
validator = get_validator_by_header(header)
|
||||
validator.call(value)
|
||||
end
|
||||
|
||||
def define_validator(header_key, validator = nil, &block)
|
||||
header = table.headers.find { |h| h.strip == header_key }
|
||||
raise "Cannot define validator for header #{header_key}: header does not exist in CSV" if header.nil?
|
||||
|
||||
column_validators[header] = validator || block
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_accessor :column_validators
|
||||
|
||||
def get_validator_by_header(header)
|
||||
column_validators&.dig(header) || ->(_v) { true }
|
||||
end
|
||||
end
|
|
@ -1,37 +0,0 @@
|
|||
class Import::Field
|
||||
def self.iso_date_validator(value)
|
||||
Date.iso8601(value)
|
||||
true
|
||||
rescue
|
||||
false
|
||||
end
|
||||
|
||||
def self.bigdecimal_validator(value)
|
||||
BigDecimal(value)
|
||||
true
|
||||
rescue
|
||||
false
|
||||
end
|
||||
|
||||
attr_reader :key, :label, :validator
|
||||
|
||||
def initialize(key:, label:, is_optional: false, validator: nil)
|
||||
@key = key.to_s
|
||||
@label = label
|
||||
@is_optional = is_optional
|
||||
@validator = validator
|
||||
end
|
||||
|
||||
def optional?
|
||||
@is_optional
|
||||
end
|
||||
|
||||
def define_validator(validator = nil, &block)
|
||||
@validator = validator || block
|
||||
end
|
||||
|
||||
def validate(value)
|
||||
return true if validator.nil?
|
||||
validator.call(value)
|
||||
end
|
||||
end
|
56
app/models/import/mapping.rb
Normal file
56
app/models/import/mapping.rb
Normal file
|
@ -0,0 +1,56 @@
|
|||
class Import::Mapping < ApplicationRecord
|
||||
CREATE_NEW_KEY = "internal_new_resource"
|
||||
|
||||
belongs_to :import
|
||||
belongs_to :mappable, polymorphic: true, optional: true
|
||||
|
||||
validates :key, presence: true, uniqueness: { scope: [ :import_id, :type ] }, allow_blank: true
|
||||
|
||||
scope :for_import, ->(import) { where(import: import) }
|
||||
scope :creational, -> { where(create_when_empty: true, mappable: nil) }
|
||||
scope :categories, -> { where(type: "Import::CategoryMapping") }
|
||||
scope :tags, -> { where(type: "Import::TagMapping") }
|
||||
scope :accounts, -> { where(type: "Import::AccountMapping") }
|
||||
scope :account_types, -> { where(type: "Import::AccountTypeMapping") }
|
||||
|
||||
class << self
|
||||
def mappable_for(key)
|
||||
find_by(key: key)&.mappable
|
||||
end
|
||||
|
||||
def sync(import)
|
||||
unique_values = mapping_values(import).uniq
|
||||
|
||||
unique_values.each do |value|
|
||||
mapping = find_or_initialize_by(key: value, import: import, create_when_empty: value.present?)
|
||||
mapping.save(validate: false) if mapping.new_record?
|
||||
end
|
||||
|
||||
where(import: import).where.not(key: unique_values).destroy_all
|
||||
end
|
||||
|
||||
def mapping_values(import)
|
||||
raise NotImplementedError, "Subclass must implement mapping_values"
|
||||
end
|
||||
end
|
||||
|
||||
def selectable_values
|
||||
raise NotImplementedError, "Subclass must implement selectable_values"
|
||||
end
|
||||
|
||||
def values_count
|
||||
raise NotImplementedError, "Subclass must implement values_count"
|
||||
end
|
||||
|
||||
def mappable_class
|
||||
nil
|
||||
end
|
||||
|
||||
def creatable?
|
||||
mappable.nil? && key.present? && create_when_empty
|
||||
end
|
||||
|
||||
def create_mappable!
|
||||
raise NotImplementedError, "Subclass must implement create_mappable!"
|
||||
end
|
||||
end
|
70
app/models/import/row.rb
Normal file
70
app/models/import/row.rb
Normal file
|
@ -0,0 +1,70 @@
|
|||
class Import::Row < ApplicationRecord
|
||||
belongs_to :import
|
||||
|
||||
validates :amount, numericality: true, allow_blank: true
|
||||
validates :currency, presence: true
|
||||
|
||||
validate :date_matches_user_format
|
||||
validate :required_columns
|
||||
validate :currency_is_valid
|
||||
|
||||
scope :ordered, -> { order(:id) }
|
||||
|
||||
def tags_list
|
||||
if tags.blank?
|
||||
[ "" ]
|
||||
else
|
||||
tags.split("|").map(&:strip)
|
||||
end
|
||||
end
|
||||
|
||||
def date_iso
|
||||
Date.strptime(date, import.date_format).iso8601
|
||||
end
|
||||
|
||||
def signed_amount
|
||||
if import.type == "TradeImport"
|
||||
price.to_d * apply_signage_convention(qty.to_d)
|
||||
else
|
||||
apply_signage_convention(amount.to_d)
|
||||
end
|
||||
end
|
||||
|
||||
def sync_mappings
|
||||
Import::CategoryMapping.sync(import) if import.column_keys.include?(:category)
|
||||
Import::TagMapping.sync(import) if import.column_keys.include?(:tags)
|
||||
Import::AccountMapping.sync(import) if import.column_keys.include?(:account)
|
||||
Import::AccountTypeMapping.sync(import) if import.column_keys.include?(:entity_type)
|
||||
end
|
||||
|
||||
private
|
||||
def apply_signage_convention(value)
|
||||
value * (import.signage_convention == "inflows_positive" ? 1 : -1)
|
||||
end
|
||||
|
||||
def required_columns
|
||||
import.required_column_keys.each do |required_key|
|
||||
errors.add(required_key, "is required") if self[required_key].blank?
|
||||
end
|
||||
end
|
||||
|
||||
def date_matches_user_format
|
||||
return if date.blank?
|
||||
|
||||
parsed_date = Date.strptime(date, import.date_format) rescue nil
|
||||
|
||||
if parsed_date.nil?
|
||||
errors.add(:date, "must exactly match the format: #{import.date_format}")
|
||||
end
|
||||
end
|
||||
|
||||
def currency_is_valid
|
||||
return true if currency.blank?
|
||||
|
||||
begin
|
||||
Money::Currency.new(currency)
|
||||
rescue Money::Currency::UnknownCurrencyError
|
||||
errors.add(:currency, "is not a valid currency code")
|
||||
end
|
||||
end
|
||||
end
|
36
app/models/import/tag_mapping.rb
Normal file
36
app/models/import/tag_mapping.rb
Normal file
|
@ -0,0 +1,36 @@
|
|||
class Import::TagMapping < Import::Mapping
|
||||
class << self
|
||||
def mapping_values(import)
|
||||
import.rows.map(&:tags_list).flatten.uniq
|
||||
end
|
||||
end
|
||||
|
||||
def selectable_values
|
||||
family_tags = import.family.tags.alphabetically.map { |tag| [ tag.name, tag.id ] }
|
||||
|
||||
unless key.blank?
|
||||
family_tags.unshift [ "Add as new tag", CREATE_NEW_KEY ]
|
||||
end
|
||||
|
||||
family_tags
|
||||
end
|
||||
|
||||
def requires_selection?
|
||||
false
|
||||
end
|
||||
|
||||
def values_count
|
||||
import.rows.map(&:tags_list).flatten.count { |tag| tag == key }
|
||||
end
|
||||
|
||||
def mappable_class
|
||||
Tag
|
||||
end
|
||||
|
||||
def create_mappable!
|
||||
return unless creatable?
|
||||
|
||||
self.mappable = import.family.tags.find_or_create_by!(name: key)
|
||||
save!
|
||||
end
|
||||
end
|
94
app/models/mint_import.rb
Normal file
94
app/models/mint_import.rb
Normal file
|
@ -0,0 +1,94 @@
|
|||
class MintImport < Import
|
||||
after_create :set_mappings
|
||||
|
||||
def generate_rows_from_csv
|
||||
rows.destroy_all
|
||||
|
||||
mapped_rows = csv_rows.map do |row|
|
||||
{
|
||||
account: row[account_col_label].to_s,
|
||||
date: row[date_col_label].to_s,
|
||||
amount: signed_csv_amount(row).to_s,
|
||||
currency: (row[currency_col_label] || default_currency).to_s,
|
||||
name: (row[name_col_label] || default_row_name).to_s,
|
||||
category: row[category_col_label].to_s,
|
||||
tags: row[tags_col_label].to_s,
|
||||
notes: row[notes_col_label].to_s
|
||||
}
|
||||
end
|
||||
|
||||
rows.insert_all!(mapped_rows)
|
||||
end
|
||||
|
||||
def import!
|
||||
transaction do
|
||||
mappings.each(&:create_mappable!)
|
||||
|
||||
rows.each do |row|
|
||||
account = mappings.accounts.mappable_for(row.account)
|
||||
category = mappings.categories.mappable_for(row.category)
|
||||
tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact
|
||||
|
||||
entry = account.entries.build \
|
||||
date: row.date_iso,
|
||||
amount: row.signed_amount,
|
||||
name: row.name,
|
||||
currency: row.currency,
|
||||
entryable: Account::Transaction.new(category: category, tags: tags, notes: row.notes),
|
||||
import: self
|
||||
|
||||
entry.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def mapping_steps
|
||||
[ Import::CategoryMapping, Import::TagMapping, Import::AccountMapping ]
|
||||
end
|
||||
|
||||
def required_column_keys
|
||||
%i[date amount]
|
||||
end
|
||||
|
||||
def column_keys
|
||||
%i[date amount name currency category tags account notes]
|
||||
end
|
||||
|
||||
def csv_template
|
||||
template = <<-CSV
|
||||
Date,Amount,Account Name,Description,Category,Labels,Currency,Notes,Transaction Type
|
||||
01/01/2024,-8.55,Checking,Starbucks,Food & Drink,Coffee|Breakfast,USD,Morning coffee,debit
|
||||
04/15/2024,2000,Savings,Paycheck,Income,,USD,Bi-weekly salary,credit
|
||||
CSV
|
||||
|
||||
CSV.parse(template, headers: true)
|
||||
end
|
||||
|
||||
def signed_csv_amount(csv_row)
|
||||
amount = csv_row[amount_col_label]
|
||||
type = csv_row["Transaction Type"]
|
||||
|
||||
if type == "credit"
|
||||
amount.to_d
|
||||
else
|
||||
amount.to_d * -1
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
def set_mappings
|
||||
self.signage_convention = "inflows_positive"
|
||||
self.date_col_label = "Date"
|
||||
self.date_format = "%m/%d/%Y"
|
||||
self.name_col_label = "Description"
|
||||
self.amount_col_label = "Amount"
|
||||
self.currency_col_label = "Currency"
|
||||
self.account_col_label = "Account Name"
|
||||
self.category_col_label = "Category"
|
||||
self.tags_col_label = "Labels"
|
||||
self.notes_col_label = "Notes"
|
||||
self.entity_type_col_label = "Transaction Type"
|
||||
|
||||
save!
|
||||
end
|
||||
end
|
|
@ -2,6 +2,7 @@ class Tag < ApplicationRecord
|
|||
belongs_to :family
|
||||
has_many :taggings, dependent: :destroy
|
||||
has_many :transactions, through: :taggings, source: :taggable, source_type: "Account::Transaction"
|
||||
has_many :import_mappings, as: :mappable, dependent: :destroy, class_name: "Import::Mapping"
|
||||
|
||||
validates :name, presence: true, uniqueness: { scope: :family }
|
||||
|
||||
|
|
52
app/models/trade_import.rb
Normal file
52
app/models/trade_import.rb
Normal file
|
@ -0,0 +1,52 @@
|
|||
class TradeImport < Import
|
||||
def import!
|
||||
transaction do
|
||||
mappings.each(&:create_mappable!)
|
||||
|
||||
rows.each do |row|
|
||||
account = mappings.accounts.mappable_for(row.account)
|
||||
security = Security.find_or_create_by(ticker: row.ticker)
|
||||
|
||||
entry = account.entries.build \
|
||||
date: row.date_iso,
|
||||
amount: row.signed_amount,
|
||||
name: row.name,
|
||||
currency: row.currency,
|
||||
entryable: Account::Trade.new(security: security, qty: row.qty, currency: row.currency, price: row.price),
|
||||
import: self
|
||||
|
||||
entry.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def mapping_steps
|
||||
[ Import::AccountMapping ]
|
||||
end
|
||||
|
||||
def required_column_keys
|
||||
%i[date ticker qty price]
|
||||
end
|
||||
|
||||
def column_keys
|
||||
%i[date ticker qty price currency account name]
|
||||
end
|
||||
|
||||
def dry_run
|
||||
{
|
||||
transactions: rows.count,
|
||||
accounts: Import::AccountMapping.for_import(self).creational.count
|
||||
}
|
||||
end
|
||||
|
||||
def csv_template
|
||||
template = <<-CSV
|
||||
date*,ticker*,qty*,price*,currency,account,name
|
||||
05/15/2024,AAPL,10,150.00,USD,Trading Account,Apple Inc. Purchase
|
||||
05/16/2024,GOOGL,-5,2500.00,USD,Investment Account,Alphabet Inc. Sale
|
||||
05/17/2024,TSLA,2,700.50,USD,Retirement Account,Tesla Inc. Purchase
|
||||
CSV
|
||||
|
||||
CSV.parse(template, headers: true)
|
||||
end
|
||||
end
|
46
app/models/transaction_import.rb
Normal file
46
app/models/transaction_import.rb
Normal file
|
@ -0,0 +1,46 @@
|
|||
class TransactionImport < Import
|
||||
def import!
|
||||
transaction do
|
||||
mappings.each(&:create_mappable!)
|
||||
|
||||
rows.each do |row|
|
||||
account = mappings.accounts.mappable_for(row.account)
|
||||
category = mappings.categories.mappable_for(row.category)
|
||||
tags = row.tags_list.map { |tag| mappings.tags.mappable_for(tag) }.compact
|
||||
|
||||
entry = account.entries.build \
|
||||
date: row.date_iso,
|
||||
amount: row.signed_amount,
|
||||
name: row.name,
|
||||
currency: row.currency,
|
||||
entryable: Account::Transaction.new(category: category, tags: tags, notes: row.notes),
|
||||
import: self
|
||||
|
||||
entry.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def required_column_keys
|
||||
%i[date amount]
|
||||
end
|
||||
|
||||
def column_keys
|
||||
%i[date amount name currency category tags account notes]
|
||||
end
|
||||
|
||||
def mapping_steps
|
||||
[ Import::CategoryMapping, Import::TagMapping, Import::AccountMapping ]
|
||||
end
|
||||
|
||||
def csv_template
|
||||
template = <<-CSV
|
||||
date*,amount*,name,currency,category,tags,account,notes
|
||||
05/15/2024,-45.99,Grocery Store,USD,Food,groceries|essentials,Checking Account,Monthly grocery run
|
||||
05/16/2024,1500.00,Salary,,Income,,Main Account,
|
||||
05/17/2024,-12.50,Coffee Shop,,,coffee,,
|
||||
CSV
|
||||
|
||||
CSV.parse(template, headers: true)
|
||||
end
|
||||
end
|
Loading…
Add table
Add a link
Reference in a new issue