diff --git a/app/models/import.rb b/app/models/import.rb
index 472b5e12..1fd85d02 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -2,8 +2,17 @@ class Import < ApplicationRecord
TYPES = %w[TransactionImport TradeImport AccountImport MintImport].freeze
SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]
+ NUMBER_FORMATS = {
+ "1,234.56" => { separator: ".", delimiter: "," }, # US/UK/Asia
+ "1.234,56" => { separator: ",", delimiter: "." }, # Most of Europe
+ "1 234,56" => { separator: ",", delimiter: " " }, # French/Scandinavian
+ "1,234" => { separator: "", delimiter: "," } # Zero-decimal currencies like JPY
+ }.freeze
+
belongs_to :family
+ before_validation :set_default_number_format
+
scope :ordered, -> { order(created_at: :desc) }
enum :status, {
@@ -18,6 +27,7 @@ class Import < ApplicationRecord
validates :type, inclusion: { in: TYPES }
validates :col_sep, inclusion: { in: [ ",", ";" ] }
validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }
+ validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys }
has_many :rows, dependent: :destroy
has_many :mappings, dependent: :destroy
@@ -95,8 +105,8 @@ class Import < ApplicationRecord
def generate_rows_from_csv
rows.destroy_all
- mapped_rows = csv_rows.map do |row|
- {
+ csv_rows.each do |row|
+ rows.create!(
account: row[account_col_label].to_s,
date: row[date_col_label].to_s,
qty: sanitize_number(row[qty_col_label]).to_s,
@@ -109,10 +119,8 @@ class Import < ApplicationRecord
tags: row[tags_col_label].to_s,
entity_type: row[entity_type_col_label].to_s,
notes: row[notes_col_label].to_s
- }
+ )
end
-
- rows.insert_all!(mapped_rows)
end
def sync_mappings
@@ -177,6 +185,39 @@ class Import < ApplicationRecord
def sanitize_number(value)
return "" if value.nil?
- value.gsub(/[^\d.\-]/, "")
+
+ format = NUMBER_FORMATS[number_format]
+ return "" unless format
+
+ # First, normalize spaces and remove any characters that aren't numbers, delimiters, separators, or minus signs
+ sanitized = value.to_s.strip
+
+ # Handle French/Scandinavian format specially
+ if format[:delimiter] == " "
+ sanitized = sanitized.gsub(/\s+/, "") # Remove all spaces first
+ else
+ sanitized = sanitized.gsub(/[^\d#{Regexp.escape(format[:delimiter])}#{Regexp.escape(format[:separator])}\-]/, "")
+
+ # Replace delimiter with empty string
+ if format[:delimiter].present?
+ sanitized = sanitized.gsub(format[:delimiter], "")
+ end
+ end
+
+ # Replace separator with period for proper float parsing
+ if format[:separator].present?
+ sanitized = sanitized.gsub(format[:separator], ".")
+ end
+
+ # Return empty string if not a valid number
+ unless sanitized =~ /\A-?\d+\.?\d*\z/
+ return ""
+ end
+
+ sanitized
+ end
+
+ def set_default_number_format
+ self.number_format ||= "1,234.56" # Default to US/UK format
end
end
diff --git a/app/views/import/configurations/_mint_import.html.erb b/app/views/import/configurations/_mint_import.html.erb
index d0eeced4..6e07168a 100644
--- a/app/views/import/configurations/_mint_import.html.erb
+++ b/app/views/import/configurations/_mint_import.html.erb
@@ -16,7 +16,10 @@
<%= form.select :signage_convention, [["Incomes are negative", "inflows_negative"], ["Incomes are positive", "inflows_positive"]], { label: true }, disabled: import.complete? %>
- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" }, disabled: import.complete? %>
+
+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" }, disabled: import.complete? %>
+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+
<%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %>
<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" }, disabled: import.complete? %>
diff --git a/app/views/import/configurations/_trade_import.html.erb b/app/views/import/configurations/_trade_import.html.erb
index b8dab627..e49d29c3 100644
--- a/app/views/import/configurations/_trade_import.html.erb
+++ b/app/views/import/configurations/_trade_import.html.erb
@@ -11,7 +11,10 @@
<%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %>
- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+
+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+
<%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %>
<%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %>
diff --git a/app/views/import/configurations/_transaction_import.html.erb b/app/views/import/configurations/_transaction_import.html.erb
index 82abb1c5..65c9f5f4 100644
--- a/app/views/import/configurations/_transaction_import.html.erb
+++ b/app/views/import/configurations/_transaction_import.html.erb
@@ -11,7 +11,10 @@
<%= form.select :signage_convention, [["Incomes are positive", "inflows_positive"], ["Incomes are negative", "inflows_negative"]], label: true %>
- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+
+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+ <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+
<%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>
<%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>
diff --git a/config/locales/models/import/en.yml b/config/locales/models/import/en.yml
index 0019291c..713ad905 100644
--- a/config/locales/models/import/en.yml
+++ b/config/locales/models/import/en.yml
@@ -7,3 +7,7 @@ en:
attributes:
raw_file_str:
invalid_csv_format: is not a valid CSV format
+ attributes:
+ import:
+ currency: Currency
+ number_format: Number Format
diff --git a/db/migrate/20250207014022_add_number_format_to_imports.rb b/db/migrate/20250207014022_add_number_format_to_imports.rb
new file mode 100644
index 00000000..265fbc6b
--- /dev/null
+++ b/db/migrate/20250207014022_add_number_format_to_imports.rb
@@ -0,0 +1,5 @@
+class AddNumberFormatToImports < ActiveRecord::Migration[7.2]
+ def change
+ add_column :imports, :number_format, :string
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5d6a8e12..f44329c6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
+ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
t.decimal "balance", precision: 19, scale: 4
t.string "currency"
t.boolean "is_active", default: true, null: false
- t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
+ t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true
t.uuid "import_id"
t.uuid "plaid_account_id"
t.boolean "scheduled_for_deletion", default: false
@@ -414,6 +414,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
t.string "date_format", default: "%m/%d/%Y"
t.string "signage_convention", default: "inflows_positive"
t.string "error"
+ t.string "number_format"
t.index ["family_id"], name: "index_imports_on_family_id"
end
diff --git a/test/interfaces/import_interface_test.rb b/test/interfaces/import_interface_test.rb
index 68b09306..9ee56b37 100644
--- a/test/interfaces/import_interface_test.rb
+++ b/test/interfaces/import_interface_test.rb
@@ -54,4 +54,233 @@ module ImportInterfaceTest
assert_equal "Failed to publish", import.error
assert_equal "failed", import.status
end
+
+ test "parses US/UK number format correctly" do
+ import = imports(:transaction)
+ import.update!(
+ number_format: "1,234.56",
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,name\n01/01/2024,\"1,234.56\",Test"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "1234.56", row.amount
+ end
+
+ test "parses European number format correctly" do
+ import = imports(:transaction)
+ import.update!(
+ number_format: "1.234,56",
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,name\n01/01/2024,\"1.234,56\",Test"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "1234.56", row.amount
+ end
+
+ test "parses French/Scandinavian number format correctly" do
+ import = imports(:transaction)
+ import.update!(
+ number_format: "1 234,56",
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ date_format: "%m/%d/%Y"
+ )
+
+ # Quote the amount field to ensure proper CSV parsing
+ csv_data = "date,amount,name\n01/01/2024,\"1 234,56\",Test"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "1234.56", row.amount
+ end
+
+ test "parses zero-decimal currency format correctly" do
+ import = imports(:transaction)
+ import.update!(
+ number_format: "1,234",
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,name\n01/01/2024,1234,Test"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "1234", row.amount
+ end
+
+ test "currency from CSV takes precedence over default" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ currency_col_label: "currency",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+ import.family.update!(currency: "USD")
+
+ csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,EUR"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "EUR", row.currency
+ end
+
+ test "uses default currency when CSV currency column is empty" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ currency_col_label: "currency",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+ import.family.update!(currency: "USD")
+
+ csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "USD", row.currency
+ end
+
+ test "uses default currency when CSV has no currency column" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+ import.family.update!(currency: "USD")
+
+ csv_data = "date,amount,name\n01/01/2024,123.45,Test"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "USD", row.currency
+ end
+
+ test "generates rows with all optional fields" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ account_col_label: "account",
+ category_col_label: "category",
+ tags_col_label: "tags",
+ notes_col_label: "notes",
+ currency_col_label: "currency",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,name,account,category,tags,notes,currency\n" \
+ "01/01/2024,1234.56,Salary,Bank Account,Income,\"monthly,salary\",Salary payment,EUR"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "01/01/2024", row.date
+ assert_equal "1234.56", row.amount
+ assert_equal "Salary", row.name
+ assert_equal "Bank Account", row.account
+ assert_equal "Income", row.category
+ assert_equal "monthly,salary", row.tags
+ assert_equal "Salary payment", row.notes
+ assert_equal "EUR", row.currency
+ end
+
+ test "generates rows with minimal required fields" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount\n01/01/2024,1234.56"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "01/01/2024", row.date
+ assert_equal "1234.56", row.amount
+ assert_equal "Imported item", row.name # Default name
+ assert_equal import.family.currency, row.currency # Default currency
+ end
+
+ test "handles empty values in optional fields" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ name_col_label: "name",
+ category_col_label: "category",
+ tags_col_label: "tags",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,name,category,tags\n01/01/2024,1234.56,,,"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "01/01/2024", row.date
+ assert_equal "1234.56", row.amount
+ assert_equal "Imported item", row.name # Falls back to default
+ assert_equal "", row.category
+ assert_equal "", row.tags
+ end
+
+ test "handles trade-specific fields" do
+ import = imports(:transaction)
+ import.update!(
+ amount_col_label: "amount",
+ date_col_label: "date",
+ qty_col_label: "quantity",
+ ticker_col_label: "symbol",
+ price_col_label: "price",
+ number_format: "1,234.56",
+ date_format: "%m/%d/%Y"
+ )
+
+ csv_data = "date,amount,quantity,symbol,price\n01/01/2024,1234.56,10,AAPL,123.456"
+ import.update!(raw_file_str: csv_data)
+ import.generate_rows_from_csv
+
+ row = import.rows.first
+ assert_equal "10", row.qty
+ assert_equal "AAPL", row.ticker
+ assert_equal "123.456", row.price
+ end
end
diff --git a/test/models/transaction_import_test.rb b/test/models/transaction_import_test.rb
index fc3e2ab9..2299194c 100644
--- a/test/models/transaction_import_test.rb
+++ b/test/models/transaction_import_test.rb
@@ -28,14 +28,19 @@ class TransactionImportTest < ActiveSupport::TestCase
end
test "imports transactions, categories, tags, and accounts" do
- import = <<-CSV
+ import = <<~CSV
date,name,amount,category,tags,account,notes
01/01/2024,Txn1,100,TestCategory1,TestTag1,TestAccount1,notes1
01/02/2024,Txn2,200,TestCategory2,TestTag1|TestTag2,TestAccount2,notes2
01/03/2024,Txn3,300,,,,notes3
CSV
- @import.update!(raw_file_str: import)
+ @import.update!(
+ raw_file_str: import,
+ date_col_label: "date",
+ amount_col_label: "amount",
+ date_format: "%m/%d/%Y"
+ )
@import.generate_rows_from_csv