feat(import): add currency and number format support for CSV imports (#1819)

* feat(import): add currency and number format support for CSV imports

* feat(import): add currency and format for mint and trade

* fix(imports): remove currency field in favor of currency csv col

* fix(imports): remove validate column from import model

* test(import): add some tests for imports

* test(import): add some tests for generate_rows_from_csv

* fix: change method name for import model

* fix: change before validation

---------

Co-authored-by: danestves <danestves@users.noreply.github.com>

app/models/import.rb

@@ -2,8 +2,17 @@ class Import < ApplicationRecord
   TYPES = %w[TransactionImport TradeImport AccountImport MintImport].freeze
   SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative]
 
+  NUMBER_FORMATS = {
+    "1,234.56" => { separator: ".", delimiter: "," },  # US/UK/Asia
+    "1.234,56" => { separator: ",", delimiter: "." },  # Most of Europe
+    "1 234,56" => { separator: ",", delimiter: " " },  # French/Scandinavian
+    "1,234"    => { separator: "",  delimiter: "," }   # Zero-decimal currencies like JPY
+  }.freeze
+
   belongs_to :family
 
+  before_validation :set_default_number_format
+
   scope :ordered, -> { order(created_at: :desc) }
 
   enum :status, {
@@ -18,6 +27,7 @@ class Import < ApplicationRecord
   validates :type, inclusion: { in: TYPES }
   validates :col_sep, inclusion: { in: [ ",", ";" ] }
   validates :signage_convention, inclusion: { in: SIGNAGE_CONVENTIONS }
+  validates :number_format, presence: true, inclusion: { in: NUMBER_FORMATS.keys }
 
   has_many :rows, dependent: :destroy
   has_many :mappings, dependent: :destroy
@@ -95,8 +105,8 @@ class Import < ApplicationRecord
   def generate_rows_from_csv
     rows.destroy_all
 
-    mapped_rows = csv_rows.map do |row|
-      {
+    csv_rows.each do |row|
+      rows.create!(
         account: row[account_col_label].to_s,
         date: row[date_col_label].to_s,
         qty: sanitize_number(row[qty_col_label]).to_s,
@@ -109,10 +119,8 @@ class Import < ApplicationRecord
         tags: row[tags_col_label].to_s,
         entity_type: row[entity_type_col_label].to_s,
         notes: row[notes_col_label].to_s
-      }
+      )
     end
-
-    rows.insert_all!(mapped_rows)
   end
 
   def sync_mappings
@@ -177,6 +185,39 @@ class Import < ApplicationRecord
 
     def sanitize_number(value)
       return "" if value.nil?
-      value.gsub(/[^\d.\-]/, "")
+
+      format = NUMBER_FORMATS[number_format]
+      return "" unless format
+
+      # First, normalize spaces and remove any characters that aren't numbers, delimiters, separators, or minus signs
+      sanitized = value.to_s.strip
+
+      # Handle French/Scandinavian format specially
+      if format[:delimiter] == " "
+        sanitized = sanitized.gsub(/\s+/, "") # Remove all spaces first
+      else
+        sanitized = sanitized.gsub(/[^\d#{Regexp.escape(format[:delimiter])}#{Regexp.escape(format[:separator])}\-]/, "")
+
+        # Replace delimiter with empty string
+        if format[:delimiter].present?
+          sanitized = sanitized.gsub(format[:delimiter], "")
+        end
+      end
+
+      # Replace separator with period for proper float parsing
+      if format[:separator].present?
+        sanitized = sanitized.gsub(format[:separator], ".")
+      end
+
+      # Return empty string if not a valid number
+      unless sanitized =~ /\A-?\d+\.?\d*\z/
+        return ""
+      end
+
+      sanitized
+    end
+
+    def set_default_number_format
+      self.number_format ||= "1,234.56" # Default to US/UK format
     end
 end

app/views/import/configurations/_mint_import.html.erb

@@ -16,7 +16,10 @@
     <%= form.select :signage_convention, [["Incomes are negative", "inflows_negative"], ["Incomes are positive", "inflows_positive"]], { label: true }, disabled: import.complete? %>
   </div>
 
+  <div class="flex items-center gap-2">
     <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" }, disabled: import.complete? %>
+    <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+  </div>
 
   <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %>
   <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" }, disabled: import.complete? %>

app/views/import/configurations/_trade_import.html.erb

@@ -11,7 +11,10 @@
     <%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %>
   </div>
 
+  <div class="flex items-center gap-2">
     <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+    <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+  </div>
 
   <%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %>
   <%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %>

app/views/import/configurations/_transaction_import.html.erb

@@ -11,7 +11,10 @@
     <%= form.select :signage_convention, [["Incomes are positive", "inflows_positive"], ["Incomes are negative", "inflows_negative"]], label: true %>
   </div>
 
+  <div class="flex items-center gap-2">
     <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %>
+    <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %>
+  </div>
 
   <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>
   <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>

config/locales/models/import/en.yml

@@ -7,3 +7,7 @@ en:
           attributes:
             raw_file_str:
               invalid_csv_format: is not a valid CSV format
+    attributes:
+      import:
+        currency: Currency
+        number_format: Number Format

db/migrate/20250207014022_add_number_format_to_imports.rb

@@ -0,0 +1,5 @@
+class AddNumberFormatToImports < ActiveRecord::Migration[7.2]
+  def change
+    add_column :imports, :number_format, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
+ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
     t.decimal "balance", precision: 19, scale: 4
     t.string "currency"
     t.boolean "is_active", default: true, null: false
-    t.virtual "classification", type: :string, as: "\nCASE\n    WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n    ELSE 'asset'::text\nEND", stored: true
+    t.virtual "classification", type: :string, as: "\nCASE\n    WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n    ELSE 'asset'::text\nEND", stored: true
     t.uuid "import_id"
     t.uuid "plaid_account_id"
     t.boolean "scheduled_for_deletion", default: false
@@ -414,6 +414,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_204404) do
     t.string "date_format", default: "%m/%d/%Y"
     t.string "signage_convention", default: "inflows_positive"
     t.string "error"
+    t.string "number_format"
     t.index ["family_id"], name: "index_imports_on_family_id"
   end
 

test/interfaces/import_interface_test.rb

@@ -54,4 +54,233 @@ module ImportInterfaceTest
     assert_equal "Failed to publish", import.error
     assert_equal "failed", import.status
   end
+
+  test "parses US/UK number format correctly" do
+    import = imports(:transaction)
+    import.update!(
+      number_format: "1,234.56",
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,name\n01/01/2024,\"1,234.56\",Test"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "1234.56", row.amount
+  end
+
+  test "parses European number format correctly" do
+    import = imports(:transaction)
+    import.update!(
+      number_format: "1.234,56",
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,name\n01/01/2024,\"1.234,56\",Test"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "1234.56", row.amount
+  end
+
+  test "parses French/Scandinavian number format correctly" do
+    import = imports(:transaction)
+    import.update!(
+      number_format: "1 234,56",
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      date_format: "%m/%d/%Y"
+    )
+
+    # Quote the amount field to ensure proper CSV parsing
+    csv_data = "date,amount,name\n01/01/2024,\"1 234,56\",Test"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "1234.56", row.amount
+  end
+
+  test "parses zero-decimal currency format correctly" do
+    import = imports(:transaction)
+    import.update!(
+      number_format: "1,234",
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,name\n01/01/2024,1234,Test"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "1234", row.amount
+  end
+
+  test "currency from CSV takes precedence over default" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      currency_col_label: "currency",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+    import.family.update!(currency: "USD")
+
+    csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,EUR"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "EUR", row.currency
+  end
+
+  test "uses default currency when CSV currency column is empty" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      currency_col_label: "currency",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+    import.family.update!(currency: "USD")
+
+    csv_data = "date,amount,name,currency\n01/01/2024,123.45,Test,"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "USD", row.currency
+  end
+
+  test "uses default currency when CSV has no currency column" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+    import.family.update!(currency: "USD")
+
+    csv_data = "date,amount,name\n01/01/2024,123.45,Test"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "USD", row.currency
+  end
+
+  test "generates rows with all optional fields" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      account_col_label: "account",
+      category_col_label: "category",
+      tags_col_label: "tags",
+      notes_col_label: "notes",
+      currency_col_label: "currency",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,name,account,category,tags,notes,currency\n" \
+               "01/01/2024,1234.56,Salary,Bank Account,Income,\"monthly,salary\",Salary payment,EUR"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "01/01/2024", row.date
+    assert_equal "1234.56", row.amount
+    assert_equal "Salary", row.name
+    assert_equal "Bank Account", row.account
+    assert_equal "Income", row.category
+    assert_equal "monthly,salary", row.tags
+    assert_equal "Salary payment", row.notes
+    assert_equal "EUR", row.currency
+  end
+
+  test "generates rows with minimal required fields" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount\n01/01/2024,1234.56"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "01/01/2024", row.date
+    assert_equal "1234.56", row.amount
+    assert_equal "Imported item", row.name # Default name
+    assert_equal import.family.currency, row.currency # Default currency
+  end
+
+  test "handles empty values in optional fields" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      name_col_label: "name",
+      category_col_label: "category",
+      tags_col_label: "tags",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,name,category,tags\n01/01/2024,1234.56,,,"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "01/01/2024", row.date
+    assert_equal "1234.56", row.amount
+    assert_equal "Imported item", row.name # Falls back to default
+    assert_equal "", row.category
+    assert_equal "", row.tags
+  end
+
+  test "handles trade-specific fields" do
+    import = imports(:transaction)
+    import.update!(
+      amount_col_label: "amount",
+      date_col_label: "date",
+      qty_col_label: "quantity",
+      ticker_col_label: "symbol",
+      price_col_label: "price",
+      number_format: "1,234.56",
+      date_format: "%m/%d/%Y"
+    )
+
+    csv_data = "date,amount,quantity,symbol,price\n01/01/2024,1234.56,10,AAPL,123.456"
+    import.update!(raw_file_str: csv_data)
+    import.generate_rows_from_csv
+
+    row = import.rows.first
+    assert_equal "10", row.qty
+    assert_equal "AAPL", row.ticker
+    assert_equal "123.456", row.price
+  end
 end

test/models/transaction_import_test.rb

@@ -28,14 +28,19 @@ class TransactionImportTest < ActiveSupport::TestCase
   end
 
   test "imports transactions, categories, tags, and accounts" do
-    import = <<-CSV
+    import = <<~CSV
       date,name,amount,category,tags,account,notes
       01/01/2024,Txn1,100,TestCategory1,TestTag1,TestAccount1,notes1
       01/02/2024,Txn2,200,TestCategory2,TestTag1|TestTag2,TestAccount2,notes2
       01/03/2024,Txn3,300,,,,notes3
     CSV
 
-    @import.update!(raw_file_str: import)
+    @import.update!(
+      raw_file_str: import,
+      date_col_label: "date",
+      amount_col_label: "amount",
+      date_format: "%m/%d/%Y"
+    )
 
     @import.generate_rows_from_csv