Refactor: Allow other import files (#1099)

* Rename stimulus controller

* feature: rename raw_csv_str to raw_file_str

app/controllers/imports_controller.rb

@@ -40,7 +40,7 @@ class ImportsController < ApplicationController
 
   def upload_csv
     begin
-      @import.raw_csv_str = import_params[:raw_csv_str].read
+      @import.raw_file_str = import_params[:raw_file_str].read
     rescue NoMethodError
       flash.now[:alert] = "Please select a file to upload"
       render :load, status: :unprocessable_entity and return
@@ -113,6 +113,6 @@ class ImportsController < ApplicationController
     end
 
     def import_params(permitted_mappings = nil)
-      params.require(:import).permit(:raw_csv_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
+      params.require(:import).permit(:raw_file_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
     end
 end

app/javascript/controllers/import_upload_controller.js

@@ -2,6 +2,11 @@ import { Controller } from "@hotwired/stimulus"
 
 export default class extends Controller {
   static targets = ["input", "preview", "submit", "filename", "filesize"]
+  static values = {
+    acceptedTypes: Array, // ["text/csv", "application/csv", ".csv"]
+    acceptedExtension: String, // "csv"
+    unacceptableTypeLabel: String, // "Only CSV files are allowed."
+  };
 
   connect() {
     this.submitTarget.disabled = true
@@ -30,15 +35,19 @@ export default class extends Controller {
     event.currentTarget.classList.remove("bg-gray-100")
 
     const file = event.dataTransfer.files[0]
-    if (file && this._isCSVFile(file)) {
+    if (file && this._formatAcceptable(file)) {
       this._setFileInput(file);
       this._fileAdded(file)
     } else {
       this.previewTarget.classList.add("text-red-500")
-      this.previewTarget.textContent = "Only CSV files are allowed."
+      this.previewTarget.textContent = this.unacceptableTypeLabelValue
     }
   }
 
+  click() {
+    this.inputTarget.click();
+  }
+
   // Private
 
   _fetchFileSize(size) {
@@ -57,7 +66,7 @@ export default class extends Controller {
     if (file) {
       if (file.size > fileSizeLimit) {
         this.previewTarget.classList.add("text-red-500")
-        this.previewTarget.textContent = "File size exceeds the limit of 5MB"
+        this.previewTarget.textContent = this.unacceptableTypeLabelValue
         return
       }
 
@@ -80,10 +89,9 @@ export default class extends Controller {
     }
   }
 
-  _isCSVFile(file) {
+  _formatAcceptable(file) {
-    const acceptedTypes = ["text/csv", "application/csv", ".csv"]
     const extension = file.name.split('.').pop().toLowerCase()
-    return acceptedTypes.includes(file.type) || extension === "csv"
+    return this.acceptedTypesValue.includes(file.type) || extension === this.acceptedExtensionValue
   }
 
   _setFileInput(file) {

app/models/import.rb

@@ -1,7 +1,7 @@
 class Import < ApplicationRecord
   belongs_to :account
 
-  validate :raw_csv_must_be_parsable
+  validate :raw_file_must_be_parsable
   validates :col_sep, inclusion: { in: Csv::COL_SEP_LIST }
 
   before_save :initialize_csv, if: :should_initialize_csv?
@@ -19,7 +19,7 @@ class Import < ApplicationRecord
   end
 
   def loaded?
-    raw_csv_str.present?
+    raw_file_str.present?
   end
 
   def configured?
@@ -88,16 +88,16 @@ class Import < ApplicationRecord
     end
 
     def get_raw_csv
-      return nil if raw_csv_str.nil?
+      return nil if raw_file_str.nil?
-      Import::Csv.new(raw_csv_str, col_sep:)
+      Import::Csv.new(raw_file_str, col_sep:)
     end
 
     def should_initialize_csv?
-      raw_csv_str_changed? || column_mappings_changed?
+      raw_file_str_changed? || column_mappings_changed?
     end
 
     def initialize_csv
-      generated_csv = generate_normalized_csv(raw_csv_str)
+      generated_csv = generate_normalized_csv(raw_file_str)
       self.normalized_csv_str = generated_csv.table.to_s
     end
 
@@ -175,12 +175,12 @@ class Import < ApplicationRecord
       end
     end
 
-    def raw_csv_must_be_parsable
+    def raw_file_must_be_parsable
       begin
-        CSV.parse(raw_csv_str || "", col_sep:)
+        CSV.parse(raw_file_str || "", col_sep:)
       rescue CSV::MalformedCSVError
-        # i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_csv_str.invalid_csv_format')
+        # i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_file_str.invalid_csv_format')
-        errors.add(:raw_csv_str, :invalid_csv_format)
+        errors.add(:raw_file_str, :invalid_csv_format)
       end
     end
 end

app/models/import/csv.rb

@@ -11,8 +11,8 @@ class Import::Csv
     )
   end
 
-  def self.create_with_field_mappings(raw_csv_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
+  def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
-    raw_csv = self.parse_csv(raw_csv_str, col_sep:)
+    raw_csv = self.parse_csv(raw_file_str, col_sep:)
 
     generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|
       raw_csv.each do |row|

app/views/imports/_csv_paste.html.erb

@@ -1,11 +1,11 @@
 <%= styled_form_with model: @import, url: load_import_path(@import), class: "space-y-4" do |form| %>
-    <%= form.text_area :raw_csv_str,
+    <%= form.text_area :raw_file_str,
                        rows: 10,
                        required: true,
                        placeholder: "Paste your CSV file contents here",
                        class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400" %>
 
-  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
 <% end %>
 
 <div class="bg-alpha-black-25 rounded-xl p-1 mt-5">

app/views/imports/_csv_upload.html.erb

@@ -1,24 +1,24 @@
-<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>
+<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "import-upload", import_upload_accepted_types_value: ["text/csv", "application/csv", ".csv"], import_upload_extension_value: "csv", import_upload_unacceptable_type_label_value: t(".allowed_filetypes") }, method: :patch, multipart: true do |form| %>
   <div class="flex items-center justify-center w-full">
-    <label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">
+    <label for="import_raw_file_str" class="raw-file-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->import-upload#dragover dragleave->import-upload#dragleave drop->import-upload#drop click->import-upload#click">
       <div class="flex flex-col items-center justify-center pt-5 pb-6">
         <%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>
-        <%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>
+        <%= form.file_field :raw_file_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { import_upload_target: "input", action: "change->import-upload#addFile" } %>
         <p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p>
         <p class="text-xs text-gray-500">CSV (Max. 5MB)</p>
-        <div class="csv-preview" data-csv-upload-target="preview"></div>
+        <div class="csv-preview" data-import-upload-target="preview"></div>
       </div>
     </label>
   </div>
-  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { import_upload_target: "submit", turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
 <% end %>
 
 <div id="template-preview" class="hidden">
   <div class="flex flex-col items-center justify-center">
     <%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %>
     <div class="flex flex-row items-center justify-center gap-0.5">
-      <div><span data-csv-upload-target="filename"></span></div>
+      <div><span data-import-upload-target="filename"></span></div>
-      <div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>
+      <div><span data-import-upload-target="filesize" class="font-semibold"></span></div>
     </div>
   </div>
 </div>

config/locales/models/import/en.yml

@@ -5,5 +5,5 @@ en:
       models:
         import:
           attributes:
-            raw_csv_str:
+            raw_file_str:
               invalid_csv_format: is not a valid CSV format

config/locales/views/imports/en.yml

@@ -40,6 +40,7 @@ en:
         "inflow" (income)
       requirement3: Can have 0 or more tags separated by |
     csv_upload:
+      allowed_filetypes: Only CSV files are allowed.
       confirm_accept: Yep, start over!
       confirm_body: This will reset your import. Any changes you have made to the
         CSV will be erased.

db/migrate/20240817144454_rename_import_raw_csv_str_to_raw_file_str.rb

@@ -0,0 +1,5 @@
+class RenameImportRawCsvStrToRawFileStr < ActiveRecord::Migration[7.2]
+  def change
+    rename_column :imports, :raw_csv_str, :raw_file_str
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do
+ActiveRecord::Schema[7.2].define(version: 2024_08_17_144454) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -289,7 +289,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do
     t.uuid "account_id", null: false
     t.jsonb "column_mappings"
     t.enum "status", default: "pending", enum_type: "import_status"
-    t.string "raw_csv_str"
+    t.string "raw_file_str"
     t.string "normalized_csv_str"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false

test/controllers/imports_controller_test.rb

@@ -8,7 +8,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
     @empty_import = imports(:empty_import)
 
     @loaded_import = @empty_import.dup
-    @loaded_import.update! raw_csv_str: valid_csv_str
+    @loaded_import.update! raw_file_str: valid_csv_str
 
     @completed_import = imports(:completed_import)
   end
@@ -59,7 +59,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
   end
 
   test "should save raw CSV if valid" do
-    patch load_import_url(@empty_import), params: { import: { raw_csv_str: valid_csv_str } }
+    patch load_import_url(@empty_import), params: { import: { raw_file_str: valid_csv_str } }
 
     assert_redirected_to configure_import_path(@empty_import)
     assert_equal "Import CSV loaded", flash[:notice]
@@ -71,17 +71,17 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
         valid_csv_str.split("\n").each { |row| csv << row.split(",") }
       end
 
-      patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+      patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
       assert_redirected_to configure_import_path(@empty_import)
       assert_equal "CSV File loaded", flash[:notice]
     end
   end
 
   test "should flash error message if invalid CSV input" do
-    patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }
+    patch load_import_url(@empty_import), params: { import: { raw_file_str: malformed_csv_str } }
 
     assert_response :unprocessable_entity
-    assert_equal "Raw csv str is not a valid CSV format", flash[:alert]
+    assert_equal "Raw file str is not a valid CSV format", flash[:alert]
   end
 
   test "should flash error message if invalid CSV file upload" do
@@ -89,14 +89,14 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
       temp.write(malformed_csv_str)
       temp.rewind
 
-      patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+      patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
       assert_response :unprocessable_entity
-      assert_equal "Raw csv str is not a valid CSV format", flash[:alert]
+      assert_equal "Raw file str is not a valid CSV format", flash[:alert]
     end
   end
 
   test "should flash error message if no fileprovided for upload" do
-    patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }
+    patch upload_import_url(@empty_import), params: { import: { raw_file_str: nil } }
     assert_response :unprocessable_entity
     assert_equal "Please select a file to upload", flash[:alert]
   end
@@ -158,7 +158,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
   end
 
   test "should redirect back to clean if data is invalid" do
-    @empty_import.update! raw_csv_str: valid_csv_with_invalid_values
+    @empty_import.update! raw_file_str: valid_csv_with_invalid_values
 
     get confirm_import_url(@empty_import)
     assert_equal "You have invalid data, please fix before continuing", flash[:alert]

test/fixtures/imports.yml

@@ -9,7 +9,7 @@ completed_import:
     name: name
     category: category
     amount: amount
-  raw_csv_str: |
+  raw_file_str: |
     date,name,category,tags,amount
     2024-01-01,Starbucks drink,Food & Drink,Test Tag,-20
   normalized_csv_str: |

test/jobs/import_job_test.rb

@@ -5,7 +5,7 @@ class ImportJobTest < ActiveJob::TestCase
 
   test "import is published" do
     import = imports(:empty_import)
-    import.update! raw_csv_str: valid_csv_str
+    import.update! raw_file_str: valid_csv_str
 
     assert import.pending?
 

test/models/import/csv_test.rb

@@ -72,7 +72,7 @@ class Import::CsvTest < ActiveSupport::TestCase
 
     fields = [ date_field, name_field ]
 
-    raw_csv_str = <<-ROWS
+    raw_file_str = <<-ROWS
       date,Custom Field Header,extra_field
       invalid_date_value,Starbucks drink,Food
       2024-01-02,Amazon stuff,Shopping
@@ -82,7 +82,7 @@ class Import::CsvTest < ActiveSupport::TestCase
       "name" => "Custom Field Header"
     }
 
-    csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings)
+    csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings)
 
     assert_equal %w[ date name ], csv.table.headers
     assert_equal 2, csv.table.size
@@ -101,7 +101,7 @@ class Import::CsvTest < ActiveSupport::TestCase
 
     fields = [ date_field, name_field ]
 
-    raw_csv_str = <<-ROWS
+    raw_file_str = <<-ROWS
       date;Custom Field Header;extra_field
       invalid_date_value;Starbucks drink;Food
       2024-01-02;Amazon stuff;Shopping
@@ -111,7 +111,7 @@ class Import::CsvTest < ActiveSupport::TestCase
       "name" => "Custom Field Header"
     }
 
-    csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings, ";")
+    csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings, ";")
 
     assert_equal %w[ date name ], csv.table.headers
     assert_equal 2, csv.table.size

test/models/import_test.rb

@@ -7,7 +7,7 @@ class ImportTest < ActiveSupport::TestCase
     @empty_import = imports(:empty_import)
 
     @loaded_import = @empty_import.dup
-    @loaded_import.update! raw_csv_str: valid_csv_str
+    @loaded_import.update! raw_file_str: valid_csv_str
   end
 
   test "validates the correct col_sep" do
@@ -26,17 +26,17 @@ class ImportTest < ActiveSupport::TestCase
   end
 
   test "raw csv input must conform to csv spec" do
-    @empty_import.raw_csv_str = malformed_csv_str
+    @empty_import.raw_file_str = malformed_csv_str
     assert_not @empty_import.valid?
 
-    @empty_import.raw_csv_str = valid_csv_str
+    @empty_import.raw_file_str = valid_csv_str
     assert @empty_import.valid?
   end
 
   test "can update csv value without affecting raw input" do
     assert_equal "Starbucks drink", @loaded_import.csv.table[0][1]
 
-    prior_raw_csv_str_value = @loaded_import.raw_csv_str
+    prior_raw_file_str_value = @loaded_import.raw_file_str
     prior_normalized_csv_str_value = @loaded_import.normalized_csv_str
 
     @loaded_import.update_csv! \
@@ -45,7 +45,7 @@ class ImportTest < ActiveSupport::TestCase
       value: "new_category"
 
     assert_equal "new_category", @loaded_import.csv.table[0][1]
-    assert_equal prior_raw_csv_str_value, @loaded_import.raw_csv_str
+    assert_equal prior_raw_file_str_value, @loaded_import.raw_file_str
     assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str
   end
 
@@ -74,7 +74,7 @@ class ImportTest < ActiveSupport::TestCase
   end
 
   test "publishes a valid import with missing data" do
-    @empty_import.update! raw_csv_str: valid_csv_with_missing_data
+    @empty_import.update! raw_file_str: valid_csv_with_missing_data
     assert_difference -> { Category.count } => 1,
                       -> { Account::Transaction.count } => 2,
                       -> { Account::Entry.count } => 2 do
@@ -89,7 +89,7 @@ class ImportTest < ActiveSupport::TestCase
   end
 
   test "failed publish results in error status" do
-    @empty_import.update! raw_csv_str: valid_csv_with_invalid_values
+    @empty_import.update! raw_file_str: valid_csv_with_invalid_values
 
     assert_difference "Account::Transaction.count", 0 do
       @empty_import.publish
@@ -102,7 +102,7 @@ class ImportTest < ActiveSupport::TestCase
   test "can create transactions from csv with custom column separator" do
     loaded_import = @empty_import.dup
 
-    loaded_import.update! raw_csv_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
+    loaded_import.update! raw_file_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
     transactions = loaded_import.dry_run
 
     assert_equal 4, transactions.count

test/system/imports_test.rb

@@ -57,7 +57,7 @@ class ImportsTest < ApplicationSystemTestCase
     assert_selector "h1", text: "Load import"
 
     within "form" do
-      fill_in "import_raw_csv_str", with: <<-ROWS
+      fill_in "import_raw_file_str", with: <<-ROWS
         date,Custom Name Column,category,amount
         invalid_date,Starbucks drink,Food,-20.50
         2024-01-01,Amazon purchase,Shopping,-89.50
@@ -115,7 +115,7 @@ class ImportsTest < ApplicationSystemTestCase
 
     click_button "Upload CSV"
 
-    find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")
+    find(".raw-file-drop-box").drop File.join(file_fixture_path, "transactions.csv")
     assert_selector "div.csv-preview", text: "transactions.csv"
 
     click_button "Next"