Allow CSV file upload in import flow (#986)

* Add .tool-versions to gitignore

* Add dropzone js for drag and drop file uploads

* UI for csv file uploads for import

* dropzone controller and use lucide_icon instead of svg

* Preview for file chosen

* File upload

* Remove dropzone

* Normalize I18n keys and fix lint issues

* Add system tests

* Cleanup

* Remove unwanted

.gitignore

@@ -51,6 +51,9 @@
 # Ignore .devcontainer files
 compose-dev.yaml
 
+# Ignore asdf ruby version file
+.tool-versions
+
 # Ignore GCP keyfile
 gcp-storage-keyfile.json
 

app/controllers/imports_controller.rb

@@ -38,6 +38,21 @@ class ImportsController < ApplicationController
   def load
   end
 
+  def upload_csv
+    begin
+      @import.raw_csv_str = import_params[:raw_csv_str].read
+    rescue NoMethodError
+      flash.now[:error] = "Please select a file to upload"
+      render :load, status: :unprocessable_entity and return
+    end
+    if @import.save
+      redirect_to configure_import_path(@import), notice: t(".import_loaded")
+    else
+      flash.now[:error] = @import.errors.full_messages.to_sentence
+      render :load, status: :unprocessable_entity
+    end
+  end
+
   def load_csv
     if @import.update(import_params)
       redirect_to configure_import_path(@import), notice: t(".import_loaded")

app/javascript/controllers/csv_upload_controller.js

@@ -0,0 +1,94 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["input", "preview", "submit", "filename", "filesize"]
+
+  connect() {
+    this.submitTarget.disabled = true
+  }
+
+  addFile(event) {
+    const file = event.target.files[0]
+    this._fileAdded(file)
+  }
+
+  dragover(event) {
+    event.preventDefault()
+    event.stopPropagation()
+    event.currentTarget.classList.add("bg-gray-100")
+  }
+
+  dragleave(event) {
+    event.preventDefault()
+    event.stopPropagation()
+    event.currentTarget.classList.remove("bg-gray-100")
+  }
+
+  drop(event) {
+    event.preventDefault()
+    event.stopPropagation()
+    event.currentTarget.classList.remove("bg-gray-100")
+
+    const file = event.dataTransfer.files[0]
+    if (file && this._isCSVFile(file)) {
+      this._setFileInput(file);
+      this._fileAdded(file)
+    } else {
+      this.previewTarget.classList.add("text-red-500")
+      this.previewTarget.textContent = "Only CSV files are allowed."
+    }
+  }
+
+  // Private
+
+  _fetchFileSize(size) {
+    let fileSize = '';
+    if (size < 1024 * 1024) {
+      fileSize = (size / 1024).toFixed(2) + ' KB'; // Convert bytes to KB
+    } else {
+      fileSize = (size / (1024 * 1024)).toFixed(2) + ' MB'; // Convert bytes to MB
+    }
+    return fileSize;
+  }
+
+  _fileAdded(file) {
+    const fileSizeLimit = 5 * 1024 * 1024 // 5MB
+
+    if (file) {
+      if (file.size > fileSizeLimit) {
+        this.previewTarget.classList.add("text-red-500")
+        this.previewTarget.textContent = "File size exceeds the limit of 5MB"
+        return
+      }
+
+      this.submitTarget.classList.remove([
+        "bg-alpha-black-25",
+        "text-gray",
+        "cursor-not-allowed",
+      ]);
+      this.submitTarget.classList.add(
+        "bg-gray-900",
+        "text-white",
+        "cursor-pointer",
+      );
+      this.submitTarget.disabled = false;
+      this.previewTarget.innerHTML = document.querySelector("#template-preview").innerHTML;
+      this.previewTarget.classList.remove("text-red-500")
+      this.previewTarget.classList.add("text-gray-900")
+      this.filenameTarget.textContent = file.name;
+      this.filesizeTarget.textContent = this._fetchFileSize(file.size);
+    }
+  }
+
+  _isCSVFile(file) {
+    const acceptedTypes = ["text/csv", "application/csv", ".csv"]
+    const extension = file.name.split('.').pop().toLowerCase()
+    return acceptedTypes.includes(file.type) || extension === "csv"
+  }
+
+  _setFileInput(file) {
+    const dataTransfer = new DataTransfer();
+    dataTransfer.items.add(file);
+    this.inputTarget.files = dataTransfer.files;
+  }
+}

app/views/imports/_csv_paste.html.erb

@@ -0,0 +1,27 @@
+<%= form_with model: @import, url: load_import_path(@import) do |form| %>
+  <div>
+    <%= form.text_area :raw_csv_str,
+                       rows: 10,
+                       required: true,
+                       placeholder: "Paste your CSV file contents here",
+                       class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>
+  </div>
+
+  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+<% end %>
+
+<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">
+  <div class="text-gray-500 p-2 mb-2">
+    <div class="flex gap-2 mb-2">
+      <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
+      <p class="text-sm"><%= t(".instructions") %></p>
+    </div>
+
+    <ul class="list-disc text-sm pl-10">
+      <li><%= t(".requirement1") %></li>
+      <li><%= t(".requirement2") %></li>
+      <li><%= t(".requirement3") %></li>
+    </ul>
+  </div>
+  <%= render partial: "imports/sample_table" %>
+</div>

app/views/imports/_csv_upload.html.erb

@@ -0,0 +1,39 @@
+<%= form_with model: @import, url: upload_import_path(@import), class: "dropzone", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>
+  <div class="flex items-center justify-center w-full">
+    <label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">
+      <div class="flex flex-col items-center justify-center pt-5 pb-6">
+        <%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>
+        <%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>
+        <p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p>
+        <p class="text-xs text-gray-500">CSV (Max. 5MB)</p>
+        <div class="csv-preview" data-csv-upload-target="preview"></div>
+      </div>
+    </label>
+  </div>
+  <%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
+<% end %>
+
+<div id="template-preview" class="hidden">
+  <div class="flex flex-col items-center justify-center">
+    <%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %>
+    <div class="flex flex-row items-center justify-center gap-0.5">
+      <div><span data-csv-upload-target="filename"></span></div>
+      <div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>
+    </div>
+  </div>
+</div>
+
+<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">
+  <div class="text-gray-500 p-2 mb-2">
+    <div class="flex gap-2 mb-2">
+      <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
+      <p class="text-sm">
+        <%= t(".instructions") %>
+        <span class="text-black underline">
+          <%= link_to "download this template", "/transactions.csv", download: "" %>
+        </span>
+      </p>
+    </div>
+  </div>
+  <%= render partial: "imports/sample_table" %>
+</div>

app/views/imports/load.html.erb

@@ -8,33 +8,18 @@
     <p class="text-gray-500 text-sm"><%= t(".description") %></p>
   </div>
 
-  <%= form_with model: @import, url: load_import_path(@import) do |form| %>
+  <div data-controller="tabs" data-tabs-active-class="bg-white" data-tabs-default-tab-value="csv-upload-tab">
-    <div>
+    <div class="flex justify-center mb-4">
-      <%= form.text_area :raw_csv_str,
+      <div class="bg-gray-50 rounded-lg inline-flex p-1 space-x-2 text-sm text-gray-900 font-medium">
-                         rows: 10,
+        <button data-id="csv-upload-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Upload CSV</button>
-                         required: true,
+        <button data-id="csv-paste-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Copy & Paste</button>
-                         placeholder: "Paste your CSV file contents here",
-                         class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %>
-    </div>
-
-    <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium cursor-pointer hover:bg-gray-700", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
-  <% end %>
-
-  <div class="bg-alpha-black-25 rounded-xl p-1">
-    <div class="text-gray-500 p-2 mb-2">
-      <div class="flex gap-2 mb-2">
-        <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
-        <p class="text-sm"><%= t(".instructions") %></p>
       </div>
-
-      <ul class="list-disc text-sm pl-10">
-        <li><%= t(".requirement1") %></li>
-        <li><%= t(".requirement2") %></li>
-        <li><%= t(".requirement3") %></li>
-      </ul>
     </div>
-
+    <div data-tabs-target="tab" id="csv-upload-tab">
-    <%= render partial: "imports/sample_table" %>
+      <%= render partial: "imports/csv_upload", locals: { import: @import } %>
-
+    </div>
+    <div data-tabs-target="tab" id="csv-paste-tab" class="hidden">
+      <%= render partial: "imports/csv_paste", locals: { import: @import } %>
+    </div>
   </div>
 </div>

config/locales/views/imports/en.yml

@@ -27,6 +27,25 @@ en:
       invalid_data: You have invalid data, please fix before continuing
     create:
       import_created: Import created
+    csv_paste:
+      confirm_accept: Yep, start over!
+      confirm_body: This will reset your import. Any changes you have made to the
+        CSV will be erased.
+      confirm_title: Are you sure?
+      instructions: Your CSV should have the following columns and formats for the
+        best import experience.
+      next: Next
+      requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)
+      requirement2: Negative transaction is an "outflow" (expense), positive is an
+        "inflow" (income)
+      requirement3: Can have 0 or more tags separated by |
+    csv_upload:
+      confirm_accept: Yep, start over!
+      confirm_body: This will reset your import. Any changes you have made to the
+        CSV will be erased.
+      confirm_title: Are you sure?
+      instructions: The csv file must be in the format below. You can also reuse and
+      next: Next
     destroy:
       import_destroyed: Import destroyed
     edit:
@@ -56,20 +75,9 @@ en:
       new: New import
       title: Imports
     load:
-      confirm_accept: Yep, start over!
-      confirm_body: This will reset your import. Any changes you have made to the
-        CSV will be erased.
-      confirm_title: Are you sure?
       description: Create a spreadsheet or upload an exported CSV from your financial
         institution.
-      instructions: Your CSV should have the following columns and formats for the
-        best import experience.
       load_title: Load import
-      next: Next
-      requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD)
-      requirement2: Negative transaction is an "outflow" (expense), positive is an
-        "inflow" (income)
-      requirement3: Can have 0 or more tags separated by |
       subtitle: Import your transactions
     load_csv:
       import_loaded: Import CSV loaded
@@ -95,3 +103,5 @@ en:
       import_updated: Import updated
     update_mappings:
       column_mappings_saved: Column mappings saved
+    upload_csv:
+      import_loaded: CSV File loaded

config/routes.rb

@@ -25,6 +25,7 @@ Rails.application.routes.draw do
     member do
       get "load"
       patch "load" => "imports#load_csv"
+      patch "upload" => "imports#upload_csv"
 
       get "configure"
       patch "configure" => "imports#update_mappings"

public/transactions.csv

@@ -0,0 +1,4 @@
+date,name,category,tags,amount
+2024-01-01,Amazon,Shopping,Tag1|Tag2,-24.99
+2024-03-01,Spotify,,,-16.32
+2023-01-06,Acme,Income,Tag3,151.22

test/controllers/imports_controller_test.rb

@@ -65,6 +65,18 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
     assert_equal "Import CSV loaded", flash[:notice]
   end
 
+  test "should upload CSV file if valid" do
+    Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
+      CSV.open(temp, "wb", headers: true) do |csv|
+        valid_csv_str.split("\n").each { |row| csv << row.split(",") }
+      end
+
+      patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+      assert_redirected_to configure_import_path(@empty_import)
+      assert_equal "CSV File loaded", flash[:notice]
+    end
+  end
+
   test "should flash error message if invalid CSV input" do
     patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }
 
@@ -72,6 +84,23 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
     assert_equal "Raw csv str is not a valid CSV format", flash[:error]
   end
 
+  test "should flash error message if invalid CSV file upload" do
+    Tempfile.open([ "transactions.csv", ".csv" ]) do |temp|
+      temp.write(malformed_csv_str)
+      temp.rewind
+
+      patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
+      assert_response :unprocessable_entity
+      assert_equal "Raw csv str is not a valid CSV format", flash[:error]
+    end
+  end
+
+  test "should flash error message if no fileprovided for upload" do
+    patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }
+    assert_response :unprocessable_entity
+    assert_equal "Please select a file to upload", flash[:error]
+  end
+
   test "should get configure" do
     get configure_import_url(@loaded_import)
     assert_response :success

test/fixtures/files/transactions.csv

@@ -0,0 +1,3 @@
+date,Custom Name Column,category,amount
+invalid_date,Starbucks drink,Food,-20.50
+2024-01-01,Amazon purchase,Shopping,-89.50

test/support/import_test_helper.rb

@@ -1,6 +1,6 @@
 module ImportTestHelper
   def valid_csv_str
-    <<-ROWS
+    <<~ROWS
       date,name,category,tags,amount
       2024-01-01,Starbucks drink,Food & Drink,Tag1|Tag2,-8.55
       2024-01-01,Etsy,Shopping,Tag1,-80.98
@@ -10,14 +10,14 @@ module ImportTestHelper
   end
 
   def valid_csv_with_invalid_values
-    <<-ROWS
+    <<~ROWS
       date,name,category,tags,amount
       invalid_date,Starbucks drink,Food,,invalid_amount
     ROWS
   end
 
   def valid_csv_with_missing_data
-    <<-ROWS
+    <<~ROWS
       date,name,category,"optional id",amount
       2024-01-01,Drink,Food,1234,-200
       2024-01-02,,,,-100
@@ -25,7 +25,7 @@ module ImportTestHelper
   end
 
   def malformed_csv_str
-    <<-ROWS
+    <<~ROWS
       name,age
       "John Doe,23
       "Jane Doe",25

test/system/imports_test.rb

@@ -51,6 +51,8 @@ class ImportsTest < ApplicationSystemTestCase
 
     click_button "Next"
 
+    click_button "Copy & Paste"
+
     # 2) Load Step
     assert_selector "h1", text: "Load import"
 
@@ -94,6 +96,60 @@ class ImportsTest < ApplicationSystemTestCase
     assert_selector "h1", text: "Imports"
   end
 
+  test "can perform import by CSV upload" do
+    trigger_import_from_settings
+    verify_import_modal
+
+    within "#modal" do
+      click_link "New import from CSV"
+    end
+
+    # 1) Create import step
+    assert_selector "h1", text: "New import"
+
+    within "form" do
+      select "Checking Account", from: "import_account_id"
+    end
+
+    click_button "Next"
+
+    click_button "Upload CSV"
+
+    find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")
+    assert_selector "div.csv-preview", text: "transactions.csv"
+
+    click_button "Next"
+
+    # 3) Configure step
+    assert_selector "h1", text: "Configure import"
+
+    within "form" do
+      select "Custom Name Column", from: "import_column_mappings_name"
+    end
+
+    click_button "Next"
+
+    # 4) Clean step
+    assert_selector "h1", text: "Clean import"
+
+    # We have an invalid value, so user cannot click next yet
+    assert_no_text "Next"
+
+    # Replace invalid date with valid date
+    fill_in "cell-0-0", with: "2024-01-02"
+
+    # Trigger blur event so value saves
+    find("body").click
+
+    click_link "Next"
+
+    # 5) Confirm step
+    assert_selector "h1", text: "Confirm import"
+    assert_selector "#new_account_entry", count: 2
+    click_button "Import 2 transactions"
+    assert_selector "h1", text: "Imports"
+  end
+
   private
 
     def trigger_import_from_settings