CSV Imports Overhaul (Transactions, Trades, Accounts, and Mint import support) (#1209)

* Remove stale 1.0 import logic and model

* Fresh start

* Checkpoint before removing nav

* First working prototype

* Add trade, account, and mint import flows

* Basic working version with tests

* System tests for each import type

* Clean up mappings flow

* Clean up PR, refactor stale code, tests

* Add back row validations

* Row validations

* Fix import job test

* Fix import navigation

* Fix mint import configuration form

* Currency preset for new accounts

app/views/import/cleans/show.html.erb

@@ -0,0 +1,59 @@
+<%= content_for :header_nav do %>
+  <%= render "imports/nav", import: @import %>
+<% end %>
+
+<%= content_for :previous_path, import_configuration_path(@import) %>
+
+<div class="space-y-4 mx-auto max-w-screen-lg">
+  <div class="text-center space-y-2 max-w-[400px] mx-auto mb-4">
+    <h2 class="text-3xl text-gray-900 font-medium"><%= t(".title") %></h2>
+    <p class="text-gray-500 text-sm"><%= t(".description") %></p>
+  </div>
+
+  <% if @import.cleaned? %>
+    <div class="bg-white border border-alpha-black-100 rounded-lg p-3 flex items-center justify-between">
+      <div class="flex items-center gap-2">
+        <%= lucide_icon "check-circle", class: "w-4 h-4 text-green-500" %>
+        <p class="text-green-500">Your data has been cleaned</p>
+      </div>
+
+      <%= link_to "Next step", import_confirm_path(@import), class: "btn btn--primary" %>
+    </div>
+  <% else %>
+    <div class="bg-white border border-alpha-black-100 rounded-lg p-3 flex items-center justify-between">
+      <div class="flex items-center gap-2">
+        <%= lucide_icon "alert-triangle", class: "w-4 h-4 text-red-500" %>
+        <p class="text-red-500">You have errors in your data</p>
+      </div>
+
+      <div class="flex justify-center">
+        <div class="bg-gray-50 rounded-lg inline-flex p-1 space-x-2 text-sm text-gray-900 font-medium">
+          <%= link_to "All rows", import_clean_path(@import, per_page: params[:per_page], view: "all"), class: "p-2 rounded-lg #{params[:view] != 'errors' ? 'bg-white' : ''}" %>
+          <%= link_to "Error rows", import_clean_path(@import, per_page: params[:per_page], view: "errors"), class: "p-2 rounded-lg #{params[:view] == 'errors' ? 'bg-white' : ''}" %>
+        </div>
+      </div>
+    </div>
+  <% end %>
+
+  <div class="pb-12">
+    <div class="bg-gray-25 rounded-xl p-1 mb-6">
+      <div style="grid-template-columns: repeat(<%= @import.column_keys.count %>, 1fr)" class="grid items-center uppercase text-xs font-medium text-gray-500 py-3">
+        <% @import.column_keys.each do |key| %>
+          <div class="px-5"><%= import_col_label(key) %></div>
+        <% end %>
+      </div>
+
+      <div class="bg-white border border-alpha-black-200 rounded-xl shadow-xs divide-y divide-alpha-black-200">
+        <% @rows.each do |row| %>
+          <%= render "import/rows/form", row: row %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+
+  <div class="fixed bottom-0 left-1/2 -translate-x-1/2 w-full p-12">
+    <div class="border border-alpha-black-100 rounded-lg p-3 max-w-screen-sm mx-auto bg-white shadow-xs">
+      <%= render "application/pagination", pagy: @pagy %>
+    </div>
+  </div>
+</div>

app/views/import/configurations/_account_import.html.erb

@@ -0,0 +1,9 @@
+<%# locals: (import:) %>
+
+<%= styled_form_with model: @import, url: import_configuration_path(@import), scope: :import, method: :patch, class: "space-y-2" do |form| %>
+  <%= form.select :entity_type_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Entity Type" } %>
+  <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>
+  <%= form.select :amount_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Balance" } %>
+
+  <%= form.submit "Apply configuration", class: "w-full btn btn--primary", disabled: import.complete? %>
+<% end %>

app/views/import/configurations/_mint_import.html.erb

@@ -0,0 +1,25 @@
+<%# locals: (import:) %>
+
+<div class="flex items-center justify-between border border-alpha-black-200 rounded-lg bg-green-500/5 p-5 gap-4">
+  <%= lucide_icon("check-circle", class: "w-5 h-5 shrink-0 text-green-500") %>
+  <p class="text-sm text-gray-900 italic">We have pre-configured your Mint import for you. Please proceed to the next step.</p>
+</div>
+
+<%= styled_form_with model: @import, url: import_configuration_path(@import), scope: :import, method: :patch, class: "space-y-2" do |form| %>
+  <div class="flex items-center gap-2">
+    <%= form.select :date_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Date" }, required: true, disabled: import.complete? %>
+    <%= form.select :date_format, [["DD-MM-YYYY", "%d-%m-%Y"], ["MM-DD-YYYY", "%m-%d-%Y"], ["YYYY-MM-DD", "%Y-%m-%d"], ["DD/MM/YYYY", "%d/%m/%Y"], ["YYYY/MM/DD", "%Y/%m/%d"], ["MM/DD/YYYY", "%m/%d/%Y"]], { label: true }, required: true, disabled: import.complete? %>
+  </div>
+
+  <div class="flex items-center gap-2">
+    <%= form.select :amount_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Amount" }, required: true, disabled: import.complete? %>
+    <%= form.select :signage_convention, [["Incomes are negative", "inflows_negative"], ["Incomes are positive", "inflows_positive"]], { label: true }, disabled: import.complete? %>
+  </div>
+
+  <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" }, disabled: import.complete? %>
+  <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" }, disabled: import.complete? %>
+  <%= form.select :category_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Category (optional)" }, disabled: import.complete? %>
+  <%= form.select :tags_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Tags (optional)" }, disabled: import.complete? %>
+
+  <%= form.submit "Apply configuration", class: "w-full btn btn--primary", disabled: import.complete? %>
+<% end %>

app/views/import/configurations/_trade_import.html.erb

@@ -0,0 +1,20 @@
+<%# locals: (import:) %>
+
+<%= styled_form_with model: @import, url: import_configuration_path(@import), scope: :import, method: :patch, class: "space-y-2" do |form| %>
+  <div class="flex items-center gap-2">
+    <%= form.select :date_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Date" }, required: true %>
+    <%= form.select :date_format, [["DD-MM-YYYY", "%d-%m-%Y"], ["MM-DD-YYYY", "%m-%d-%Y"], ["YYYY-MM-DD", "%Y-%m-%d"], ["DD/MM/YYYY", "%d/%m/%Y"], ["YYYY/MM/DD", "%Y/%m/%d"], ["MM/DD/YYYY", "%m/%d/%Y"]], label: true, required: true %>
+  </div>
+
+  <div class="flex items-center gap-2">
+    <%= form.select :qty_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Quantity" } %>
+    <%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %>
+  </div>
+
+  <%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %>
+  <%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %>
+  <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>
+  <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>
+
+  <%= form.submit "Apply configuration", class: "w-full btn btn--primary", disabled: import.complete? %>
+<% end %>

app/views/import/configurations/_transaction_import.html.erb

@@ -0,0 +1,21 @@
+<%# locals: (import:) %>
+
+<%= styled_form_with model: @import, url: import_configuration_path(@import), scope: :import, method: :patch, class: "space-y-2" do |form| %>
+  <div class="flex items-center gap-2">
+    <%= form.select :date_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Date" }, required: true %>
+    <%= form.select :date_format, [["DD-MM-YYYY", "%d-%m-%Y"], ["MM-DD-YYYY", "%m-%d-%Y"], ["YYYY-MM-DD", "%Y-%m-%d"], ["DD/MM/YYYY", "%d/%m/%Y"], ["YYYY/MM/DD", "%Y/%m/%d"], ["MM/DD/YYYY", "%m/%d/%Y"]], label: true, required: true %>
+  </div>
+
+  <div class="flex items-center gap-2">
+    <%= form.select :amount_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Amount" }, required: true %>
+    <%= form.select :signage_convention, [["Incomes are positive", "inflows_positive"], ["Incomes are negative", "inflows_negative"]], label: true %>
+  </div>
+
+  <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %>
+  <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %>
+  <%= form.select :category_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Category (optional)" } %>
+  <%= form.select :tags_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Tags (optional)" } %>
+  <%= form.select :notes_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Notes (optional)" } %>
+
+  <%= form.submit "Apply configuration", class: "w-full btn btn--primary", disabled: import.complete? %>
+<% end %>

app/views/import/configurations/show.html.erb

@@ -0,0 +1,22 @@
+<%= content_for :header_nav do %>
+  <%= render "imports/nav", import: @import %>
+<% end %>
+
+<%= content_for :previous_path, import_upload_path(@import) %>
+
+<div>
+  <div class="space-y-4">
+    <div class="text-center space-y-2">
+      <h1 class="text-3xl text-gray-900 font-medium"><%= t(".title") %></h1>
+      <p class="text-gray-500 text-sm"><%= t(".description") %></p>
+    </div>
+
+    <div class="mx-auto max-w-lg">
+      <%= render partial: permitted_import_configuration_path(@import), locals: { import: @import } %>
+    </div>
+  </div>
+
+  <div class="mx-auto max-w-screen-lg my-12">
+    <%= render "imports/table", headers: @import.csv_headers, rows: @import.csv_sample, caption: "Sample data from your uploaded CSV" %>
+  </div>
+</div>

app/views/import/confirms/_mappings.html.erb

@@ -0,0 +1,29 @@
+<%# locals: (import:, mapping_class:, step_idx:) %>
+
+<% mappings = mapping_class.for_import(import) %>
+<% is_last_step = step_idx == import.mapping_steps.count - 1 %>
+
+<div class="space-y-4">
+  <div class="bg-gray-25 rounded-xl p-1 space-y-1 w-[650px]">
+    <div class="grid grid-cols-3 gap-2 text-xs font-medium text-gray-500 uppercase px-5 py-3">
+      <p>CSV <%= mapping_label(mapping_class) %></p>
+      <p>Maybe <%= mapping_label(mapping_class) %></p>
+      <p class="justify-self-end">Rows</p>
+    </div>
+
+    <div class="border border-alpha-black-25 rounded-md shadow-xs divide-y divide-alpha-black-100 text-sm">
+      <% mappings.sort_by(&:key).each do |mapping| %>
+        <div class="px-5 py-3 bg-white first:rounded-tl-xl first:rounded-tr-xl last:rounded-bl-xl last:rounded-br-xl">
+          <%= render partial: "import/mappings/form", locals: { mapping: mapping } %>
+        </div>
+      <% end %>
+    </div>
+  </div>
+
+  <div class="flex justify-center">
+    <%= link_to is_last_step ? import_path(import) : url_for(step: step_idx + 2), class: "btn btn--primary w-36 flex items-center justify-between gap-2" do %>
+      <span>Next</span>
+      <%= lucide_icon "arrow-right", class: "w-5 h-5" %>
+    <% end %>
+  </div>
+</div>

app/views/import/confirms/show.html.erb

@@ -0,0 +1,33 @@
+<%= content_for :header_nav do %>
+  <%= render "imports/nav", import: @import %>
+<% end %>
+
+<%= content_for :previous_path, import_clean_path(@import) %>
+
+<% step_idx = (params[:step] || "1").to_i - 1 %>
+<% step_mapping_class = @import.mapping_steps[step_idx] %>
+
+<div class="space-y-12 mx-auto max-w-md mb-6">
+  <div class="flex justify-center items-center gap-2">
+    <% @import.mapping_steps.each_with_index do |step_mapping_class, idx| %>
+      <% is_active = step_idx == idx %>
+
+      <%= link_to url_for(step: idx + 1), class: "w-5 h-[3px] #{is_active ? 'bg-gray-900' : 'bg-gray-100'} rounded-xl hover:bg-gray-300 transition-colors duration-200" do %>
+        <span class="sr-only">Step <%= idx + 1 %></span>
+      <% end %>
+    <% end %>
+  </div>
+
+  <div class="text-center space-y-2">
+    <h1 class="text-3xl text-gray-900 font-medium">
+      <%= t(".#{step_mapping_class.name.demodulize.underscore}_title", import_type: @import.type.underscore.humanize) %>
+    </h1>
+    <p class="text-gray-500 text-sm">
+      <%= t(".#{step_mapping_class.name.demodulize.underscore}_description", import_type: @import.type.underscore.humanize) %>
+    </p>
+  </div>
+</div>
+
+<div class="max-w-screen-md mx-auto flex justify-center">
+  <%= render partial: "import/confirms/mappings", locals: { import: @import, mapping_class: step_mapping_class, step_idx: step_idx } %>
+</div>

app/views/import/mappings/_form.html.erb

@@ -0,0 +1,29 @@
+<%# locals: (mapping:) %>
+
+<%= styled_form_with model: mapping,
+                     scope: :import_mapping,
+                     url: import_mapping_path(mapping.import, mapping),
+                     class: "grid grid-cols-3 gap-2 items-center",
+                     data: { controller: "auto-submit-form" },
+                     html: { id: dom_id(mapping, :form) } do |form| %>
+  <span><%= mapping.key.blank? ? "(unassigned)" : mapping.key %></span>
+
+  <% if mapping.mappable_class.present? %>
+    <%= form.hidden_field :mappable_type, value: mapping.mappable_class, id: dom_id(mapping, :mappable_type) %>
+    <%= form.select :mappable_id,
+                      mapping.selectable_values,
+                      { container_class: mapping.invalid? ? "border-red-500" : nil, include_blank: mapping.requires_selection? ? "Select an option" : "Leave unassigned", selected: mapping.create_when_empty? ? mapping.class::CREATE_NEW_KEY : mapping.mappable_id },
+                      "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "change", disabled: mapping.import.complete?, id: dom_id(mapping, :mappable_id) %>
+  <% else %>
+    <%= form.select :value, mapping.selectable_values,
+                      { container_class: mapping.invalid? ? "border-red-500" : nil, include_blank: mapping.requires_selection? ? "Select an option" : "Leave unassigned" },
+                      "data-auto-submit-form-target": "auto", "data-autosubmit-trigger-event": "change", disabled: mapping.import.complete?, id: dom_id(mapping, :value) %>
+  <% end %>
+
+  <%= form.hidden_field :key, value: mapping.key, id: dom_id(mapping, :key) %>
+  <%= form.hidden_field :type, value: mapping.type, id: dom_id(mapping, :type) %>
+
+  <span class="justify-self-end">
+    <%= mapping.values_count %>
+  </span>
+<% end %>

app/views/import/rows/_form.html.erb

@@ -0,0 +1,27 @@
+<%# locals: (row:) %>
+
+<div style="grid-template-columns: repeat(<%= row.import.column_keys.count %>, 1fr)" class="first:rounded-tl-lg first:rounded-tr-lg last:rounded-bl-lg last:rounded-br-lg grid divide-x divide-alpha-black-200 group">
+  <% row.import.column_keys.each_with_index do |key, idx| %>
+    <%= turbo_frame_tag dom_id(row, key), title: row.valid? ? nil : row.errors.full_messages.join(", ") do %>
+      <%= form_with(
+            model: [row.import, row],
+            scope: :import_row,
+            url: import_row_path(row.import, row),
+            method: :patch,
+            data: {
+              controller: "auto-submit-form",
+              auto_submit_form_trigger_event_value: "blur"
+            }
+          ) do |form| %>
+        <%= form.text_field key,
+              "data-auto-submit-form-target": "auto",
+              class: [
+                cell_class(row, key),
+                idx == 0 ? "group-first:rounded-tl-lg group-last:rounded-bl-lg" : "",
+                idx == row.import.column_keys.count - 1 ? "group-first:rounded-tr-lg group-last:rounded-br-lg" : "",
+              ],
+              disabled: row.import.complete? %>
+      <% end %>
+    <% end %>
+  <% end %>
+</div>

app/views/import/rows/show.html.erb

@@ -0,0 +1 @@
+<%= render "import/rows/form", row: @row %>

app/views/import/uploads/show.html.erb

@@ -0,0 +1,69 @@
+<%= content_for :header_nav do %>
+  <%= render "imports/nav", import: @import %>
+<% end %>
+
+<%= content_for :previous_path, imports_path %>
+
+<div class="space-y-12">
+  <div class="space-y-4 mx-auto max-w-md">
+    <div class="text-center space-y-2">
+      <h1 class="text-3xl text-gray-900 font-medium"><%= t(".title") %></h1>
+      <p class="text-gray-500 text-sm"><%= t(".description") %></p>
+    </div>
+
+    <div data-controller="tabs" data-tabs-active-class="bg-white" data-tabs-default-tab-value="csv-paste-tab">
+      <div class="flex justify-center mb-4">
+        <div class="bg-gray-50 rounded-lg inline-flex p-1 space-x-2 text-sm text-gray-900 font-medium">
+          <button type="button" data-id="csv-paste-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Copy & Paste</button>
+          <button type="button" data-id="csv-upload-tab" class="p-2 rounded-lg" data-tabs-target="btn" data-action="click->tabs#select">Upload CSV</button>
+        </div>
+      </div>
+
+      <div data-tabs-target="tab" id="csv-paste-tab">
+        <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %>
+          <%= form.select :col_sep, [["Comma (,)", ","], ["Semicolon (;)", ";"]], label: true %>
+          <%= form.text_area :raw_file_str,
+                       rows: 10,
+                       required: true,
+                       placeholder: "Paste your CSV file contents here",
+                       "data-auto-submit-form-target": "auto" %>
+
+          <%= form.submit "Upload CSV", disabled: @import.complete? %>
+        <% end %>
+
+      </div>
+
+      <div data-tabs-target="tab" id="csv-upload-tab" class="hidden">
+        <%= styled_form_with model: @import, scope: :import, url: import_upload_path(@import), multipart: true, class: "space-y-2" do |form| %>
+          <%= form.select :col_sep, [["Comma (,)", ","], ["Semicolon (;)", ";"]], label: true %>
+
+          <label for="import_csv_file" class="flex flex-col items-center justify-center w-full h-56 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50">
+            <div class="flex flex-col items-center justify-center pt-5 pb-6">
+              <%= form.file_field :csv_file, class: "ml-32", "data-auto-submit-form-target": "auto" %>
+            </div>
+          </label>
+
+          <%= form.submit "Upload CSV", disabled: @import.complete? %>
+        <% end %>
+      </div>
+    </div>
+  </div>
+
+  <div class="bg-alpha-black-25 rounded-xl p-1 mt-5 mx-auto max-w-screen-xl">
+    <div class="text-gray-500 p-2 mb-2">
+      <div class="flex gap-2 mb-2">
+        <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %>
+        <p class="text-sm"><%= t(".instructions_1") %></p>
+
+      </div>
+
+      <ul class="list-disc list-inside text-sm pl-8">
+        <li><%= t(".instructions_2") %></li>
+        <li><%= t(".instructions_3") %></li>
+        <li><%= t(".instructions_4") %></li>
+      </ul>
+    </div>
+
+    <%= render partial: "imports/table", locals: { headers: @import.csv_template.headers, rows: @import.csv_template } %>
+  </div>
+</div>