diff --git a/Gemfile b/Gemfile index 194186db..41c7d699 100644 --- a/Gemfile +++ b/Gemfile @@ -46,6 +46,7 @@ gem "octokit" gem "pagy" gem "rails-settings-cached" gem "tzinfo-data", platforms: %i[ windows jruby ] +gem "csv" group :development, :test do gem "debug", platforms: %i[ mri windows ] diff --git a/Gemfile.lock b/Gemfile.lock index 6f604816..4f60365a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -156,6 +156,7 @@ GEM bigdecimal rexml crass (1.0.6) + csv (3.2.8) date (3.3.4) debug (1.9.2) irb (~> 1.10) @@ -456,6 +457,7 @@ DEPENDENCIES brakeman capybara climate_control + csv debug dotenv-rails erb_lint diff --git a/app/assets/images/apple-logo.png b/app/assets/images/apple-logo.png new file mode 100644 index 00000000..107a7858 Binary files /dev/null and b/app/assets/images/apple-logo.png differ diff --git a/app/assets/images/empower-logo.jpeg b/app/assets/images/empower-logo.jpeg new file mode 100644 index 00000000..7193dcc2 Binary files /dev/null and b/app/assets/images/empower-logo.jpeg differ diff --git a/app/assets/images/mint-logo.jpeg b/app/assets/images/mint-logo.jpeg new file mode 100644 index 00000000..9e9723c6 Binary files /dev/null and b/app/assets/images/mint-logo.jpeg differ diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index ac4b5492..ca1b9cce 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -1,4 +1,6 @@ class AccountsController < ApplicationController + layout "with_sidebar" + include Filterable before_action :set_account, only: %i[ show update destroy sync ] @@ -48,7 +50,7 @@ class AccountsController < ApplicationController end end else - render "edit", status: :unprocessable_entity + render "show", status: :unprocessable_entity end end @@ -84,11 +86,11 @@ class AccountsController < ApplicationController private - def set_account - @account = Current.family.accounts.find(params[:id]) - end + def set_account + @account = Current.family.accounts.find(params[:id]) + end - def account_params - params.require(:account).permit(:name, :accountable_type, :balance, :start_date, :currency, :subtype, :is_active) - end + def account_params + params.require(:account).permit(:name, :accountable_type, :balance, :start_date, :currency, :subtype, :is_active) + end end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb new file mode 100644 index 00000000..722a11a9 --- /dev/null +++ b/app/controllers/imports_controller.rb @@ -0,0 +1,102 @@ +require "ostruct" + +class ImportsController < ApplicationController + before_action :set_import, except: %i[ index new create ] + + def index + @imports = Current.family.imports + render layout: "with_sidebar" + end + + def new + @import = Import.new + end + + def edit + end + + def update + account = Current.family.accounts.find(params[:import][:account_id]) + + @import.update! account: account + redirect_to load_import_path(@import), notice: t(".import_updated") + end + + def create + account = Current.family.accounts.find(params[:import][:account_id]) + @import = Import.create!(account: account) + + redirect_to load_import_path(@import), notice: t(".import_created") + end + + def destroy + @import.destroy! + redirect_to imports_url, notice: t(".import_destroyed"), status: :see_other + end + + def load + end + + def load_csv + if @import.update(import_params) + redirect_to configure_import_path(@import), notice: t(".import_loaded") + else + flash.now[:error] = @import.errors.full_messages.to_sentence + render :load, status: :unprocessable_entity + end + end + + def configure + unless @import.loaded? + redirect_to load_import_path(@import), alert: t(".invalid_csv") + end + end + + def update_mappings + @import.update! import_params(@import.expected_fields.map(&:key)) + redirect_to clean_import_path(@import), notice: t(".column_mappings_saved") + end + + def clean + unless @import.loaded? + redirect_to load_import_path(@import), alert: t(".invalid_csv") + end + end + + def update_csv + update_params = import_params[:csv_update] + + @import.update_csv! \ + row_idx: update_params[:row_idx], + col_idx: update_params[:col_idx], + value: update_params[:value] + + render :clean + end + + def confirm + unless @import.cleaned? + redirect_to clean_import_path(@import), alert: t(".invalid_data") + end + end + + def publish + if @import.valid? + @import.publish_later + redirect_to imports_path, notice: t(".import_published") + else + flash.now[:error] = t(".invalid_data") + render :confirm, status: :unprocessable_entity + end + end + + private + + def set_import + @import = Current.family.imports.find(params[:id]) + end + + def import_params(permitted_mappings = nil) + params.require(:import).permit(:raw_csv_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ]) + end +end diff --git a/app/controllers/pages_controller.rb b/app/controllers/pages_controller.rb index ba584e9f..943dcf75 100644 --- a/app/controllers/pages_controller.rb +++ b/app/controllers/pages_controller.rb @@ -1,4 +1,6 @@ class PagesController < ApplicationController + layout "with_sidebar" + include Filterable def dashboard diff --git a/app/controllers/settings/billings_controller.rb b/app/controllers/settings/billings_controller.rb index d218beaa..c4bdd1f5 100644 --- a/app/controllers/settings/billings_controller.rb +++ b/app/controllers/settings/billings_controller.rb @@ -1,4 +1,4 @@ -class Settings::BillingsController < ApplicationController +class Settings::BillingsController < SettingsController def edit end diff --git a/app/controllers/settings/hostings_controller.rb b/app/controllers/settings/hostings_controller.rb index b64a926d..36b9a0ee 100644 --- a/app/controllers/settings/hostings_controller.rb +++ b/app/controllers/settings/hostings_controller.rb @@ -1,4 +1,4 @@ -class Settings::HostingsController < ApplicationController +class Settings::HostingsController < SettingsController before_action :verify_hosting_mode def show diff --git a/app/controllers/settings/notifications_controller.rb b/app/controllers/settings/notifications_controller.rb index 8fceff14..bb458dc7 100644 --- a/app/controllers/settings/notifications_controller.rb +++ b/app/controllers/settings/notifications_controller.rb @@ -1,4 +1,4 @@ -class Settings::NotificationsController < ApplicationController +class Settings::NotificationsController < SettingsController def edit end diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb index 87138e67..dbc5bcb3 100644 --- a/app/controllers/settings/preferences_controller.rb +++ b/app/controllers/settings/preferences_controller.rb @@ -1,4 +1,4 @@ -class Settings::PreferencesController < ApplicationController +class Settings::PreferencesController < SettingsController def edit end @@ -14,11 +14,12 @@ class Settings::PreferencesController < ApplicationController redirect_to settings_preferences_path, notice: t(".success") else redirect_to settings_preferences_path, notice: t(".success") - render :edit, status: :unprocessable_entity + render :show, status: :unprocessable_entity end end private + def preference_params params.require(:user).permit(family_attributes: [ :id, :currency ]) end diff --git a/app/controllers/settings/profiles_controller.rb b/app/controllers/settings/profiles_controller.rb index 83aa94da..326a06f0 100644 --- a/app/controllers/settings/profiles_controller.rb +++ b/app/controllers/settings/profiles_controller.rb @@ -1,4 +1,4 @@ -class Settings::ProfilesController < ApplicationController +class Settings::ProfilesController < SettingsController def show end diff --git a/app/controllers/settings/securities_controller.rb b/app/controllers/settings/securities_controller.rb index 6e49acf0..9d3bac42 100644 --- a/app/controllers/settings/securities_controller.rb +++ b/app/controllers/settings/securities_controller.rb @@ -1,4 +1,4 @@ -class Settings::SecuritiesController < ApplicationController +class Settings::SecuritiesController < SettingsController def edit end diff --git a/app/controllers/settings_controller.rb b/app/controllers/settings_controller.rb new file mode 100644 index 00000000..f908b939 --- /dev/null +++ b/app/controllers/settings_controller.rb @@ -0,0 +1,3 @@ +class SettingsController < ApplicationController + layout "with_sidebar" +end diff --git a/app/controllers/transactions/categories/deletions_controller.rb b/app/controllers/transactions/categories/deletions_controller.rb index ee868abe..c5960ebd 100644 --- a/app/controllers/transactions/categories/deletions_controller.rb +++ b/app/controllers/transactions/categories/deletions_controller.rb @@ -1,4 +1,6 @@ class Transactions::Categories::DeletionsController < ApplicationController + layout "with_sidebar" + before_action :set_category before_action :set_replacement_category, only: :create diff --git a/app/controllers/transactions/categories_controller.rb b/app/controllers/transactions/categories_controller.rb index b01e4439..99bcf6fc 100644 --- a/app/controllers/transactions/categories_controller.rb +++ b/app/controllers/transactions/categories_controller.rb @@ -1,4 +1,6 @@ class Transactions::CategoriesController < ApplicationController + layout "with_sidebar" + before_action :set_category, only: %i[ edit update ] before_action :set_transaction, only: :create diff --git a/app/controllers/transactions/merchants_controller.rb b/app/controllers/transactions/merchants_controller.rb index 2025a6d0..809a8e91 100644 --- a/app/controllers/transactions/merchants_controller.rb +++ b/app/controllers/transactions/merchants_controller.rb @@ -1,4 +1,6 @@ class Transactions::MerchantsController < ApplicationController + layout "with_sidebar" + before_action :set_merchant, only: %i[ edit update destroy ] def index diff --git a/app/controllers/transactions/rules_controller.rb b/app/controllers/transactions/rules_controller.rb index ff27f690..3c1db811 100644 --- a/app/controllers/transactions/rules_controller.rb +++ b/app/controllers/transactions/rules_controller.rb @@ -1,4 +1,6 @@ class Transactions::RulesController < ApplicationController + layout "with_sidebar" + def index end end diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index 36dbe312..ad2cc6e1 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -1,4 +1,6 @@ class TransactionsController < ApplicationController + layout "with_sidebar" + before_action :set_transaction, only: %i[ show edit update destroy ] def index diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 9180c92b..288c3a78 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -51,6 +51,11 @@ module ApplicationHelper end end + def return_to_path(params, fallback = root_path) + uri = URI.parse(params[:return_to] || fallback) + uri.relative? ? uri.path : root_path + end + def trend_styles(trend) fallback = { bg_class: "bg-gray-500/5", text_class: "text-gray-500", symbol: "", icon: "minus" } return fallback if trend.nil? || trend.direction.flat? diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb new file mode 100644 index 00000000..0b924d36 --- /dev/null +++ b/app/helpers/imports_helper.rb @@ -0,0 +1,19 @@ +module ImportsHelper + def table_corner_class(row_idx, col_idx, rows, cols) + return "rounded-tl-xl" if row_idx == 0 && col_idx == 0 + return "rounded-tr-xl" if row_idx == 0 && col_idx == cols.size - 1 + return "rounded-bl-xl" if row_idx == rows.size - 1 && col_idx == 0 + return "rounded-br-xl" if row_idx == rows.size - 1 && col_idx == cols.size - 1 + "" + end + + def nav_steps(import = Import.new) + [ + { name: "Select", complete: import.persisted?, path: import.persisted? ? edit_import_path(import) : new_import_path }, + { name: "Import", complete: import.loaded?, path: import.persisted? ? load_import_path(import) : nil }, + { name: "Setup", complete: import.configured?, path: import.persisted? ? configure_import_path(import) : nil }, + { name: "Clean", complete: import.cleaned?, path: import.persisted? ? clean_import_path(import) : nil }, + { name: "Confirm", complete: import.complete?, path: import.persisted? ? confirm_import_path(import) : nil } + ] + end +end diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb new file mode 100644 index 00000000..f7fc2c01 --- /dev/null +++ b/app/jobs/import_job.rb @@ -0,0 +1,7 @@ +class ImportJob < ApplicationJob + queue_as :default + + def perform(import) + import.publish + end +end diff --git a/app/models/account.rb b/app/models/account.rb index dc0cb30a..2550acef 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -9,6 +9,7 @@ class Account < ApplicationRecord has_many :balances, dependent: :destroy has_many :valuations, dependent: :destroy has_many :transactions, dependent: :destroy + has_many :imports, dependent: :destroy monetize :balance diff --git a/app/models/family.rb b/app/models/family.rb index 3f7747d9..e3e174b1 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -2,6 +2,7 @@ class Family < ApplicationRecord has_many :users, dependent: :destroy has_many :accounts, dependent: :destroy has_many :transactions, through: :accounts + has_many :imports, through: :accounts has_many :transaction_categories, dependent: :destroy, class_name: "Transaction::Category" has_many :transaction_merchants, dependent: :destroy, class_name: "Transaction::Merchant" diff --git a/app/models/import.rb b/app/models/import.rb new file mode 100644 index 00000000..39deb9c3 --- /dev/null +++ b/app/models/import.rb @@ -0,0 +1,161 @@ +class Import < ApplicationRecord + belongs_to :account + + validate :raw_csv_must_be_parsable + + before_save :initialize_csv, if: :should_initialize_csv? + + enum :status, { pending: "pending", complete: "complete", importing: "importing", failed: "failed" }, validate: true + + store_accessor :column_mappings, :define_column_mapping_keys + + scope :ordered, -> { order(created_at: :desc) } + + def publish_later + ImportJob.perform_later(self) + end + + def loaded? + raw_csv_str.present? + end + + def configured? + csv.present? + end + + def cleaned? + loaded? && configured? && csv.valid? + end + + def csv + get_normalized_csv_with_validation + end + + def available_headers + get_raw_csv.table.headers + end + + def get_selected_header_for_field(field) + column_mappings&.dig(field) || field.key + end + + def update_csv!(row_idx:, col_idx:, value:) + updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value) + update! normalized_csv_str: updated_csv.to_s + end + + # Type-specific methods (potential STI inheritance in future when more import types added) + def publish + update!(status: "importing") + + transaction do + generate_transactions.each do |txn| + txn.save! + end + end + + update!(status: "complete") + rescue => e + update!(status: "failed") + Rails.logger.error("Import with id #{id} failed: #{e}") + end + + def dry_run + generate_transactions + end + + def expected_fields + @expected_fields ||= create_expected_fields + end + + private + + def get_normalized_csv_with_validation + return nil if normalized_csv_str.nil? + + csv = Import::Csv.new(normalized_csv_str) + + expected_fields.each do |field| + csv.define_validator(field.key, field.validator) if field.validator + end + + csv + end + + def get_raw_csv + return nil if raw_csv_str.nil? + Import::Csv.new(raw_csv_str) + end + + def should_initialize_csv? + raw_csv_str_changed? || column_mappings_changed? + end + + def initialize_csv + generated_csv = generate_normalized_csv(raw_csv_str) + self.normalized_csv_str = generated_csv.table.to_s + end + + # Uses the user-provided raw CSV + mappings to generate a normalized CSV for the import + def generate_normalized_csv(csv_str) + Import::Csv.create_with_field_mappings(csv_str, expected_fields, column_mappings) + end + + def update_csv(row_idx, col_idx, value) + updated_csv = csv.update_cell(row_idx.to_i, col_idx.to_i, value) + update! normalized_csv_str: updated_csv.to_s + end + + def generate_transactions + transactions = [] + + csv.table.each do |row| + category = account.family.transaction_categories.find_or_initialize_by(name: row["category"]) + txn = account.transactions.build \ + name: row["name"] || "Imported transaction", + date: Date.iso8601(row["date"]), + category: category, + amount: BigDecimal(row["amount"]) * -1 # User inputs amounts with opposite signage of our internal representation + + transactions << txn + end + + transactions + end + + def create_expected_fields + date_field = Import::Field.new \ + key: "date", + label: "Date", + validator: ->(value) { Import::Field.iso_date_validator(value) } + + name_field = Import::Field.new \ + key: "name", + label: "Name" + + category_field = Import::Field.new \ + key: "category", + label: "Category" + + amount_field = Import::Field.new \ + key: "amount", + label: "Amount", + validator: ->(value) { Import::Field.bigdecimal_validator(value) } + + [ date_field, name_field, category_field, amount_field ] + end + + def define_column_mapping_keys + expected_fields.each do |field| + field.key.to_sym + end + end + + def raw_csv_must_be_parsable + begin + CSV.parse(raw_csv_str || "") + rescue CSV::MalformedCSVError + errors.add(:raw_csv_str, "is not a valid CSV format") + end + end +end diff --git a/app/models/import/csv.rb b/app/models/import/csv.rb new file mode 100644 index 00000000..5018af9c --- /dev/null +++ b/app/models/import/csv.rb @@ -0,0 +1,74 @@ +class Import::Csv + def self.parse_csv(csv_str) + CSV.parse((csv_str || "").strip, headers: true, converters: [ ->(str) { str.strip } ]) + end + + def self.create_with_field_mappings(raw_csv_str, fields, field_mappings) + raw_csv = self.parse_csv(raw_csv_str) + + generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true do |csv| + raw_csv.each do |row| + row_values = [] + + fields.each do |field| + # Finds the column header name the user has designated for the expected field + mapped_field_key = field_mappings[field.key] if field_mappings + mapped_header = mapped_field_key || field.key + + row_values << row.fetch(mapped_header, "") + end + + csv << row_values + end + end + + new(generated_csv_str) + end + + attr_reader :csv_str + + def initialize(csv_str, column_validators: nil) + @csv_str = csv_str + @column_validators = column_validators || {} + end + + def table + @table ||= self.class.parse_csv(csv_str) + end + + def update_cell(row_idx, col_idx, value) + copy = table.by_col_or_row + copy[row_idx][col_idx] = value + copy + end + + def valid? + table.each_with_index.all? do |row, row_idx| + row.each_with_index.all? do |cell, col_idx| + cell_valid?(row_idx, col_idx) + end + end + end + + def cell_valid?(row_idx, col_idx) + value = table.dig(row_idx, col_idx) + header = table.headers[col_idx] + validator = get_validator_by_header(header) + validator.call(value) + end + + def define_validator(header_key, validator = nil, &block) + header = table.headers.find { |h| h.strip == header_key } + raise "Cannot define validator for header #{header_key}: header does not exist in CSV" if header.nil? + + column_validators[header] = validator || block + end + + private + + attr_accessor :column_validators + + def get_validator_by_header(header) + column_validators&.dig(header) || ->(_v) { true } + end +end diff --git a/app/models/import/field.rb b/app/models/import/field.rb new file mode 100644 index 00000000..aff9f186 --- /dev/null +++ b/app/models/import/field.rb @@ -0,0 +1,32 @@ +class Import::Field + def self.iso_date_validator(value) + Date.iso8601(value) + true + rescue + false + end + + def self.bigdecimal_validator(value) + BigDecimal(value) + true + rescue + false + end + + attr_reader :key, :label, :validator + + def initialize(key:, label:, validator: nil) + @key = key.to_s + @label = label + @validator = validator + end + + def define_validator(validator = nil, &block) + @validator = validator || block + end + + def validate(value) + return true if validator.nil? + validator.call(value) + end +end diff --git a/app/views/imports/_empty.html.erb b/app/views/imports/_empty.html.erb new file mode 100644 index 00000000..ed7a3b32 --- /dev/null +++ b/app/views/imports/_empty.html.erb @@ -0,0 +1,9 @@ +
+
+

<%= t(".message") %>

+ <%= link_to new_import_path(enable_type_selector: true), class: "w-fit flex text-white text-sm font-medium items-center gap-1 bg-gray-900 rounded-lg p-2 pr-3", data: { turbo_frame: "modal" } do %> + <%= lucide_icon("plus", class: "w-5 h-5") %> + <%= t(".new") %> + <% end %> +
+
diff --git a/app/views/imports/_form.html.erb b/app/views/imports/_form.html.erb new file mode 100644 index 00000000..783a1350 --- /dev/null +++ b/app/views/imports/_form.html.erb @@ -0,0 +1,7 @@ +<%= form_with model: @import do |form| %> +
+ <%= form.collection_select :account_id, Current.family.accounts.alphabetically, :id, :name, { prompt: t(".select_account"), label: t(".account"), required: true } %> +
+ + <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium" %> +<% end %> diff --git a/app/views/imports/_import.html.erb b/app/views/imports/_import.html.erb new file mode 100644 index 00000000..0983e01b --- /dev/null +++ b/app/views/imports/_import.html.erb @@ -0,0 +1,61 @@ +
+
+ +
+

+ <%= t(".label", account: import.account.name) %> +

+ + <% if import.pending? %> + + <%= t(".in_progress") %> + + <% elsif import.importing? %> + + <%= t(".uploading") %> + + <% elsif import.failed? %> + + <%= t(".failed") %> + + <% elsif import.complete? %> + + <%= t(".complete") %> + + <% end %> +
+ + <% if import.complete? %> +

<%= t(".completed_on", datetime: import.updated_at.strftime("%Y-%m-%d")) %>

+ <% else %> +

<%= t(".started_on", datetime: import.created_at.strftime("%Y-%m-%d")) %>

+ <% end %> +
+ + <% if import.complete? %> +
+ <%= lucide_icon("check", class: "text-green-500 w-4 h-4") %> +
+ <% else %> + <%= contextual_menu do %> +
+ <%= link_to edit_import_path(import), + class: "block w-full py-2 px-3 space-x-2 text-gray-900 hover:bg-gray-50 flex items-center rounded-lg" do %> + <%= lucide_icon "pencil-line", class: "w-5 h-5 text-gray-500" %> + + <%= t(".edit") %> + <% end %> + + <%= button_to import_path(import), + method: :delete, + class: "block w-full py-2 px-3 space-x-2 text-red-600 hover:bg-red-50 flex items-center rounded-lg", + data: { turbo_confirm: true } do %> + <%= lucide_icon "trash-2", class: "w-5 h-5" %> + + <%= t(".delete") %> + <% end %> +
+ <% end %> + <% end %> + +
diff --git a/app/views/imports/_nav_step.html.erb b/app/views/imports/_nav_step.html.erb new file mode 100644 index 00000000..375f1e92 --- /dev/null +++ b/app/views/imports/_nav_step.html.erb @@ -0,0 +1,18 @@ +<% is_current = request.path == step[:path] %> +<% text_class = if is_current + "text-gray-900" + else + step[:complete] ? "text-green-600" : "text-gray-500" + end %> +<% step_class = if is_current + "bg-gray-900 text-white" + else + step[:complete] ? "bg-green-600/10 border-alpha-black-25" : "bg-gray-50" + end %> + +
+ + <%= step[:complete] && !is_current ? lucide_icon("check", class: "w-4 h-4") : step_idx + 1 %> + + <%= step[:name] %> +
diff --git a/app/views/imports/_sample_table.html.erb b/app/views/imports/_sample_table.html.erb new file mode 100644 index 00000000..83a16f5c --- /dev/null +++ b/app/views/imports/_sample_table.html.erb @@ -0,0 +1,22 @@ + +
+
Date
+
Name
+
Category
+
Amount
+ +
2024-01-01
+
Amazon
+
Shopping
+
-24.99
+ +
2024-03-01
+
Spotify
+
+
-16.32
+ +
2023-01-06
+
Acme
+
Income
+
151.22
+
diff --git a/app/views/imports/_type_selector.html.erb b/app/views/imports/_type_selector.html.erb new file mode 100644 index 00000000..9f68c343 --- /dev/null +++ b/app/views/imports/_type_selector.html.erb @@ -0,0 +1,90 @@ +
+ +
+
+

<%= t(".import_transactions") %>

+ +
+ +

<%= t(".description") %>

+
+ +
+

<%= t(".sources") %>

+ +
+ +
diff --git a/app/views/imports/clean.html.erb b/app/views/imports/clean.html.erb new file mode 100644 index 00000000..05f94db9 --- /dev/null +++ b/app/views/imports/clean.html.erb @@ -0,0 +1,49 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +
+

<%= t(".clean_import") %>

+ +
+

<%= t(".clean_and_edit") %>

+

<%= t(".clean_description") %>

+
+ +
+
+ <% @import.expected_fields.each do |field| %> +
<%= field.label %>
+ <% end %> +
+ +
+ <% @import.csv.table.each_with_index do |row, row_index| %> +
+ <% row.fields.each_with_index do |value, col_index| %> + <%= form_with model: @import, + builder: ActionView::Helpers::FormBuilder, + url: clean_import_url(@import), + method: :patch, + data: { turbo: false, controller: "auto-submit-form", "auto-submit-form-trigger-event-value" => "blur" } do |form| %> + <%= form.fields_for :csv_update do |ff| %> + <%= ff.hidden_field :row_idx, value: row_index %> + <%= ff.hidden_field :col_idx, value: col_index %> + <%= ff.text_field :value, value: value, + id: "cell-#{row_index}-#{col_index}", + class: "#{@import.csv.cell_valid?(row_index, col_index) ? "focus:border-transparent border-transparent" : "border-red-500"} border px-4 py-3.5 text-sm w-full bg-transparent focus:ring-gray-900 focus:ring-2 focus-visible:outline-none #{table_corner_class(row_index, col_index, @import.csv.table, row.fields)}", + data: { "auto-submit-form-target" => "auto" } %> + <% end %> + <% end %> + <% end %> +
+ <% end %> +
+
+ + <% if @import.csv.valid? %> + <%= link_to "Next", confirm_import_path(@import), class: "px-4 py-2 block w-60 text-center mx-auto rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo: false } %> + <% end %> +
diff --git a/app/views/imports/configure.html.erb b/app/views/imports/configure.html.erb new file mode 100644 index 00000000..78a186f0 --- /dev/null +++ b/app/views/imports/configure.html.erb @@ -0,0 +1,24 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +
+

<%= t(".configure_title") %>

+ +
+

<%= t(".configure_subtitle") %>

+

<%= t(".configure_description") %>

+
+ + <%= form_with model: @import, url: configure_import_path(@import) do |form| %> +
+ <%= form.fields_for :column_mappings do |mappings| %> + <% @import.expected_fields.each do |field| %> + <%= mappings.select field.key, + options_for_select(@import.available_headers, @import.get_selected_header_for_field(field)), + label: field.label %> + <% end %> + <% end %> +
+ + <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.column_mappings? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %> + <% end %> +
diff --git a/app/views/imports/confirm.html.erb b/app/views/imports/confirm.html.erb new file mode 100644 index 00000000..56ac1975 --- /dev/null +++ b/app/views/imports/confirm.html.erb @@ -0,0 +1,16 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +
+

<%= t(".confirm_title") %>

+ +
+

<%= t(".confirm_subtitle") %>

+

<%= t(".confirm_description") %>

+
+ +
+ <%= render partial: "imports/transactions/transaction_group", collection: @import.dry_run.group_by(&:date) %> +
+ + <%= button_to "Import " + @import.csv.table.size.to_s + " transactions", confirm_import_path(@import), method: :patch, class: "px-4 py-2 block w-60 text-center mx-auto rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo: false } %> +
diff --git a/app/views/imports/edit.html.erb b/app/views/imports/edit.html.erb new file mode 100644 index 00000000..c873e237 --- /dev/null +++ b/app/views/imports/edit.html.erb @@ -0,0 +1,10 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +
+

<%= t(".edit_title") %>

+
+

<%= t(".header_text") %>

+

<%= t(".description_text") %>

+
+ <%= render "form", import: @import %> +
diff --git a/app/views/imports/index.html.erb b/app/views/imports/index.html.erb new file mode 100644 index 00000000..97d1ee7e --- /dev/null +++ b/app/views/imports/index.html.erb @@ -0,0 +1,30 @@ +<% content_for :sidebar do %> + <%= render "settings/nav" %> +<% end %> + +
+
+

<%= t(".title") %>

+ <%= link_to new_import_path(enable_type_selector: true), class: "flex text-white text-sm font-medium items-center gap-1 bg-gray-900 rounded-lg p-2 pr-3", data: { turbo_frame: "modal" } do %> + <%= lucide_icon("plus", class: "w-5 h-5") %> + <%= t(".new") %> + <% end %> +
+
+ <% if @imports.empty? %> + <%= render partial: "imports/empty" %> + <% else %> +
+

<%= t(".imports") %> ยท <%= @imports.size %>

+ +
+ <%= render @imports.ordered %> +
+
+ <% end %> +
+
+ <%= previous_setting("Rules", transaction_rules_path) %> + <%= next_setting("What's new", changelog_path) %> +
+
diff --git a/app/views/imports/load.html.erb b/app/views/imports/load.html.erb new file mode 100644 index 00000000..68fab9b9 --- /dev/null +++ b/app/views/imports/load.html.erb @@ -0,0 +1,39 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +
+

<%= t(".load_title") %>

+ +
+

<%= t(".subtitle") %>

+

<%= t(".description") %>

+
+ + <%= form_with model: @import, url: load_import_path(@import) do |form| %> +
+ <%= form.text_area :raw_csv_str, + rows: 10, + required: true, + placeholder: "Paste your CSV file contents here", + class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400 p-4" %> +
+ + <%= form.submit t(".next"), class: "px-4 py-2 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %> + <% end %> + +
+
+
+ <%= lucide_icon("info", class: "w-5 h-5 shrink-0") %> +

<%= t(".instructions") %>

+
+ +
    +
  • <%= t(".requirement1") %>
    • +
  • <%= t(".requirement2") %>
    • +
+
+ + <%= render partial: "imports/sample_table" %> + +
+
diff --git a/app/views/imports/new.html.erb b/app/views/imports/new.html.erb new file mode 100644 index 00000000..afecb846 --- /dev/null +++ b/app/views/imports/new.html.erb @@ -0,0 +1,16 @@ +<%= content_for :return_to_path, return_to_path(params, imports_path) %> + +<% if params[:enable_type_selector].present? %> + <%= modal do %> + <%= render "type_selector" %> + <% end %> +<% end %> + +
+

New import

+
+

<%= t(".header_text") %>

+

<%= t(".description_text") %>

+
+ <%= render "form", import: @import %> +
diff --git a/app/views/imports/show.html.erb b/app/views/imports/show.html.erb new file mode 100644 index 00000000..83dc2e69 --- /dev/null +++ b/app/views/imports/show.html.erb @@ -0,0 +1,15 @@ +
+
+ <% if notice.present? %> +

<%= notice %>

+ <% end %> + + <%= render @import %> + + <%= link_to "Edit this import", edit_import_path(@import), class: "mt-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
+ <%= button_to "Destroy this import", import_path(@import), method: :delete, class: "mt-2 rounded-lg py-3 px-5 bg-gray-100 font-medium" %> +
+ <%= link_to "Back to imports", imports_path, class: "ml-2 rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %> +
+
diff --git a/app/views/imports/transactions/_transaction.html.erb b/app/views/imports/transactions/_transaction.html.erb new file mode 100644 index 00000000..202e0ca7 --- /dev/null +++ b/app/views/imports/transactions/_transaction.html.erb @@ -0,0 +1,12 @@ +<%# locals: (transaction:) %> +
+ <%= render partial: "transactions/transaction_name", locals: { name: transaction.name } %> + +
+ <%= render partial: "transactions/categories/badge", locals: { category: transaction.category } %> +
+ +
+ <%= content_tag :p, format_money(-transaction.amount), class: ["whitespace-nowrap", BigDecimal(transaction.amount).negative? ? "text-green-600" : "text-red-600"] %> +
+
diff --git a/app/views/imports/transactions/_transaction_group.html.erb b/app/views/imports/transactions/_transaction_group.html.erb new file mode 100644 index 00000000..19c1ed4c --- /dev/null +++ b/app/views/imports/transactions/_transaction_group.html.erb @@ -0,0 +1,13 @@ +<%# locals: (transaction_group:) %> +<% date = transaction_group[0] %> +<% transactions = transaction_group[1] %> + +
+
+

<%= date.strftime("%b %d, %Y") %> · <%= transactions.size %>

+ <%= format_money -transactions.sum { |t| t.amount } %> +
+
+ <%= render partial: "imports/transactions/transaction", collection: transactions %> +
+
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index e279c9e9..ddef53e2 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -1,52 +1,39 @@ - + <%= content_for(:title) || "Maybe" %> + + <%= csrf_meta_tags %> + <%= csp_meta_tag %> + + <%= stylesheet_link_tag "tailwind", "inter-font", "data-turbo-track": "reload" %> + <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %> + + <%= javascript_importmap_tags %> + <%= hotwire_livereload_tags if Rails.env.development? %> + <%= turbo_refreshes_with method: :morph, scroll: :preserve %> + - <%= csrf_meta_tags %> - <%= csp_meta_tag %> - - - - - - <%= stylesheet_link_tag "tailwind", "inter-font", "data-turbo-track": "reload" %> - <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %> - <%= javascript_importmap_tags %> - <%= hotwire_livereload_tags if Rails.env.development? %> - <%= turbo_refreshes_with method: :morph, scroll: :preserve %> + <%= yield :head %> +
<%= safe_join(flash.map { |type, message| notification(message, type: type) }) %> -
-
- <% if content_for?(:sidebar) %> - <%= yield :sidebar %> - <% else %> - <%= render "layouts/sidebar" %> - <% end %> -
-
- <%= yield %> -
-
+ + <%= content_for?(:content) ? yield(:content) : yield %> + <%= turbo_frame_tag "modal" %> <%= render "shared/confirm_modal" %> - <%= render "shared/upgrade_notification" %> + <% if self_hosted? %> -
-

Self-hosted Maybe: <%= Maybe.version.to_release_tag %>

- <%= link_to settings_hosting_path, class: "flex gap-1 items-center hover:bg-gray-50 rounded-md p-2" do %> - <%= lucide_icon("settings", class: "w-4 h-4 text-gray-500 shrink-0") %> - <% end %> -
+ <%= render "shared/app_version" %> <% end %> diff --git a/app/views/layouts/auth.html.erb b/app/views/layouts/auth.html.erb index 2e009d33..05c81e68 100644 --- a/app/views/layouts/auth.html.erb +++ b/app/views/layouts/auth.html.erb @@ -1,61 +1,31 @@ - - - - Maybe - - - - +<%= content_for :content do %> +
+
+ <%= render "shared/logo" %> - - - - - - - +

+ <%= content_for?(:header_title) ? yield(:header_title).html_safe : t(".your_account") %> +

- <%= csrf_meta_tags %> - <%= csp_meta_tag %> - <%= stylesheet_link_tag "tailwind", "inter-font", "data-turbo-track": "reload" %> - - <%= stylesheet_link_tag "application", "data-turbo-track": "reload" %> - <%= javascript_importmap_tags %> - - <%= hotwire_livereload_tags if Rails.env.development? %> - <%= turbo_refreshes_with method: :morph, scroll: :preserve %> - - - -
- -
- <%= render "shared/logo" %> - -

- <%= content_for?(:header_title) ? yield(:header_title).html_safe : t(".your_account") %> -

- - <% if controller_name == "sessions" %> + <% if controller_name == "sessions" %>

<%= t(".or") %> <%= link_to t(".sign_up"), new_registration_path, class: "font-medium text-gray-600 hover:text-gray-400 transition" %>

- <% elsif controller_name == "registrations" %> + <% elsif controller_name == "registrations" %>

<%= t(".or") %> <%= link_to t(".sign_in"), new_session_path, class: "font-medium text-gray-600 hover:text-gray-400 transition" %>

- <% end %> - -
- -
- <%= yield %> -
- -
-

<%= link_to t(".privacy_policy"), "/privacy", class: "font-medium text-gray-600 hover:text-gray-400 transition" %> • <%= link_to t(".terms_of_service"), "/terms", class: "font-medium text-gray-600 hover:text-gray-400 transition" %>

-
- + <% end %>
- - + +
+ <%= yield %> +
+ +
+

<%= link_to t(".privacy_policy"), "/privacy", class: "font-medium text-gray-600 hover:text-gray-400 transition" %> • <%= link_to t(".terms_of_service"), "/terms", class: "font-medium text-gray-600 hover:text-gray-400 transition" %>

+
+
+<% end %> + +<%= render template: "layouts/application" %> diff --git a/app/views/layouts/imports.html.erb b/app/views/layouts/imports.html.erb new file mode 100644 index 00000000..e44ffb25 --- /dev/null +++ b/app/views/layouts/imports.html.erb @@ -0,0 +1,34 @@ +<%= content_for :content do %> +
+ <%= link_to root_path do %> + <%= image_tag "logo.svg", alt: "Maybe", class: "h-[22px]" %> + <% end %> + + <%= link_to content_for(:return_to_path) do %> + <%= lucide_icon("x", class: "text-gray-500 w-5 h-5") %> + <% end %> +
+ + <%= yield %> +<% end %> + +<%= render template: "layouts/application" %> diff --git a/app/views/layouts/with_sidebar.html.erb b/app/views/layouts/with_sidebar.html.erb new file mode 100644 index 00000000..7950d53a --- /dev/null +++ b/app/views/layouts/with_sidebar.html.erb @@ -0,0 +1,18 @@ +<%= content_for :content do %> +
+
+ <% if content_for?(:sidebar) %> + <%= yield :sidebar %> + <% else %> + <%= render "layouts/sidebar" %> + <% end %> +
+
+ <%= yield %> +
+
+ + <%= render "shared/upgrade_notification" %> +<% end %> + +<%= render template: "layouts/application" %> diff --git a/app/views/pages/changelog.html.erb b/app/views/pages/changelog.html.erb index 700b358e..f44eef8c 100644 --- a/app/views/pages/changelog.html.erb +++ b/app/views/pages/changelog.html.erb @@ -9,7 +9,7 @@
- <%= previous_setting("Rules", transaction_rules_path) %> + <%= previous_setting("Imports", imports_path) %> <%= next_setting("Feedback", feedback_path) %>
diff --git a/app/views/settings/_nav.html.erb b/app/views/settings/_nav.html.erb index 838684f7..2741bc87 100644 --- a/app/views/settings/_nav.html.erb +++ b/app/views/settings/_nav.html.erb @@ -55,6 +55,9 @@
  • <%= sidebar_link_to t(".rules_label"), transaction_rules_path, icon: "list-checks" %>
    • +
  • + <%= sidebar_link_to t(".imports_label"), imports_path, icon: "download" %> +
    • diff --git a/app/views/shared/_app_version.html.erb b/app/views/shared/_app_version.html.erb new file mode 100644 index 00000000..0632359e --- /dev/null +++ b/app/views/shared/_app_version.html.erb @@ -0,0 +1,6 @@ +
    +

    Version: <%= Maybe.version.to_release_tag %>

    + <%= link_to settings_hosting_path, class: "flex gap-1 items-center hover:bg-gray-50 rounded-md p-2" do %> + <%= lucide_icon("settings", class: "w-4 h-4 text-gray-500 shrink-0") %> + <% end %> +
    diff --git a/app/views/transactions/index.html.erb b/app/views/transactions/index.html.erb index 5184296e..d7151e73 100644 --- a/app/views/transactions/index.html.erb +++ b/app/views/transactions/index.html.erb @@ -6,16 +6,28 @@ <%= contextual_menu do %>
    <%= link_to transaction_categories_path, - class: "block w-full py-2 px-3 space-x-2 text-gray-900 hover:bg-gray-50 flex items-center rounded-lg font-normal" do %> + class: "block w-full py-2 px-3 space-x-2 text-gray-900 hover:bg-gray-50 flex items-center rounded-lg font-normal" do %> <%= lucide_icon "tags", class: "w-5 h-5 text-gray-500" %> <%= t(".edit_categories") %> <% end %> + + <%= link_to imports_path, + class: "block w-full py-2 px-3 space-x-2 text-gray-900 hover:bg-gray-50 flex items-center rounded-lg font-normal" do %> + <%= lucide_icon "hard-drive-upload", class: "w-5 h-5 text-gray-500" %> + <%= t(".edit_imports") %> + <% end %>
    + + <% end %> + + <%= link_to new_import_path(enable_type_selector: true), class: "rounded-lg bg-gray-50 border border-gray-200 flex items-center gap-1 justify-center px-3 py-2", data: { turbo_frame: "modal" } do %> + <%= lucide_icon("download", class: "text-gray-500 w-4 h-4") %> +

    <%= t(".import") %>

    <% end %> <%= link_to new_transaction_path, class: "rounded-lg bg-gray-900 text-white flex items-center gap-1 justify-center hover:bg-gray-700 px-3 py-2", data: { turbo_frame: :modal } do %> <%= lucide_icon("plus", class: "w-5 h-5") %> -

    New transaction

    +

    New transaction

    <% end %> diff --git a/app/views/transactions/rules/index.html.erb b/app/views/transactions/rules/index.html.erb index 7fce52ab..bb0bf5f6 100644 --- a/app/views/transactions/rules/index.html.erb +++ b/app/views/transactions/rules/index.html.erb @@ -10,6 +10,6 @@
    <%= previous_setting("Merchants", transaction_merchants_path) %> - <%= next_setting("What's New", changelog_path) %> + <%= next_setting("Imports", imports_path) %>
    diff --git a/config/environments/test.rb b/config/environments/test.rb index e7e507b7..501daafe 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -62,4 +62,6 @@ Rails.application.configure do # Raise error when a before_action's only/except options reference missing actions config.action_controller.raise_on_missing_callback_actions = true + + config.autoload_paths += %w[ test/support ] end diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.yml new file mode 100644 index 00000000..8a089ca5 --- /dev/null +++ b/config/locales/views/imports/en.yml @@ -0,0 +1,95 @@ +--- +en: + imports: + clean: + clean_and_edit: Clean and edit your data + clean_description: Edit your transactions in the table below. Click on any cell + to change the date, name, category, or amount. + clean_import: Clean import + invalid_csv: Please load a CSV first + configure: + configure_description: Select the columns that match the necessary data fields, + so that the columns in your CSV can be correctly mapped with our format. + configure_subtitle: Setup your CSV file + configure_title: Configure import + confirm_accept: Change mappings + confirm_body: Changing your mappings may erase any edits you have made to the + CSV so far. + confirm_title: Are you sure? + invalid_csv: Please load a CSV first + next: Next + confirm: + confirm_description: Preview your transactions below and check to see if there + are any changes that are required. + confirm_subtitle: Confirm your transactions + confirm_title: Confirm import + invalid_data: You have invalid data, please fix before continuing + create: + import_created: Import created + destroy: + import_destroyed: Import destroyed + edit: + description_text: Importing transactions can only be done for one account at + a time. You will need to go through this process again for other accounts. + edit_title: Edit import + header_text: Select the account your transactions will belong to + empty: + message: No imports to show + new: New Import + form: + account: Account + next: Next + select_account: Select account + import: + complete: Complete + completed_on: Completed on %{datetime} + delete: Delete + edit: Edit + failed: Failed + in_progress: In progress + label: 'Import for: %{account}' + started_on: Started on %{datetime} + uploading: Processing rows + index: + imports: Imports + new: New import + title: Imports + load: + confirm_accept: Yep, start over! + confirm_body: This will reset your import. Any changes you have made to the + CSV will be erased. + confirm_title: Are you sure? + description: Create a spreadsheet or upload an exported CSV from your financial + institution. + instructions: Your CSV should have the following columns and formats for the + best import experience. + load_title: Load import + next: Next + requirement1: Dates must be in ISO 8601 format (YYYY-MM-DD) + requirement2: Negative transaction is an "outflow" (expense), positive is an + "inflow" (income) + subtitle: Import your transactions + load_csv: + import_loaded: Import CSV loaded + new: + description_text: Importing transactions can only be done for one account at + a time. You will need to go through this process again for other accounts. + header_text: Select the account your transactions will belong to + publish: + import_published: Import has started in the background + invalid_data: Your import is invalid + type_selector: + description: You can manually import transactions from CSVs or from other financial + apps like Mint, Empower (formerly Personal Capital) or Apple Card. + import_from_apple: Import from Apple Card + import_from_csv: New import from CSV + import_from_empower: Import from Empower + import_from_mint: Import from Mint + import_transactions: Import transactions + resume_latest_import: Resume latest import + soon: Soon + sources: Sources + update: + import_updated: Import updated + update_mappings: + column_mappings_saved: Column mappings saved diff --git a/config/locales/views/settings/en.yml b/config/locales/views/settings/en.yml index a39160fa..21071ce8 100644 --- a/config/locales/views/settings/en.yml +++ b/config/locales/views/settings/en.yml @@ -55,6 +55,7 @@ en: categories_label: Categories feedback_label: Feedback general_section_title: General + imports_label: Imports invite_label: Invite friends merchants_label: Merchants notifications_label: Notifications diff --git a/config/locales/views/transaction/en.yml b/config/locales/views/transaction/en.yml index 12e2e841..ab928126 100644 --- a/config/locales/views/transaction/en.yml +++ b/config/locales/views/transaction/en.yml @@ -57,6 +57,8 @@ en: transfer: Transfer index: edit_categories: Edit categories + edit_imports: Edit imports + import: Import merchants: create: success: New merchant created successfully diff --git a/config/routes.rb b/config/routes.rb index 3f63169c..909870af 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -21,8 +21,24 @@ Rails.application.routes.draw do resource :security, only: %i[show update] end + resources :imports, except: :show do + member do + get "load" + patch "load" => "imports#load_csv" + + get "configure" + patch "configure" => "imports#update_mappings" + + get "clean" + patch "clean" => "imports#update_csv" + + get "confirm" + patch "confirm" => "imports#publish" + end + end + resources :transactions do - match "search" => "transactions#search", on: :collection, via: [ :get, :post ], as: :search + match "search" => "transactions#search", on: :collection, via: %i[ get post ], as: :search collection do scope module: :transactions do diff --git a/db/migrate/20240502205006_create_imports.rb b/db/migrate/20240502205006_create_imports.rb new file mode 100644 index 00000000..79c67284 --- /dev/null +++ b/db/migrate/20240502205006_create_imports.rb @@ -0,0 +1,15 @@ +class CreateImports < ActiveRecord::Migration[7.2] + def change + create_enum :import_status, %w[pending importing complete failed] + + create_table :imports, id: :uuid do |t| + t.references :account, null: false, foreign_key: true, type: :uuid + t.jsonb :column_mappings + t.enum :status, enum_type: :import_status, default: "pending" + t.string :raw_csv_str + t.string :normalized_csv_str + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index c3e2f2da..75a4c530 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_04_30_111641) do +ActiveRecord::Schema[7.2].define(version: 2024_05_02_205006) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -18,6 +18,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_04_30_111641) do # Custom types defined in this database. # Note that some types may not work with other database engines. Be careful if changing database. create_enum "account_status", ["ok", "syncing", "error"] + create_enum "import_status", ["pending", "importing", "complete", "failed"] create_enum "user_role", ["admin", "member"] create_table "account_balances", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| @@ -222,6 +223,17 @@ ActiveRecord::Schema[7.2].define(version: 2024_04_30_111641) do t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" end + create_table "imports", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "account_id", null: false + t.jsonb "column_mappings" + t.enum "status", default: "pending", enum_type: "import_status" + t.string "raw_csv_str" + t.string "normalized_csv_str" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["account_id"], name: "index_imports_on_account_id" + end + create_table "invite_codes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.string "token", null: false t.datetime "created_at", null: false @@ -305,6 +317,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_04_30_111641) do add_foreign_key "accounts", "families" add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "imports", "accounts" add_foreign_key "transaction_categories", "families" add_foreign_key "transaction_merchants", "families" add_foreign_key "transactions", "accounts", on_delete: :cascade diff --git a/test/application_system_test_case.rb b/test/application_system_test_case.rb index 145d7302..4713e4b6 100644 --- a/test/application_system_test_case.rb +++ b/test/application_system_test_case.rb @@ -1,21 +1,27 @@ require "test_helper" class ApplicationSystemTestCase < ActionDispatch::SystemTestCase - driven_by :selenium, using: :headless_chrome, screen_size: [ 1400, 1400 ] + driven_by :selenium, using: ENV["CI"].present? ? :headless_chrome : :chrome, screen_size: [ 1400, 1400 ] private - def sign_in(user) - visit new_session_path - within "form" do - fill_in "Email", with: user.email - fill_in "Password", with: "password" - click_button "Log in" - end - end + def sign_in(user) + visit new_session_path + within "form" do + fill_in "Email", with: user.email + fill_in "Password", with: "password" + click_on "Log in" + end - def sign_out - find("#user-menu").click - click_button "Logout" - end + # Trigger Capybara's wait mechanism to avoid timing issues with logins + find("h1", text: "Dashboard") + end + + def sign_out + find("#user-menu").click + click_button "Logout" + + # Trigger Capybara's wait mechanism to avoid timing issues with logout + find("h2", text: "Sign in to your account") + end end diff --git a/test/controllers/imports_controller_test.rb b/test/controllers/imports_controller_test.rb new file mode 100644 index 00000000..3a71aa07 --- /dev/null +++ b/test/controllers/imports_controller_test.rb @@ -0,0 +1,141 @@ +require "test_helper" + +class ImportsControllerTest < ActionDispatch::IntegrationTest + include ImportTestHelper + + setup do + sign_in @user = users(:family_admin) + @empty_import = imports(:empty_import) + @loaded_import = imports(:loaded_import) + @completed_import = imports(:completed_import) + end + + test "should get index" do + get imports_url + assert_response :success + + @user.family.imports.ordered.each do |import| + assert_select "#" + dom_id(import), count: 1 + end + end + + test "should get new" do + get new_import_url + assert_response :success + end + + test "should create import" do + assert_difference("Import.count") do + post imports_url, params: { import: { account_id: @user.family.accounts.first.id } } + end + + assert_redirected_to load_import_path(Import.ordered.first) + end + + test "should get edit" do + get edit_import_url(@empty_import) + assert_response :success + end + + test "should update import" do + patch import_url(@empty_import), params: { import: { account_id: @empty_import.account_id } } + assert_redirected_to load_import_path(@empty_import) + end + + test "should destroy import" do + assert_difference("Import.count", -1) do + delete import_url(@empty_import) + end + + assert_redirected_to imports_url + end + + test "should get load" do + get load_import_url(@empty_import) + assert_response :success + end + + test "should save raw CSV if valid" do + patch load_import_url(@empty_import), params: { import: { raw_csv_str: valid_csv_str } } + + assert_redirected_to configure_import_path(@empty_import) + assert_equal "Import CSV loaded", flash[:notice] + end + + test "should flash error message if invalid CSV input" do + patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } } + + assert_response :unprocessable_entity + assert_equal "Raw csv str is not a valid CSV format", flash[:error] + end + + test "should get configure" do + get configure_import_url(@loaded_import) + assert_response :success + end + + test "should redirect back to load step with an alert message if not loaded" do + get configure_import_url(@empty_import) + assert_equal "Please load a CSV first", flash[:alert] + assert_redirected_to load_import_path(@empty_import) + end + + test "should update mappings" do + patch configure_import_url(@loaded_import), params: { + import: { + column_mappings: { + date: "date", + name: "name", + category: "category", + amount: "amount" + } + } + } + + assert_redirected_to clean_import_path(@loaded_import) + assert_equal "Column mappings saved", flash[:notice] + end + + test "can update a cell" do + assert_equal @loaded_import.csv.table[0][1], "Starbucks drink" + + patch clean_import_url(@loaded_import), params: { + import: { + csv_update: { + row_idx: 0, + col_idx: 1, + value: "new_merchant" + } + } + } + + assert_response :success + + @loaded_import.reload + assert_equal "new_merchant", @loaded_import.csv.table[0][1] + end + + test "should get clean" do + get clean_import_url(@loaded_import) + assert_response :success + end + + test "should get confirm if all values are valid" do + get confirm_import_url(@loaded_import) + assert_response :success + end + + test "should redirect back to clean if data is invalid" do + @empty_import.update! raw_csv_str: valid_csv_with_invalid_values + + get confirm_import_url(@empty_import) + assert_equal "You have invalid data, please fix before continuing", flash[:alert] + assert_redirected_to clean_import_path(@empty_import) + end + + test "should confirm import" do + patch confirm_import_url(@loaded_import) + assert_redirected_to imports_path + assert_equal "Import has started in the background", flash[:notice] + end +end diff --git a/test/fixtures/imports.yml b/test/fixtures/imports.yml new file mode 100644 index 00000000..b0c30b4a --- /dev/null +++ b/test/fixtures/imports.yml @@ -0,0 +1,32 @@ +empty_import: + account: checking + created_at: <%= 1.minute.ago %> + +loaded_import: + account: checking + raw_csv_str: | + date,name,category,amount + 2024-01-01,Starbucks drink,Food,20 + 2024-01-02,Amazon stuff,Shopping,200 + normalized_csv_str: | + date,name,category,amount + 2024-01-01,Starbucks drink,Food,20 + 2024-01-02,Amazon stuff,Shopping,200 + created_at: <%= 2.days.ago %> + +completed_import: + account: checking + column_mappings: + date: date + name: name + category: category + amount: amount + raw_csv_str: | + date,name,category,amount + 2024-01-01,Starbucks drink,Food,20 + normalized_csv_str: | + date,name,category,amount + 2024-01-01,Starbucks drink,Food,20 + created_at: <%= 2.days.ago %> + + diff --git a/test/integration/.keep b/test/integration/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/test/jobs/import_job_test.rb b/test/jobs/import_job_test.rb new file mode 100644 index 00000000..14c21339 --- /dev/null +++ b/test/jobs/import_job_test.rb @@ -0,0 +1,18 @@ +require "test_helper" + +class ImportJobTest < ActiveJob::TestCase + include ImportTestHelper + + test "import is published" do + import = imports(:empty_import) + import.update! raw_csv_str: valid_csv_str + + assert import.pending? + + perform_enqueued_jobs do + ImportJob.perform_later(import) + end + + assert import.reload.complete? + end +end diff --git a/test/models/import/csv_test.rb b/test/models/import/csv_test.rb new file mode 100644 index 00000000..1bef280d --- /dev/null +++ b/test/models/import/csv_test.rb @@ -0,0 +1,87 @@ +require "test_helper" + +class Import::CsvTest < ActiveSupport::TestCase + include ImportTestHelper + + setup do + @csv = Import::Csv.new(valid_csv_str) + end + + test "cannot define validator for non-existent header" do + assert_raises do + @csv.define_validator "invalid", method(:validate_iso_date) + end + end + + test "csv with no validators is valid" do + assert @csv.cell_valid?(0, 0) + assert @csv.valid? + end + + test "valid csv values" do + @csv.define_validator "date", method(:validate_iso_date) + + assert_equal "2024-01-01", @csv.table[0][0] + assert @csv.cell_valid?(0, 0) + assert @csv.valid? + end + + test "invalid csv values" do + invalid_csv = Import::Csv.new valid_csv_with_invalid_values + + invalid_csv.define_validator "date", method(:validate_iso_date) + + assert_equal "invalid_date", invalid_csv.table[0][0] + assert_not invalid_csv.cell_valid?(0, 0) + assert_not invalid_csv.valid? + end + + test "updating a cell returns a copy of the original csv" do + original_date = "2024-01-01" + new_date = "2024-01-01" + + assert_equal original_date, @csv.table[0][0] + updated = @csv.update_cell(0, 0, new_date) + + assert_equal original_date, @csv.table[0][0] + assert_equal new_date, updated[0][0] + end + + test "can create CSV with expected columns and field mappings with validators" do + date_field = Import::Field.new \ + key: "date", + label: "Date", + validator: method(:validate_iso_date) + + name_field = Import::Field.new \ + key: "name", + label: "Name" + + fields = [ date_field, name_field ] + + raw_csv_str = <<-ROWS + date,Custom Field Header,extra_field + invalid_date_value,Starbucks drink,Food + 2024-01-02,Amazon stuff,Shopping + ROWS + + mappings = { + "name" => "Custom Field Header" + } + + csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings) + + assert_equal %w[ date name ], csv.table.headers + assert_equal 2, csv.table.size + assert_equal "Amazon stuff", csv.table[1][1] + end + + private + + def validate_iso_date(value) + Date.iso8601(value) + true + rescue + false + end +end diff --git a/test/models/import/field_test.rb b/test/models/import/field_test.rb new file mode 100644 index 00000000..1550a449 --- /dev/null +++ b/test/models/import/field_test.rb @@ -0,0 +1,28 @@ +require "test_helper" + +class Import::FieldTest < ActiveSupport::TestCase + test "key is always a string" do + field1 = Import::Field.new label: "Test", key: "test" + field2 = Import::Field.new label: "Test2", key: :test2 + + assert_equal "test", field1.key + assert_equal "test2", field2.key + end + + test "can set and override a validator for a field" do + field = Import::Field.new \ + label: "Test", + key: "Test", + validator: ->(val) { val == 42 } + + assert field.validate(42) + assert_not field.validate(41) + + field.define_validator do |value| + value == 100 + end + + assert field.validate(100) + assert_not field.validate(42) + end +end diff --git a/test/models/import_test.rb b/test/models/import_test.rb new file mode 100644 index 00000000..4114cf38 --- /dev/null +++ b/test/models/import_test.rb @@ -0,0 +1,61 @@ +require "test_helper" + +class ImportTest < ActiveSupport::TestCase + include ImportTestHelper, ActiveJob::TestHelper + + setup do + @empty_import = imports(:empty_import) + @loaded_import = imports(:loaded_import) + end + + test "raw csv input must conform to csv spec" do + @empty_import.raw_csv_str = malformed_csv_str + assert_not @empty_import.valid? + + @empty_import.raw_csv_str = valid_csv_str + assert @empty_import.valid? + end + + test "can update csv value without affecting raw input" do + assert_equal "Starbucks drink", @loaded_import.csv.table[0][1] + + prior_raw_csv_str_value = @loaded_import.raw_csv_str + prior_normalized_csv_str_value = @loaded_import.normalized_csv_str + + @loaded_import.update_csv! \ + row_idx: 0, + col_idx: 1, + value: "new_category" + + assert_equal "new_category", @loaded_import.csv.table[0][1] + assert_equal prior_raw_csv_str_value, @loaded_import.raw_csv_str + assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str + end + + test "publishes later" do + assert_enqueued_with(job: ImportJob) do + @loaded_import.publish_later + end + end + + test "publishes a valid import" do + assert_difference "Transaction.count", 2 do + @loaded_import.publish + end + + @loaded_import.reload + + assert @loaded_import.complete? + end + + test "failed publish results in error status" do + @empty_import.update! raw_csv_str: valid_csv_with_invalid_values + + assert_difference "Transaction.count", 0 do + @empty_import.publish + end + + @empty_import.reload + assert @empty_import.failed? + end +end diff --git a/test/support/import_test_helper.rb b/test/support/import_test_helper.rb new file mode 100644 index 00000000..ee8eb3c2 --- /dev/null +++ b/test/support/import_test_helper.rb @@ -0,0 +1,24 @@ +module ImportTestHelper + def valid_csv_str + <<-ROWS + date,name,category,amount + 2024-01-01,Starbucks drink,Food,20 + 2024-01-02,Amazon stuff,Shopping,200 + ROWS + end + + def valid_csv_with_invalid_values + <<-ROWS + date,name,category,amount + invalid_date,Starbucks drink,Food,invalid_amount + ROWS + end + + def malformed_csv_str + <<-ROWS + name,age + "John Doe,23 + "Jane Doe",25 + ROWS + end +end diff --git a/test/system/imports_test.rb b/test/system/imports_test.rb new file mode 100644 index 00000000..b9ee4f8c --- /dev/null +++ b/test/system/imports_test.rb @@ -0,0 +1,113 @@ +require "application_system_test_case" + +class ImportsTest < ApplicationSystemTestCase + include ImportTestHelper + + setup do + sign_in @user = users(:family_admin) + + @imports = @user.family.imports.ordered.to_a + end + + test "can trigger new import from settings" do + trigger_import_from_settings + verify_import_modal + end + + test "can resume existing import from settings" do + visit imports_url + + within "#" + dom_id(@imports.first) do + click_button + click_link "Edit" + end + + assert_current_path edit_import_path(@imports.first) + end + + test "can resume latest import" do + trigger_import_from_transactions + verify_import_modal + + click_link "Resume latest import" + + assert_current_path edit_import_path(@imports.first) + end + + test "can perform basic CSV import" do + trigger_import_from_settings + verify_import_modal + + within "#modal" do + click_link "New import from CSV" + end + + # 1) Create import step + assert_selector "h1", text: "New import" + + within "form" do + select "Checking Account", from: "import_account_id" + end + + click_button "Next" + + # 2) Load Step + assert_selector "h1", text: "Load import" + + within "form" do + fill_in "import_raw_csv_str", with: <<-ROWS + date,Custom Name Column,category,amount + invalid_date,Starbucks drink,Food,-20.50 + 2024-01-01,Amazon purchase,Shopping,-89.50 + ROWS + end + + click_button "Next" + + # 3) Configure step + assert_selector "h1", text: "Configure import" + + within "form" do + select "Custom Name Column", from: "import_column_mappings_name" + end + + click_button "Next" + + # 4) Clean step + assert_selector "h1", text: "Clean import" + + # We have an invalid value, so user cannot click next yet + assert_no_text "Next" + + # Replace invalid date with valid date + fill_in "cell-0-0", with: "2024-01-02" + + # Trigger blur event so value saves + find("body").click + + click_link "Next" + + # 5) Confirm step + assert_selector "h1", text: "Confirm import" + click_button "Import 2 transactions" + assert_selector "h1", text: "Imports" + end + + private + + def trigger_import_from_settings + visit imports_url + click_link "New import" + end + + def trigger_import_from_transactions + visit transactions_url + click_link "Import" + end + + def verify_import_modal + within "#modal" do + assert_text "Import transactions" + end + end +end diff --git a/test/system/settings_test.rb b/test/system/settings_test.rb index add35d39..29517588 100644 --- a/test/system/settings_test.rb +++ b/test/system/settings_test.rb @@ -14,6 +14,7 @@ class SettingsTest < ApplicationSystemTestCase [ "Categories", "Categories", transaction_categories_path ], [ "Merchants", "Merchants", transaction_merchants_path ], [ "Rules", "Rules", transaction_rules_path ], + [ "Imports", "Imports", imports_path ], [ "What's New", "What's New", changelog_path ], [ "Feedback", "Feedback", feedback_path ], [ "Invite friends", "Invite friends", invites_path ] @@ -27,6 +28,7 @@ class SettingsTest < ApplicationSystemTestCase end private + def open_settings_from_sidebar find("#user-menu").click click_link "Settings"