Docker/Maybe
Docker/Maybe
1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-07-19 05:09:38 +02:00
Code Issues Releases Activity Actions 1

Refactor: Allow other import files (#1099)

Browse source 
* Rename stimulus controller

* feature: rename raw_csv_str to raw_file_str
This commit is contained in:
Pedro Carmona 2024-08-19 14:25:07 +01:00 • committed by GitHub
parent e6528bafec
commit 0c1ff00c1e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 71 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/controllers/imports_controller.rb
View file

@ -40,7 +40,7 @@ class ImportsController < ApplicationController
def upload_csv
begin
@import.raw_csv_str = import_params[:raw_csv_str].read
@import.raw_file_str = import_params[:raw_file_str].read
rescue NoMethodError
flash.now[:alert] = "Please select a file to upload"
render :load, status: :unprocessable_entity and return
@ -113,6 +113,6 @@ class ImportsController < ApplicationController
end
def import_params(permitted_mappings = nil)
params.require(:import).permit(:raw_csv_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
params.require(:import).permit(:raw_file_str, column_mappings: permitted_mappings, csv_update: [ :row_idx, :col_idx, :value ])
end
end

20
app/javascript/controllers/csv_upload_controller.js → app/javascript/controllers/import_upload_controller.js
View file

@ -2,6 +2,11 @@ import { Controller } from "@hotwired/stimulus"
export default class extends Controller {
static targets = ["input", "preview", "submit", "filename", "filesize"]
static values = {
acceptedTypes: Array, // ["text/csv", "application/csv", ".csv"]
acceptedExtension: String, // "csv"
unacceptableTypeLabel: String, // "Only CSV files are allowed."
};
connect() {
this.submitTarget.disabled = true
@ -30,15 +35,19 @@ export default class extends Controller {
event.currentTarget.classList.remove("bg-gray-100")
const file = event.dataTransfer.files[0]
if (file && this._isCSVFile(file)) {
if (file && this._formatAcceptable(file)) {
this._setFileInput(file);
this._fileAdded(file)
} else {
this.previewTarget.classList.add("text-red-500")
this.previewTarget.textContent = "Only CSV files are allowed."
this.previewTarget.textContent = this.unacceptableTypeLabelValue
}
}
click() {
this.inputTarget.click();
}
// Private
_fetchFileSize(size) {
@ -57,7 +66,7 @@ export default class extends Controller {
if (file) {
if (file.size > fileSizeLimit) {
this.previewTarget.classList.add("text-red-500")
this.previewTarget.textContent = "File size exceeds the limit of 5MB"
this.previewTarget.textContent = this.unacceptableTypeLabelValue
return
}
@ -80,10 +89,9 @@ export default class extends Controller {
}
}
_isCSVFile(file) {
const acceptedTypes = ["text/csv", "application/csv", ".csv"]
_formatAcceptable(file) {
const extension = file.name.split('.').pop().toLowerCase()
return acceptedTypes.includes(file.type) || extension === "csv"
return this.acceptedTypesValue.includes(file.type) || extension === this.acceptedExtensionValue
}
_setFileInput(file) {

20
app/models/import.rb
View file

@ -1,7 +1,7 @@
class Import < ApplicationRecord
belongs_to :account
validate :raw_csv_must_be_parsable
validate :raw_file_must_be_parsable
validates :col_sep, inclusion: { in: Csv::COL_SEP_LIST }
before_save :initialize_csv, if: :should_initialize_csv?
@ -19,7 +19,7 @@ class Import < ApplicationRecord
end
def loaded?
raw_csv_str.present?
raw_file_str.present?
end
def configured?
@ -88,16 +88,16 @@ class Import < ApplicationRecord
end
def get_raw_csv
return nil if raw_csv_str.nil?
Import::Csv.new(raw_csv_str, col_sep:)
return nil if raw_file_str.nil?
Import::Csv.new(raw_file_str, col_sep:)
end
def should_initialize_csv?
raw_csv_str_changed? || column_mappings_changed?
raw_file_str_changed? || column_mappings_changed?
end
def initialize_csv
generated_csv = generate_normalized_csv(raw_csv_str)
generated_csv = generate_normalized_csv(raw_file_str)
self.normalized_csv_str = generated_csv.table.to_s
end
@ -175,12 +175,12 @@ class Import < ApplicationRecord
end
end
def raw_csv_must_be_parsable
def raw_file_must_be_parsable
begin
CSV.parse(raw_csv_str || "", col_sep:)
CSV.parse(raw_file_str || "", col_sep:)
rescue CSV::MalformedCSVError
# i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_csv_str.invalid_csv_format')
errors.add(:raw_csv_str, :invalid_csv_format)
# i18n-tasks-use t('activerecord.errors.models.import.attributes.raw_file_str.invalid_csv_format')
errors.add(:raw_file_str, :invalid_csv_format)
end
end
end

4
app/models/import/csv.rb
View file

@ -11,8 +11,8 @@ class Import::Csv
)
end
def self.create_with_field_mappings(raw_csv_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
raw_csv = self.parse_csv(raw_csv_str, col_sep:)
def self.create_with_field_mappings(raw_file_str, fields, field_mappings, col_sep = DEFAULT_COL_SEP)
raw_csv = self.parse_csv(raw_file_str, col_sep:)
generated_csv_str = CSV.generate headers: fields.map { |f| f.key }, write_headers: true, col_sep: do |csv|
raw_csv.each do |row|

4
app/views/imports/_csv_paste.html.erb
View file

@ -1,11 +1,11 @@
<%= styled_form_with model: @import, url: load_import_path(@import), class: "space-y-4" do |form| %>
<%= form.text_area :raw_csv_str,
<%= form.text_area :raw_file_str,
rows: 10,
required: true,
placeholder: "Paste your CSV file contents here",
class: "rounded-md w-full border text-sm border-alpha-black-100 bg-white placeholder:text-gray-400" %>
<%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
<%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-gray-900 text-white text-sm font-medium", data: { turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
<% end %>
<div class="bg-alpha-black-25 rounded-xl p-1 mt-5">

14
app/views/imports/_csv_upload.html.erb
View file

@ -1,24 +1,24 @@
<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "csv-upload" }, method: :patch, multipart: true do |form| %>
<%= styled_form_with model: @import, url: upload_import_path(@import), class: "dropzone space-y-4", data: { controller: "import-upload", import_upload_accepted_types_value: ["text/csv", "application/csv", ".csv"], import_upload_extension_value: "csv", import_upload_unacceptable_type_label_value: t(".allowed_filetypes") }, method: :patch, multipart: true do |form| %>
<div class="flex items-center justify-center w-full">
<label for="import_raw_csv_str" class="csv-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->csv-upload#dragover dragleave->csv-upload#dragleave drop->csv-upload#drop">
<label for="import_raw_file_str" class="raw-file-drop-box flex flex-col items-center justify-center w-full h-64 border-2 border-gray-300 border-dashed rounded-lg cursor-pointer bg-gray-50" data-action="dragover->import-upload#dragover dragleave->import-upload#dragleave drop->import-upload#drop click->import-upload#click">
<div class="flex flex-col items-center justify-center pt-5 pb-6">
<%= lucide_icon "plus", class: "w-5 h-5 text-gray-500" %>
<%= form.file_field :raw_csv_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { csv_upload_target: "input", action: "change->csv-upload#addFile" } %>
<%= form.file_field :raw_file_str, class: "hidden", direct_upload: false, accept: "text/csv,.csv,application/csv", data: { import_upload_target: "input", action: "change->import-upload#addFile" } %>
<p class="mb-2 text-sm text-gray-500 mt-3">Drag and drop your csv file here or <span class="text-black">click to browse</span></p>
<p class="text-xs text-gray-500">CSV (Max. 5MB)</p>
<div class="csv-preview" data-csv-upload-target="preview"></div>
<div class="csv-preview" data-import-upload-target="preview"></div>
</div>
</label>
</div>
<%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { csv_upload_target: "submit", turbo_confirm: (@import.raw_csv_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
<%= form.submit t(".next"), class: "px-4 py-2 mb-4 block w-full rounded-lg bg-alpha-black-25 text-gray text-sm font-medium", data: { import_upload_target: "submit", turbo_confirm: (@import.raw_file_str? ? { title: t(".confirm_title"), body: t(".confirm_body"), accept: t(".confirm_accept") } : nil) } %>
<% end %>
<div id="template-preview" class="hidden">
<div class="flex flex-col items-center justify-center">
<%= lucide_icon "file-text", class: "w-10 h-10 pt-2 text-black" %>
<div class="flex flex-row items-center justify-center gap-0.5">
<div><span data-csv-upload-target="filename"></span></div>
<div><span data-csv-upload-target="filesize" class="font-semibold"></span></div>
<div><span data-import-upload-target="filename"></span></div>
<div><span data-import-upload-target="filesize" class="font-semibold"></span></div>
</div>
</div>
</div>

2
config/locales/models/import/en.yml
View file

@ -5,5 +5,5 @@ en:
models:
import:
attributes:
raw_csv_str:
raw_file_str:
invalid_csv_format: is not a valid CSV format

1
config/locales/views/imports/en.yml
View file

@ -40,6 +40,7 @@ en:
"inflow" (income)
requirement3: Can have 0 or more tags separated by |
csv_upload:
allowed_filetypes: Only CSV files are allowed.
confirm_accept: Yep, start over!
confirm_body: This will reset your import. Any changes you have made to the
CSV will be erased.

5
db/migrate/20240817144454_rename_import_raw_csv_str_to_raw_file_str.rb Normal file
View file

@ -0,0 +1,5 @@
class RenameImportRawCsvStrToRawFileStr < ActiveRecord::Migration[7.2]
def change
rename_column :imports, :raw_csv_str, :raw_file_str
end
end

4
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do
ActiveRecord::Schema[7.2].define(version: 2024_08_17_144454) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@ -289,7 +289,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_08_16_071555) do
t.uuid "account_id", null: false
t.jsonb "column_mappings"
t.enum "status", default: "pending", enum_type: "import_status"
t.string "raw_csv_str"
t.string "raw_file_str"
t.string "normalized_csv_str"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false

18
test/controllers/imports_controller_test.rb
View file

@ -8,7 +8,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
@empty_import = imports(:empty_import)
@loaded_import = @empty_import.dup
@loaded_import.update! raw_csv_str: valid_csv_str
@loaded_import.update! raw_file_str: valid_csv_str
@completed_import = imports(:completed_import)
end
@ -59,7 +59,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
end
test "should save raw CSV if valid" do
patch load_import_url(@empty_import), params: { import: { raw_csv_str: valid_csv_str } }
patch load_import_url(@empty_import), params: { import: { raw_file_str: valid_csv_str } }
assert_redirected_to configure_import_path(@empty_import)
assert_equal "Import CSV loaded", flash[:notice]
@ -71,17 +71,17 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
valid_csv_str.split("\n").each { |row| csv << row.split(",") }
end
patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
assert_redirected_to configure_import_path(@empty_import)
assert_equal "CSV File loaded", flash[:notice]
end
end
test "should flash error message if invalid CSV input" do
patch load_import_url(@empty_import), params: { import: { raw_csv_str: malformed_csv_str } }
patch load_import_url(@empty_import), params: { import: { raw_file_str: malformed_csv_str } }
assert_response :unprocessable_entity
assert_equal "Raw csv str is not a valid CSV format", flash[:alert]
assert_equal "Raw file str is not a valid CSV format", flash[:alert]
end
test "should flash error message if invalid CSV file upload" do
@ -89,14 +89,14 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
temp.write(malformed_csv_str)
temp.rewind
patch upload_import_url(@empty_import), params: { import: { raw_csv_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
patch upload_import_url(@empty_import), params: { import: { raw_file_str: Rack::Test::UploadedFile.new(temp, ".csv") } }
assert_response :unprocessable_entity
assert_equal "Raw csv str is not a valid CSV format", flash[:alert]
assert_equal "Raw file str is not a valid CSV format", flash[:alert]
end
end
test "should flash error message if no fileprovided for upload" do
patch upload_import_url(@empty_import), params: { import: { raw_csv_str: nil } }
patch upload_import_url(@empty_import), params: { import: { raw_file_str: nil } }
assert_response :unprocessable_entity
assert_equal "Please select a file to upload", flash[:alert]
end
@ -158,7 +158,7 @@ class ImportsControllerTest < ActionDispatch::IntegrationTest
end
test "should redirect back to clean if data is invalid" do
@empty_import.update! raw_csv_str: valid_csv_with_invalid_values
@empty_import.update! raw_file_str: valid_csv_with_invalid_values
get confirm_import_url(@empty_import)
assert_equal "You have invalid data, please fix before continuing", flash[:alert]

2
test/fixtures/imports.yml vendored
View file

@ -9,7 +9,7 @@ completed_import:
name: name
category: category
amount: amount
raw_csv_str: |
raw_file_str: |
date,name,category,tags,amount
2024-01-01,Starbucks drink,Food & Drink,Test Tag,-20
normalized_csv_str: |

2
test/jobs/import_job_test.rb
View file

@ -5,7 +5,7 @@ class ImportJobTest < ActiveJob::TestCase
test "import is published" do
import = imports(:empty_import)
import.update! raw_csv_str: valid_csv_str
import.update! raw_file_str: valid_csv_str
assert import.pending?

8
test/models/import/csv_test.rb
View file

@ -72,7 +72,7 @@ class Import::CsvTest < ActiveSupport::TestCase
fields = [ date_field, name_field ]
raw_csv_str = <<-ROWS
raw_file_str = <<-ROWS
date,Custom Field Header,extra_field
invalid_date_value,Starbucks drink,Food
2024-01-02,Amazon stuff,Shopping
@ -82,7 +82,7 @@ class Import::CsvTest < ActiveSupport::TestCase
"name" => "Custom Field Header"
}
csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings)
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings)
assert_equal %w[ date name ], csv.table.headers
assert_equal 2, csv.table.size
@ -101,7 +101,7 @@ class Import::CsvTest < ActiveSupport::TestCase
fields = [ date_field, name_field ]
raw_csv_str = <<-ROWS
raw_file_str = <<-ROWS
date;Custom Field Header;extra_field
invalid_date_value;Starbucks drink;Food
2024-01-02;Amazon stuff;Shopping
@ -111,7 +111,7 @@ class Import::CsvTest < ActiveSupport::TestCase
"name" => "Custom Field Header"
}
csv = Import::Csv.create_with_field_mappings(raw_csv_str, fields, mappings, ";")
csv = Import::Csv.create_with_field_mappings(raw_file_str, fields, mappings, ";")
assert_equal %w[ date name ], csv.table.headers
assert_equal 2, csv.table.size

16
test/models/import_test.rb
View file

@ -7,7 +7,7 @@ class ImportTest < ActiveSupport::TestCase
@empty_import = imports(:empty_import)
@loaded_import = @empty_import.dup
@loaded_import.update! raw_csv_str: valid_csv_str
@loaded_import.update! raw_file_str: valid_csv_str
end
test "validates the correct col_sep" do
@ -26,17 +26,17 @@ class ImportTest < ActiveSupport::TestCase
end
test "raw csv input must conform to csv spec" do
@empty_import.raw_csv_str = malformed_csv_str
@empty_import.raw_file_str = malformed_csv_str
assert_not @empty_import.valid?
@empty_import.raw_csv_str = valid_csv_str
@empty_import.raw_file_str = valid_csv_str
assert @empty_import.valid?
end
test "can update csv value without affecting raw input" do
assert_equal "Starbucks drink", @loaded_import.csv.table[0][1]
prior_raw_csv_str_value = @loaded_import.raw_csv_str
prior_raw_file_str_value = @loaded_import.raw_file_str
prior_normalized_csv_str_value = @loaded_import.normalized_csv_str
@loaded_import.update_csv! \
@ -45,7 +45,7 @@ class ImportTest < ActiveSupport::TestCase
value: "new_category"
assert_equal "new_category", @loaded_import.csv.table[0][1]
assert_equal prior_raw_csv_str_value, @loaded_import.raw_csv_str
assert_equal prior_raw_file_str_value, @loaded_import.raw_file_str
assert_not_equal prior_normalized_csv_str_value, @loaded_import.normalized_csv_str
end
@ -74,7 +74,7 @@ class ImportTest < ActiveSupport::TestCase
end
test "publishes a valid import with missing data" do
@empty_import.update! raw_csv_str: valid_csv_with_missing_data
@empty_import.update! raw_file_str: valid_csv_with_missing_data
assert_difference -> { Category.count } => 1,
-> { Account::Transaction.count } => 2,
-> { Account::Entry.count } => 2 do
@ -89,7 +89,7 @@ class ImportTest < ActiveSupport::TestCase
end
test "failed publish results in error status" do
@empty_import.update! raw_csv_str: valid_csv_with_invalid_values
@empty_import.update! raw_file_str: valid_csv_with_invalid_values
assert_difference "Account::Transaction.count", 0 do
@empty_import.publish
@ -102,7 +102,7 @@ class ImportTest < ActiveSupport::TestCase
test "can create transactions from csv with custom column separator" do
loaded_import = @empty_import.dup
loaded_import.update! raw_csv_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
loaded_import.update! raw_file_str: valid_csv_str_with_semicolon_separator, col_sep: ";"
transactions = loaded_import.dry_run
assert_equal 4, transactions.count

4
test/system/imports_test.rb
View file

@ -57,7 +57,7 @@ class ImportsTest < ApplicationSystemTestCase
assert_selector "h1", text: "Load import"
within "form" do
fill_in "import_raw_csv_str", with: <<-ROWS
fill_in "import_raw_file_str", with: <<-ROWS
date,Custom Name Column,category,amount
invalid_date,Starbucks drink,Food,-20.50
2024-01-01,Amazon purchase,Shopping,-89.50
@ -115,7 +115,7 @@ class ImportsTest < ApplicationSystemTestCase
click_button "Upload CSV"
find(".csv-drop-box").drop File.join(file_fixture_path, "transactions.csv")
find(".raw-file-drop-box").drop File.join(file_fixture_path, "transactions.csv")
assert_selector "div.csv-preview", text: "transactions.csv"
click_button "Next"