1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-01 19:45:19 +02:00

Data exports (#2517)

* Import / export UI

* Data exports

* Lint fixes, brakeman update
This commit is contained in:
Zach Gollwitzer 2025-07-24 10:50:05 -04:00 committed by GitHub
parent b7c56e2fb7
commit 0329a5f211
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
20 changed files with 717 additions and 2 deletions

View file

@ -72,6 +72,7 @@ gem "plaid"
gem "rotp", "~> 6.3"
gem "rqrcode", "~> 3.0"
gem "activerecord-import"
gem "rubyzip", "~> 2.3"
# State machines
gem "aasm"

View file

@ -672,6 +672,7 @@ DEPENDENCIES
rubocop-rails-omakase
ruby-lsp-rails
ruby-openai
rubyzip (~> 2.3)
selenium-webdriver
sentry-rails
sentry-ruby

View file

@ -0,0 +1,47 @@
class FamilyExportsController < ApplicationController
include StreamExtensions
before_action :require_admin
before_action :set_export, only: [ :download ]
def new
# Modal view for initiating export
end
def create
@export = Current.family.family_exports.create!
FamilyDataExportJob.perform_later(@export)
respond_to do |format|
format.html { redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly." }
format.turbo_stream {
stream_redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly."
}
end
end
def index
@exports = Current.family.family_exports.ordered.limit(10)
render layout: false # For turbo frame
end
def download
if @export.downloadable?
redirect_to @export.export_file, allow_other_host: true
else
redirect_to settings_profile_path, alert: "Export not ready for download"
end
end
private
def set_export
@export = Current.family.family_exports.find(params[:id])
end
def require_admin
unless Current.user.admin?
redirect_to root_path, alert: "Access denied"
end
end
end

View file

@ -0,0 +1,22 @@
class FamilyDataExportJob < ApplicationJob
queue_as :default
def perform(family_export)
family_export.update!(status: :processing)
exporter = Family::DataExporter.new(family_export.family)
zip_file = exporter.generate_export
family_export.export_file.attach(
io: zip_file,
filename: family_export.filename,
content_type: "application/zip"
)
family_export.update!(status: :completed)
rescue => e
Rails.logger.error "Family export failed: #{e.message}"
Rails.logger.error e.backtrace.join("\n")
family_export.update!(status: :failed)
end
end

View file

@ -18,6 +18,7 @@ class Family < ApplicationRecord
has_many :invitations, dependent: :destroy
has_many :imports, dependent: :destroy
has_many :family_exports, dependent: :destroy
has_many :entries, through: :accounts
has_many :transactions, through: :accounts

View file

@ -0,0 +1,238 @@
require "zip"
require "csv"
class Family::DataExporter
def initialize(family)
@family = family
end
def generate_export
# Create a StringIO to hold the zip data in memory
zip_data = Zip::OutputStream.write_buffer do |zipfile|
# Add accounts.csv
zipfile.put_next_entry("accounts.csv")
zipfile.write generate_accounts_csv
# Add transactions.csv
zipfile.put_next_entry("transactions.csv")
zipfile.write generate_transactions_csv
# Add trades.csv
zipfile.put_next_entry("trades.csv")
zipfile.write generate_trades_csv
# Add categories.csv
zipfile.put_next_entry("categories.csv")
zipfile.write generate_categories_csv
# Add all.ndjson
zipfile.put_next_entry("all.ndjson")
zipfile.write generate_ndjson
end
# Rewind and return the StringIO
zip_data.rewind
zip_data
end
private
def generate_accounts_csv
CSV.generate do |csv|
csv << [ "id", "name", "type", "subtype", "balance", "currency", "created_at" ]
# Only export accounts belonging to this family
@family.accounts.includes(:accountable).find_each do |account|
csv << [
account.id,
account.name,
account.accountable_type,
account.subtype,
account.balance.to_s,
account.currency,
account.created_at.iso8601
]
end
end
end
def generate_transactions_csv
CSV.generate do |csv|
csv << [ "date", "account_name", "amount", "name", "category", "tags", "notes", "currency" ]
# Only export transactions from accounts belonging to this family
@family.transactions
.includes(:category, :tags, entry: :account)
.find_each do |transaction|
csv << [
transaction.entry.date.iso8601,
transaction.entry.account.name,
transaction.entry.amount.to_s,
transaction.entry.name,
transaction.category&.name,
transaction.tags.pluck(:name).join(","),
transaction.entry.notes,
transaction.entry.currency
]
end
end
end
def generate_trades_csv
CSV.generate do |csv|
csv << [ "date", "account_name", "ticker", "quantity", "price", "amount", "currency" ]
# Only export trades from accounts belonging to this family
@family.trades
.includes(:security, entry: :account)
.find_each do |trade|
csv << [
trade.entry.date.iso8601,
trade.entry.account.name,
trade.security.ticker,
trade.qty.to_s,
trade.price.to_s,
trade.entry.amount.to_s,
trade.currency
]
end
end
end
def generate_categories_csv
CSV.generate do |csv|
csv << [ "name", "color", "parent_category", "classification" ]
# Only export categories belonging to this family
@family.categories.includes(:parent).find_each do |category|
csv << [
category.name,
category.color,
category.parent&.name,
category.classification
]
end
end
end
def generate_ndjson
lines = []
# Export accounts with full accountable data
@family.accounts.includes(:accountable).find_each do |account|
lines << {
type: "Account",
data: account.as_json(
include: {
accountable: {}
}
)
}.to_json
end
# Export categories
@family.categories.find_each do |category|
lines << {
type: "Category",
data: category.as_json
}.to_json
end
# Export tags
@family.tags.find_each do |tag|
lines << {
type: "Tag",
data: tag.as_json
}.to_json
end
# Export merchants (only family merchants)
@family.merchants.find_each do |merchant|
lines << {
type: "Merchant",
data: merchant.as_json
}.to_json
end
# Export transactions with full data
@family.transactions.includes(:category, :merchant, :tags, entry: :account).find_each do |transaction|
lines << {
type: "Transaction",
data: {
id: transaction.id,
entry_id: transaction.entry.id,
account_id: transaction.entry.account_id,
date: transaction.entry.date,
amount: transaction.entry.amount,
currency: transaction.entry.currency,
name: transaction.entry.name,
notes: transaction.entry.notes,
excluded: transaction.entry.excluded,
category_id: transaction.category_id,
merchant_id: transaction.merchant_id,
tag_ids: transaction.tag_ids,
kind: transaction.kind,
created_at: transaction.created_at,
updated_at: transaction.updated_at
}
}.to_json
end
# Export trades with full data
@family.trades.includes(:security, entry: :account).find_each do |trade|
lines << {
type: "Trade",
data: {
id: trade.id,
entry_id: trade.entry.id,
account_id: trade.entry.account_id,
security_id: trade.security_id,
ticker: trade.security.ticker,
date: trade.entry.date,
qty: trade.qty,
price: trade.price,
amount: trade.entry.amount,
currency: trade.currency,
created_at: trade.created_at,
updated_at: trade.updated_at
}
}.to_json
end
# Export valuations
@family.entries.valuations.includes(:account, :entryable).find_each do |entry|
lines << {
type: "Valuation",
data: {
id: entry.entryable.id,
entry_id: entry.id,
account_id: entry.account_id,
date: entry.date,
amount: entry.amount,
currency: entry.currency,
name: entry.name,
created_at: entry.created_at,
updated_at: entry.updated_at
}
}.to_json
end
# Export budgets
@family.budgets.find_each do |budget|
lines << {
type: "Budget",
data: budget.as_json
}.to_json
end
# Export budget categories
@family.budget_categories.includes(:budget, :category).find_each do |budget_category|
lines << {
type: "BudgetCategory",
data: budget_category.as_json
}.to_json
end
lines.join("\n")
end
end

View file

@ -0,0 +1,22 @@
class FamilyExport < ApplicationRecord
belongs_to :family
has_one_attached :export_file
enum :status, {
pending: "pending",
processing: "processing",
completed: "completed",
failed: "failed"
}, default: :pending, validate: true
scope :ordered, -> { order(created_at: :desc) }
def filename
"maybe_export_#{created_at.strftime('%Y%m%d_%H%M%S')}.zip"
end
def downloadable?
completed? && export_file.attached?
end
end

View file

@ -0,0 +1,39 @@
<%= turbo_frame_tag "family_exports",
data: exports.any? { |e| e.pending? || e.processing? } ? {
turbo_refresh_url: family_exports_path,
turbo_refresh_interval: 3000
} : {} do %>
<div class="mt-4 space-y-3 max-h-96 overflow-y-auto">
<% if exports.any? %>
<% exports.each do |export| %>
<div class="flex items-center justify-between bg-container p-4 rounded-lg border border-primary">
<div>
<p class="text-sm font-medium text-primary">Export from <%= export.created_at.strftime("%B %d, %Y at %I:%M %p") %></p>
<p class="text-xs text-secondary"><%= export.filename %></p>
</div>
<% if export.processing? || export.pending? %>
<div class="flex items-center gap-2 text-secondary">
<div class="animate-spin h-4 w-4 border-2 border-secondary border-t-transparent rounded-full"></div>
<span class="text-sm">Exporting...</span>
</div>
<% elsif export.completed? %>
<%= link_to download_family_export_path(export),
class: "flex items-center gap-2 text-primary hover:text-primary-hover",
data: { turbo_frame: "_top" } do %>
<%= icon "download", class: "w-5 h-5" %>
<span class="text-sm font-medium">Download</span>
<% end %>
<% elsif export.failed? %>
<div class="flex items-center gap-2 text-destructive">
<%= icon "alert-circle", class: "w-4 h-4" %>
<span class="text-sm">Failed</span>
</div>
<% end %>
</div>
<% end %>
<% else %>
<p class="text-sm text-secondary text-center py-4">No exports yet</p>
<% end %>
</div>
<% end %>

View file

@ -0,0 +1 @@
<%= render "list", exports: @exports %>

View file

@ -0,0 +1,42 @@
<%= render DS::Dialog.new do |dialog| %>
<% dialog.with_header(title: "Export your data", subtitle: "Download all your financial data") %>
<% dialog.with_body do %>
<div class="space-y-4">
<div class="bg-container-inset rounded-lg p-4 space-y-3">
<h3 class="font-medium text-primary">What's included:</h3>
<ul class="space-y-2 text-sm text-secondary">
<li class="flex items-start gap-2">
<%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
<span>All accounts and balances</span>
</li>
<li class="flex items-start gap-2">
<%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
<span>Transaction history</span>
</li>
<li class="flex items-start gap-2">
<%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
<span>Investment trades</span>
</li>
<li class="flex items-start gap-2">
<%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
<span>Categories and tags</span>
</li>
</ul>
</div>
<div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
<p class="text-sm text-amber-800">
<strong>Note:</strong> This export includes all of your data, but only some of the data can be imported back into Maybe via the CSV import feature. We support account, transaction (with category and tags), and trade imports. Other account data cannot be imported and is for your records only.
</p>
</div>
<%= form_with url: family_exports_path, method: :post, class: "space-y-4" do |form| %>
<div class="flex gap-3">
<%= link_to "Cancel", "#", class: "flex-1 text-center px-4 py-2 border border-primary rounded-lg hover:bg-surface-hover", data: { action: "click->modal#close" } %>
<%= form.submit "Export data", class: "flex-1 bg-inverse fg-inverse rounded-lg px-4 py-2 cursor-pointer" %>
</div>
<% end %>
</div>
<% end %>
<% end %>

View file

@ -122,6 +122,29 @@
</div>
<% end %>
<% if Current.user.admin? %>
<%= settings_section title: "Data Import/Export" do %>
<div class="space-y-4">
<div class="flex gap-4 items-center">
<%= render DS::Link.new(
text: "Export data",
icon: "database",
href: new_family_export_path,
variant: "secondary",
full_width: true,
data: { turbo_frame: :modal }
) %>
</div>
<%= turbo_frame_tag "family_exports", src: family_exports_path, loading: :lazy do %>
<div class="mt-4 text-center text-secondary">
<div class="animate-spin inline-block h-4 w-4 border-2 border-secondary border-t-transparent rounded-full"></div>
</div>
<% end %>
</div>
<% end %>
<% end %>
<%= settings_section title: t(".danger_zone_title") do %>
<div class="space-y-4">
<% if Current.user.admin? %>

View file

@ -1,5 +1,28 @@
{
"ignored_warnings": [
{
"warning_type": "Redirect",
"warning_code": 18,
"fingerprint": "723b1970ca6bf16ea0c2c1afa0c00d3c54854a16568d6cb933e497947565d9ab",
"check_name": "Redirect",
"message": "Possible unprotected redirect",
"file": "app/controllers/family_exports_controller.rb",
"line": 30,
"link": "https://brakemanscanner.org/docs/warning_types/redirect/",
"code": "redirect_to(Current.family.family_exports.find(params[:id]).export_file, :allow_other_host => true)",
"render_path": null,
"location": {
"type": "method",
"class": "FamilyExportsController",
"method": "download"
},
"user_input": "Current.family.family_exports.find(params[:id]).export_file",
"confidence": "Weak",
"cwe_id": [
601
],
"note": ""
},
{
"warning_type": "Mass Assignment",
"warning_code": 105,
@ -105,5 +128,5 @@
"note": ""
}
],
"brakeman_version": "7.0.2"
"brakeman_version": "7.1.0"
}

View file

@ -24,6 +24,12 @@ Rails.application.routes.draw do
end
end
resources :family_exports, only: %i[new create index] do
member do
get :download
end
end
get "changelog", to: "pages#changelog"
get "feedback", to: "pages#feedback"

View file

@ -0,0 +1,10 @@
class CreateFamilyExports < ActiveRecord::Migration[7.2]
def change
create_table :family_exports, id: :uuid do |t|
t.references :family, null: false, foreign_key: true, type: :uuid
t.string :status, default: "pending", null: false
t.timestamps
end
end
end

11
db/schema.rb generated
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2025_07_19_121103) do
ActiveRecord::Schema[7.2].define(version: 2025_07_24_115507) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@ -270,6 +270,14 @@ ActiveRecord::Schema[7.2].define(version: 2025_07_19_121103) do
t.datetime "latest_sync_completed_at", default: -> { "CURRENT_TIMESTAMP" }
end
create_table "family_exports", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
t.uuid "family_id", null: false
t.string "status", default: "pending", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["family_id"], name: "index_family_exports_on_family_id"
end
create_table "holdings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
t.uuid "account_id", null: false
t.uuid "security_id", null: false
@ -830,6 +838,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_07_19_121103) do
add_foreign_key "chats", "users"
add_foreign_key "entries", "accounts"
add_foreign_key "entries", "imports"
add_foreign_key "family_exports", "families"
add_foreign_key "holdings", "accounts"
add_foreign_key "holdings", "securities"
add_foreign_key "impersonation_session_logs", "impersonation_sessions"

View file

@ -0,0 +1,73 @@
require "test_helper"
class FamilyExportsControllerTest < ActionDispatch::IntegrationTest
setup do
@admin = users(:family_admin)
@non_admin = users(:family_member)
@family = @admin.family
sign_in @admin
end
test "non-admin cannot access exports" do
sign_in @non_admin
get new_family_export_path
assert_redirected_to root_path
post family_exports_path
assert_redirected_to root_path
get family_exports_path
assert_redirected_to root_path
end
test "admin can view export modal" do
get new_family_export_path
assert_response :success
assert_select "h2", text: "Export your data"
end
test "admin can create export" do
assert_enqueued_with(job: FamilyDataExportJob) do
post family_exports_path
end
assert_redirected_to settings_profile_path
assert_equal "Export started. You'll be able to download it shortly.", flash[:notice]
export = @family.family_exports.last
assert_equal "pending", export.status
end
test "admin can view export list" do
export1 = @family.family_exports.create!(status: "completed")
export2 = @family.family_exports.create!(status: "processing")
get family_exports_path
assert_response :success
assert_match export1.filename, response.body
assert_match "Exporting...", response.body
end
test "admin can download completed export" do
export = @family.family_exports.create!(status: "completed")
export.export_file.attach(
io: StringIO.new("test zip content"),
filename: "test.zip",
content_type: "application/zip"
)
get download_family_export_path(export)
assert_redirected_to(/rails\/active_storage/)
end
test "cannot download incomplete export" do
export = @family.family_exports.create!(status: "processing")
get download_family_export_path(export)
assert_redirected_to settings_profile_path
assert_equal "Export not ready for download", flash[:alert]
end
end

3
test/fixtures/family_exports.yml vendored Normal file
View file

@ -0,0 +1,3 @@
# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
# Empty file - no fixtures needed, tests create them dynamically

View file

@ -0,0 +1,32 @@
require "test_helper"
class FamilyDataExportJobTest < ActiveJob::TestCase
setup do
@family = families(:dylan_family)
@export = @family.family_exports.create!
end
test "marks export as processing then completed" do
assert_equal "pending", @export.status
perform_enqueued_jobs do
FamilyDataExportJob.perform_later(@export)
end
@export.reload
assert_equal "completed", @export.status
assert @export.export_file.attached?
end
test "marks export as failed on error" do
# Mock the exporter to raise an error
Family::DataExporter.any_instance.stubs(:generate_export).raises(StandardError, "Export failed")
perform_enqueued_jobs do
FamilyDataExportJob.perform_later(@export)
end
@export.reload
assert_equal "failed", @export.status
end
end

View file

@ -0,0 +1,115 @@
require "test_helper"
class Family::DataExporterTest < ActiveSupport::TestCase
setup do
@family = families(:dylan_family)
@other_family = families(:empty)
@exporter = Family::DataExporter.new(@family)
# Create some test data for the family
@account = @family.accounts.create!(
name: "Test Account",
accountable: Depository.new,
balance: 1000,
currency: "USD"
)
@category = @family.categories.create!(
name: "Test Category",
color: "#FF0000"
)
@tag = @family.tags.create!(
name: "Test Tag",
color: "#00FF00"
)
end
test "generates a zip file with all required files" do
zip_data = @exporter.generate_export
assert zip_data.is_a?(StringIO)
# Check that the zip contains all expected files
expected_files = [ "accounts.csv", "transactions.csv", "trades.csv", "categories.csv", "all.ndjson" ]
Zip::File.open_buffer(zip_data) do |zip|
actual_files = zip.entries.map(&:name)
assert_equal expected_files.sort, actual_files.sort
end
end
test "generates valid CSV files" do
zip_data = @exporter.generate_export
Zip::File.open_buffer(zip_data) do |zip|
# Check accounts.csv
accounts_csv = zip.read("accounts.csv")
assert accounts_csv.include?("id,name,type,subtype,balance,currency,created_at")
# Check transactions.csv
transactions_csv = zip.read("transactions.csv")
assert transactions_csv.include?("date,account_name,amount,name,category,tags,notes,currency")
# Check trades.csv
trades_csv = zip.read("trades.csv")
assert trades_csv.include?("date,account_name,ticker,quantity,price,amount,currency")
# Check categories.csv
categories_csv = zip.read("categories.csv")
assert categories_csv.include?("name,color,parent_category,classification")
end
end
test "generates valid NDJSON file" do
zip_data = @exporter.generate_export
Zip::File.open_buffer(zip_data) do |zip|
ndjson_content = zip.read("all.ndjson")
lines = ndjson_content.split("\n")
lines.each do |line|
assert_nothing_raised { JSON.parse(line) }
end
# Check that each line has expected structure
first_line = JSON.parse(lines.first)
assert first_line.key?("type")
assert first_line.key?("data")
end
end
test "only exports data from the specified family" do
# Create data for another family that should NOT be exported
other_account = @other_family.accounts.create!(
name: "Other Family Account",
accountable: Depository.new,
balance: 5000,
currency: "USD"
)
other_category = @other_family.categories.create!(
name: "Other Family Category",
color: "#0000FF"
)
zip_data = @exporter.generate_export
Zip::File.open_buffer(zip_data) do |zip|
# Check accounts.csv doesn't contain other family's data
accounts_csv = zip.read("accounts.csv")
assert accounts_csv.include?(@account.name)
refute accounts_csv.include?(other_account.name)
# Check categories.csv doesn't contain other family's data
categories_csv = zip.read("categories.csv")
assert categories_csv.include?(@category.name)
refute categories_csv.include?(other_category.name)
# Check NDJSON doesn't contain other family's data
ndjson_content = zip.read("all.ndjson")
refute ndjson_content.include?(other_account.id)
refute ndjson_content.include?(other_category.id)
end
end
end

View file

@ -0,0 +1,7 @@
require "test_helper"
class FamilyExportTest < ActiveSupport::TestCase
# test "the truth" do
# assert true
# end
end