Data exports (#2517)

* Import / export UI

* Data exports

* Lint fixes, brakeman update

app/controllers/family_exports_controller.rb

@@ -0,0 +1,47 @@
+class FamilyExportsController < ApplicationController
+  include StreamExtensions
+
+  before_action :require_admin
+  before_action :set_export, only: [ :download ]
+
+  def new
+    # Modal view for initiating export
+  end
+
+  def create
+    @export = Current.family.family_exports.create!
+    FamilyDataExportJob.perform_later(@export)
+
+    respond_to do |format|
+      format.html { redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly." }
+      format.turbo_stream {
+        stream_redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly."
+      }
+    end
+  end
+
+  def index
+    @exports = Current.family.family_exports.ordered.limit(10)
+    render layout: false # For turbo frame
+  end
+
+  def download
+    if @export.downloadable?
+      redirect_to @export.export_file, allow_other_host: true
+    else
+      redirect_to settings_profile_path, alert: "Export not ready for download"
+    end
+  end
+
+  private
+
+    def set_export
+      @export = Current.family.family_exports.find(params[:id])
+    end
+
+    def require_admin
+      unless Current.user.admin?
+        redirect_to root_path, alert: "Access denied"
+      end
+    end
+end

app/jobs/family_data_export_job.rb

@@ -0,0 +1,22 @@
+class FamilyDataExportJob < ApplicationJob
+  queue_as :default
+
+  def perform(family_export)
+    family_export.update!(status: :processing)
+
+    exporter = Family::DataExporter.new(family_export.family)
+    zip_file = exporter.generate_export
+
+    family_export.export_file.attach(
+      io: zip_file,
+      filename: family_export.filename,
+      content_type: "application/zip"
+    )
+
+    family_export.update!(status: :completed)
+  rescue => e
+    Rails.logger.error "Family export failed: #{e.message}"
+    Rails.logger.error e.backtrace.join("\n")
+    family_export.update!(status: :failed)
+  end
+end

app/models/family.rb

@@ -18,6 +18,7 @@ class Family < ApplicationRecord
   has_many :invitations, dependent: :destroy
 
   has_many :imports, dependent: :destroy
+  has_many :family_exports, dependent: :destroy
 
   has_many :entries, through: :accounts
   has_many :transactions, through: :accounts

app/models/family/data_exporter.rb

@@ -0,0 +1,238 @@
+require "zip"
+require "csv"
+
+class Family::DataExporter
+  def initialize(family)
+    @family = family
+  end
+
+  def generate_export
+    # Create a StringIO to hold the zip data in memory
+    zip_data = Zip::OutputStream.write_buffer do |zipfile|
+      # Add accounts.csv
+      zipfile.put_next_entry("accounts.csv")
+      zipfile.write generate_accounts_csv
+
+      # Add transactions.csv
+      zipfile.put_next_entry("transactions.csv")
+      zipfile.write generate_transactions_csv
+
+      # Add trades.csv
+      zipfile.put_next_entry("trades.csv")
+      zipfile.write generate_trades_csv
+
+      # Add categories.csv
+      zipfile.put_next_entry("categories.csv")
+      zipfile.write generate_categories_csv
+
+      # Add all.ndjson
+      zipfile.put_next_entry("all.ndjson")
+      zipfile.write generate_ndjson
+    end
+
+    # Rewind and return the StringIO
+    zip_data.rewind
+    zip_data
+  end
+
+  private
+
+    def generate_accounts_csv
+      CSV.generate do |csv|
+        csv << [ "id", "name", "type", "subtype", "balance", "currency", "created_at" ]
+
+        # Only export accounts belonging to this family
+        @family.accounts.includes(:accountable).find_each do |account|
+          csv << [
+            account.id,
+            account.name,
+            account.accountable_type,
+            account.subtype,
+            account.balance.to_s,
+            account.currency,
+            account.created_at.iso8601
+          ]
+        end
+      end
+    end
+
+    def generate_transactions_csv
+      CSV.generate do |csv|
+        csv << [ "date", "account_name", "amount", "name", "category", "tags", "notes", "currency" ]
+
+        # Only export transactions from accounts belonging to this family
+        @family.transactions
+          .includes(:category, :tags, entry: :account)
+          .find_each do |transaction|
+            csv << [
+              transaction.entry.date.iso8601,
+              transaction.entry.account.name,
+              transaction.entry.amount.to_s,
+              transaction.entry.name,
+              transaction.category&.name,
+              transaction.tags.pluck(:name).join(","),
+              transaction.entry.notes,
+              transaction.entry.currency
+            ]
+          end
+      end
+    end
+
+    def generate_trades_csv
+      CSV.generate do |csv|
+        csv << [ "date", "account_name", "ticker", "quantity", "price", "amount", "currency" ]
+
+        # Only export trades from accounts belonging to this family
+        @family.trades
+          .includes(:security, entry: :account)
+          .find_each do |trade|
+            csv << [
+              trade.entry.date.iso8601,
+              trade.entry.account.name,
+              trade.security.ticker,
+              trade.qty.to_s,
+              trade.price.to_s,
+              trade.entry.amount.to_s,
+              trade.currency
+            ]
+          end
+      end
+    end
+
+    def generate_categories_csv
+      CSV.generate do |csv|
+        csv << [ "name", "color", "parent_category", "classification" ]
+
+        # Only export categories belonging to this family
+        @family.categories.includes(:parent).find_each do |category|
+          csv << [
+            category.name,
+            category.color,
+            category.parent&.name,
+            category.classification
+          ]
+        end
+      end
+    end
+
+    def generate_ndjson
+      lines = []
+
+      # Export accounts with full accountable data
+      @family.accounts.includes(:accountable).find_each do |account|
+        lines << {
+          type: "Account",
+          data: account.as_json(
+            include: {
+              accountable: {}
+            }
+          )
+        }.to_json
+      end
+
+      # Export categories
+      @family.categories.find_each do |category|
+        lines << {
+          type: "Category",
+          data: category.as_json
+        }.to_json
+      end
+
+      # Export tags
+      @family.tags.find_each do |tag|
+        lines << {
+          type: "Tag",
+          data: tag.as_json
+        }.to_json
+      end
+
+      # Export merchants (only family merchants)
+      @family.merchants.find_each do |merchant|
+        lines << {
+          type: "Merchant",
+          data: merchant.as_json
+        }.to_json
+      end
+
+      # Export transactions with full data
+      @family.transactions.includes(:category, :merchant, :tags, entry: :account).find_each do |transaction|
+        lines << {
+          type: "Transaction",
+          data: {
+            id: transaction.id,
+            entry_id: transaction.entry.id,
+            account_id: transaction.entry.account_id,
+            date: transaction.entry.date,
+            amount: transaction.entry.amount,
+            currency: transaction.entry.currency,
+            name: transaction.entry.name,
+            notes: transaction.entry.notes,
+            excluded: transaction.entry.excluded,
+            category_id: transaction.category_id,
+            merchant_id: transaction.merchant_id,
+            tag_ids: transaction.tag_ids,
+            kind: transaction.kind,
+            created_at: transaction.created_at,
+            updated_at: transaction.updated_at
+          }
+        }.to_json
+      end
+
+      # Export trades with full data
+      @family.trades.includes(:security, entry: :account).find_each do |trade|
+        lines << {
+          type: "Trade",
+          data: {
+            id: trade.id,
+            entry_id: trade.entry.id,
+            account_id: trade.entry.account_id,
+            security_id: trade.security_id,
+            ticker: trade.security.ticker,
+            date: trade.entry.date,
+            qty: trade.qty,
+            price: trade.price,
+            amount: trade.entry.amount,
+            currency: trade.currency,
+            created_at: trade.created_at,
+            updated_at: trade.updated_at
+          }
+        }.to_json
+      end
+
+      # Export valuations
+      @family.entries.valuations.includes(:account, :entryable).find_each do |entry|
+        lines << {
+          type: "Valuation",
+          data: {
+            id: entry.entryable.id,
+            entry_id: entry.id,
+            account_id: entry.account_id,
+            date: entry.date,
+            amount: entry.amount,
+            currency: entry.currency,
+            name: entry.name,
+            created_at: entry.created_at,
+            updated_at: entry.updated_at
+          }
+        }.to_json
+      end
+
+      # Export budgets
+      @family.budgets.find_each do |budget|
+        lines << {
+          type: "Budget",
+          data: budget.as_json
+        }.to_json
+      end
+
+      # Export budget categories
+      @family.budget_categories.includes(:budget, :category).find_each do |budget_category|
+        lines << {
+          type: "BudgetCategory",
+          data: budget_category.as_json
+        }.to_json
+      end
+
+      lines.join("\n")
+    end
+end

app/models/family_export.rb

@@ -0,0 +1,22 @@
+class FamilyExport < ApplicationRecord
+  belongs_to :family
+
+  has_one_attached :export_file
+
+  enum :status, {
+    pending: "pending",
+    processing: "processing",
+    completed: "completed",
+    failed: "failed"
+  }, default: :pending, validate: true
+
+  scope :ordered, -> { order(created_at: :desc) }
+
+  def filename
+    "maybe_export_#{created_at.strftime('%Y%m%d_%H%M%S')}.zip"
+  end
+
+  def downloadable?
+    completed? && export_file.attached?
+  end
+end

app/views/family_exports/_list.html.erb

@@ -0,0 +1,39 @@
+<%= turbo_frame_tag "family_exports",
+    data: exports.any? { |e| e.pending? || e.processing? } ? {
+      turbo_refresh_url: family_exports_path,
+      turbo_refresh_interval: 3000
+    } : {} do %>
+  <div class="mt-4 space-y-3 max-h-96 overflow-y-auto">
+    <% if exports.any? %>
+      <% exports.each do |export| %>
+        <div class="flex items-center justify-between bg-container p-4 rounded-lg border border-primary">
+          <div>
+            <p class="text-sm font-medium text-primary">Export from <%= export.created_at.strftime("%B %d, %Y at %I:%M %p") %></p>
+            <p class="text-xs text-secondary"><%= export.filename %></p>
+          </div>
+
+          <% if export.processing? || export.pending? %>
+            <div class="flex items-center gap-2 text-secondary">
+              <div class="animate-spin h-4 w-4 border-2 border-secondary border-t-transparent rounded-full"></div>
+              <span class="text-sm">Exporting...</span>
+            </div>
+          <% elsif export.completed? %>
+            <%= link_to download_family_export_path(export),
+                class: "flex items-center gap-2 text-primary hover:text-primary-hover",
+                data: { turbo_frame: "_top" } do %>
+              <%= icon "download", class: "w-5 h-5" %>
+              <span class="text-sm font-medium">Download</span>
+            <% end %>
+          <% elsif export.failed? %>
+            <div class="flex items-center gap-2 text-destructive">
+              <%= icon "alert-circle", class: "w-4 h-4" %>
+              <span class="text-sm">Failed</span>
+            </div>
+          <% end %>
+        </div>
+      <% end %>
+    <% else %>
+      <p class="text-sm text-secondary text-center py-4">No exports yet</p>
+    <% end %>
+  </div>
+<% end %>

app/views/family_exports/index.html.erb

@@ -0,0 +1 @@
+<%= render "list", exports: @exports %>

app/views/family_exports/new.html.erb

@@ -0,0 +1,42 @@
+<%= render DS::Dialog.new do |dialog| %>
+  <% dialog.with_header(title: "Export your data", subtitle: "Download all your financial data") %>
+
+  <% dialog.with_body do %>
+    <div class="space-y-4">
+      <div class="bg-container-inset rounded-lg p-4 space-y-3">
+        <h3 class="font-medium text-primary">What's included:</h3>
+        <ul class="space-y-2 text-sm text-secondary">
+          <li class="flex items-start gap-2">
+            <%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
+            <span>All accounts and balances</span>
+          </li>
+          <li class="flex items-start gap-2">
+            <%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
+            <span>Transaction history</span>
+          </li>
+          <li class="flex items-start gap-2">
+            <%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
+            <span>Investment trades</span>
+          </li>
+          <li class="flex items-start gap-2">
+            <%= icon "check", class: "shrink-0 mt-0.5 text-positive" %>
+            <span>Categories and tags</span>
+          </li>
+        </ul>
+      </div>
+
+      <div class="bg-amber-50 border border-amber-200 rounded-lg p-3">
+        <p class="text-sm text-amber-800">
+          <strong>Note:</strong> This export includes all of your data, but only some of the data can be imported back into Maybe via the CSV import feature. We support account, transaction (with category and tags), and trade imports. Other account data cannot be imported and is for your records only.
+        </p>
+      </div>
+
+      <%= form_with url: family_exports_path, method: :post, class: "space-y-4" do |form| %>
+        <div class="flex gap-3">
+          <%= link_to "Cancel", "#", class: "flex-1 text-center px-4 py-2 border border-primary rounded-lg hover:bg-surface-hover", data: { action: "click->modal#close" } %>
+          <%= form.submit "Export data", class: "flex-1 bg-inverse fg-inverse rounded-lg px-4 py-2 cursor-pointer" %>
+        </div>
+      <% end %>
+    </div>
+  <% end %>
+<% end %>

app/views/settings/profiles/show.html.erb

@@ -122,6 +122,29 @@
   </div>
 <% end %>
 
+<% if Current.user.admin? %>
+  <%= settings_section title: "Data Import/Export" do %>
+    <div class="space-y-4">
+      <div class="flex gap-4 items-center">
+        <%= render DS::Link.new(
+          text: "Export data",
+          icon: "database",
+          href: new_family_export_path,
+          variant: "secondary",
+          full_width: true,
+          data: { turbo_frame: :modal }
+        ) %>
+      </div>
+
+      <%= turbo_frame_tag "family_exports", src: family_exports_path, loading: :lazy do %>
+        <div class="mt-4 text-center text-secondary">
+          <div class="animate-spin inline-block h-4 w-4 border-2 border-secondary border-t-transparent rounded-full"></div>
+        </div>
+      <% end %>
+    </div>
+  <% end %>
+<% end %>
+
 <%= settings_section title: t(".danger_zone_title") do %>
   <div class="space-y-4">
     <% if Current.user.admin? %>