From 09b27709c0dbdc89fafcc045bc14f357132aec25 Mon Sep 17 00:00:00 2001
From: Zach Gollwitzer <zach.gollwitzer@gmail.com>
Date: Sun, 18 May 2025 13:55:47 -0400
Subject: [PATCH] Save work

---
 app/models/plaid_account/importer.rb          |  47 ++++++++
 .../plaid_account/investments_importer.rb     |  13 +++
 .../plaid_account/liabilities_importer.rb     |  13 +++
 .../plaid_account/transactions_importer.rb    |  13 +++
 app/models/plaid_item.rb                      |  19 ++-
 app/models/plaid_item/accounts_importer.rb    |  35 ------
 app/models/plaid_item/importer.rb             | 108 ++++++++++++------
 app/models/plaid_item/investments_importer.rb |   8 --
 app/models/plaid_item/liabilities_importer.rb |   8 --
 app/models/plaid_item/syncer.rb               |  20 ++--
 .../plaid_item/transactions_importer.rb       |   8 --
 ...33020_add_raw_payload_to_plaid_entities.rb |   3 +
 db/schema.rb                                  |   5 +-
 test/fixtures/plaid_items.yml                 |   4 +-
 test/models/plaid_account/importer_test.rb    |  28 +++++
 test/models/plaid_item/importer_test.rb       |  62 ++++++++++
 16 files changed, 278 insertions(+), 116 deletions(-)
 create mode 100644 app/models/plaid_account/importer.rb
 create mode 100644 app/models/plaid_account/investments_importer.rb
 create mode 100644 app/models/plaid_account/liabilities_importer.rb
 create mode 100644 app/models/plaid_account/transactions_importer.rb
 delete mode 100644 app/models/plaid_item/accounts_importer.rb
 delete mode 100644 app/models/plaid_item/investments_importer.rb
 delete mode 100644 app/models/plaid_item/liabilities_importer.rb
 delete mode 100644 app/models/plaid_item/transactions_importer.rb
 create mode 100644 test/models/plaid_account/importer_test.rb
 create mode 100644 test/models/plaid_item/importer_test.rb

diff --git a/app/models/plaid_account/importer.rb b/app/models/plaid_account/importer.rb
new file mode 100644
index 00000000..c4aa5d99
--- /dev/null
+++ b/app/models/plaid_account/importer.rb
@@ -0,0 +1,47 @@
+# PlaidItem::Importer passes a raw payload retrieved from accounts_get API call, along with the PlaidAccount entity
+# This class is responsible for making sense of that raw data, persisting it, and triggering transaction/investment/liability data imports for this account
+class PlaidAccount::Importer
+  def initialize(plaid_account, account_data:, transactions_data:, investments_data:, liabilities_data:)
+    @plaid_account = plaid_account
+    @account_data = account_data
+    @transactions_data = transactions_data
+    @investments_data = investments_data
+    @liabilities_data = liabilities_data
+  end
+
+  def import
+    update_account_info
+
+    import_transactions if transactions_data.present?
+    import_investments if investments_data.present?
+    import_liabilities if liabilities_data.present?
+  end
+
+  private
+    attr_reader :plaid_account, :account_data, :transactions_data, :investments_data, :liabilities_data
+
+    def update_account_info
+      plaid_account.raw_payload = account_data
+      plaid_account.current_balance = account_data.balances.current
+      plaid_account.available_balance = account_data.balances.available
+      plaid_account.currency = account_data.balances.iso_currency_code
+      plaid_account.plaid_type = account_data.type
+      plaid_account.plaid_subtype = account_data.subtype
+      plaid_account.name = account_data.name
+      plaid_account.mask = account_data.mask
+
+      plaid_account.save!
+    end
+
+    def import_transactions
+      PlaidAccount::TransactionsImporter.new(plaid_account, transactions_data: transactions_data).import
+    end
+
+    def import_investments
+      PlaidAccount::InvestmentsImporter.new(plaid_account, investments_data: investments_data).import
+    end
+
+    def import_liabilities
+      PlaidAccount::LiabilitiesImporter.new(plaid_account, liabilities_data: liabilities_data).import
+    end
+end
diff --git a/app/models/plaid_account/investments_importer.rb b/app/models/plaid_account/investments_importer.rb
new file mode 100644
index 00000000..65ed4b80
--- /dev/null
+++ b/app/models/plaid_account/investments_importer.rb
@@ -0,0 +1,13 @@
+class PlaidAccount::InvestmentsImporter
+  def initialize(plaid_account, plaid_provider:)
+    @plaid_account = plaid_account
+    @plaid_provider = plaid_provider
+  end
+
+  def import
+    # TODO
+  end
+
+  private
+    attr_reader :plaid_account, :plaid_provider
+end
diff --git a/app/models/plaid_account/liabilities_importer.rb b/app/models/plaid_account/liabilities_importer.rb
new file mode 100644
index 00000000..fd277df5
--- /dev/null
+++ b/app/models/plaid_account/liabilities_importer.rb
@@ -0,0 +1,13 @@
+class PlaidAccount::LiabilitiesImporter
+  def initialize(plaid_account, plaid_provider:)
+    @plaid_account = plaid_account
+    @plaid_provider = plaid_provider
+  end
+
+  def import
+    # TODO
+  end
+
+  private
+    attr_reader :plaid_account, :plaid_provider
+end
diff --git a/app/models/plaid_account/transactions_importer.rb b/app/models/plaid_account/transactions_importer.rb
new file mode 100644
index 00000000..3f03e3ac
--- /dev/null
+++ b/app/models/plaid_account/transactions_importer.rb
@@ -0,0 +1,13 @@
+class PlaidAccount::TransactionsImporter
+  def initialize(plaid_account, plaid_provider:)
+    @plaid_account = plaid_account
+    @plaid_provider = plaid_provider
+  end
+
+  def import
+    # TODO
+  end
+
+  private
+    attr_reader :plaid_account, :plaid_provider
+end
diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb
index c92f1bed..abd81dc7 100644
--- a/app/models/plaid_item.rb
+++ b/app/models/plaid_item.rb
@@ -60,18 +60,6 @@ class PlaidItem < ApplicationRecord
         .exists?
   end
 
-  def transactions_enabled?
-    true # TODO
-  end
-
-  def investments_enabled?
-    true # TODO
-  end
-
-  def liabilities_enabled?
-    true
-  end
-
   def auto_match_categories!
     if family.categories.none?
       family.categories.bootstrap!
@@ -102,6 +90,13 @@ class PlaidItem < ApplicationRecord
     end
   end
 
+  # Plaid returns mutually exclusive arrays here.  If the item has made a request for a product,
+  # it is put in the billed_products array.  If it is supported, but not yet used, it goes in the
+  # available_products array.
+  def supported_products
+    available_products + billed_products
+  end
+
   private
     # Silently swallow and report error so that we don't block the user from deleting the item
     def remove_plaid_item
diff --git a/app/models/plaid_item/accounts_importer.rb b/app/models/plaid_item/accounts_importer.rb
deleted file mode 100644
index 307bb1a1..00000000
--- a/app/models/plaid_item/accounts_importer.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-class PlaidItem::AccountsImporter
-  def initialize(plaid_item)
-    @plaid_item = plaid_item
-  end
-
-  def import
-    raw_accounts_data = plaid_provider.get_item_accounts(plaid_item).accounts
-
-    raw_accounts_data.each do |raw_account_data|
-      PlaidAccount.transaction do
-        plaid_account = plaid_item.plaid_accounts.find_or_initialize_by(
-          plaid_id: raw_account_data.account_id
-        )
-
-        plaid_account.current_balance = raw_account_data.balances.current
-        plaid_account.available_balance = raw_account_data.balances.available
-        plaid_account.currency = raw_account_data.balances.iso_currency_code
-        plaid_account.plaid_type = raw_account_data.type
-        plaid_account.plaid_subtype = raw_account_data.subtype
-
-        # Save raw payload for audit trail
-        plaid_account.raw_payload = raw_account_data.to_h
-
-        plaid_account.save!
-      end
-    end
-  end
-
-  private
-    attr_reader :plaid_item
-
-    def plaid_provider
-      plaid_item.plaid_provider
-    end
-end
diff --git a/app/models/plaid_item/importer.rb b/app/models/plaid_item/importer.rb
index 296fbd37..779e3e19 100644
--- a/app/models/plaid_item/importer.rb
+++ b/app/models/plaid_item/importer.rb
@@ -1,38 +1,22 @@
 class PlaidItem::Importer
-  def initialize(plaid_item)
+  def initialize(plaid_item, plaid_provider:)
     @plaid_item = plaid_item
+    @plaid_provider = plaid_provider
   end
 
-  def import_data
-    begin
-      import_item_metadata
-    rescue Plaid::ApiError => e
-      handle_plaid_error(e)
-    end
-
+  def import
+    import_item_metadata
+    import_institution_metadata
     import_accounts
-    import_transactions if plaid_item.transactions_enabled?
-    import_investments if plaid_item.investments_enabled?
-    import_liabilities if plaid_item.liabilities_enabled?
+  rescue Plaid::ApiError => e
+    handle_plaid_error(e)
   end
 
   private
-    attr_reader :plaid_item
+    attr_reader :plaid_item, :plaid_provider
 
-    def plaid_provider
-      plaid_item.plaid_provider
-    end
-
-    def import_item_metadata
-      raw_item_data = plaid_provider.get_item(plaid_item.access_token)
-      plaid_item.update!(
-        available_products: raw_item_data.available_products,
-        billed_products: raw_item_data.billed_products
-      )
-    end
-
-    # Re-raise all errors that should halt data importing.  Raising will propagate to
-    # the sync and mark it as failed.
+    # All errors that should halt the import should be re-raised after handling
+    # These errors will propagate up to the Sync record and mark it as failed.
     def handle_plaid_error(error)
       error_body = JSON.parse(error.response_body)
 
@@ -45,19 +29,77 @@ class PlaidItem::Importer
       end
     end
 
+    def import_item_metadata
+      item_response = plaid_provider.get_item(plaid_item.access_token)
+      item_data = item_response.item
+
+      # plaid_item.raw_payload = item_response
+      plaid_item.available_products = item_data.available_products
+      plaid_item.billed_products = item_data.billed_products
+      plaid_item.institution_id = item_data.institution_id
+
+      plaid_item.save!
+    end
+
+    def import_institution_metadata
+      institution_response = plaid_provider.get_institution(plaid_item.institution_id)
+      institution_data = institution_response.institution
+
+      # plaid_item.raw_institution_payload = institution_response
+      plaid_item.institution_id = institution_data.institution_id
+      plaid_item.institution_url = institution_data.url
+      plaid_item.institution_color = institution_data.primary_color
+
+      plaid_item.save!
+    end
+
     def import_accounts
-      PlaidItem::AccountsImporter.new(plaid_item).import
+      accounts_data = plaid_provider.get_item_accounts(plaid_item).accounts
+
+      PlaidItem.transaction do
+        accounts_data.each do |raw_account_payload|
+          plaid_account = plaid_item.plaid_accounts.find_or_initialize_by(
+            plaid_id: raw_account_payload.account_id
+          )
+
+          PlaidAccount::Importer.new(
+            plaid_account,
+            accounts_data: accounts_data,
+            transactions_data: transactions_data,
+            investments_data: investments_data,
+            liabilities_data: liabilities_data
+          ).import
+        end
+      end
     end
 
-    def import_transactions
-      PlaidItem::TransactionsImporter.new(plaid_item).import
+    def transactions_supported?
+      plaid_item.supported_products.include?("transactions")
     end
 
-    def import_investments
-      PlaidItem::InvestmentsImporter.new(plaid_item).import
+    def investments_supported?
+      plaid_item.supported_products.include?("investments")
     end
 
-    def import_liabilities
-      PlaidItem::LiabilitiesImporter.new(plaid_item).import
+    def liabilities_supported?
+      plaid_item.supported_products.include?("liabilities")
+    end
+
+    def transactions_data
+      return nil unless transactions_supported?
+
+      plaid_provider.get_item_transactions(plaid_item).transactions
+    end
+
+    def investments_data
+      return nil unless investments_supported?
+
+      plaid_provider.get_item_investments(plaid_item).investments
+    end
+
+    def liabilities_data
+      return nil unless liabilities_supported?
+
+      plaid_provider.get_item_liabilities(plaid_item).liabilities
     end
 end
diff --git a/app/models/plaid_item/investments_importer.rb b/app/models/plaid_item/investments_importer.rb
deleted file mode 100644
index a2f4c0b5..00000000
--- a/app/models/plaid_item/investments_importer.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-class PlaidItem::InvestmentsImporter
-  def initialize(plaid_item)
-    @plaid_item = plaid_item
-  end
-
-  def import_data
-  end
-end
diff --git a/app/models/plaid_item/liabilities_importer.rb b/app/models/plaid_item/liabilities_importer.rb
deleted file mode 100644
index 3404da82..00000000
--- a/app/models/plaid_item/liabilities_importer.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-class PlaidItem::LiabilitiesImporter
-  def initialize(plaid_item)
-    @plaid_item = plaid_item
-  end
-
-  def import_data
-  end
-end
diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb
index 354c0ff4..e7357b09 100644
--- a/app/models/plaid_item/syncer.rb
+++ b/app/models/plaid_item/syncer.rb
@@ -10,16 +10,16 @@ class PlaidItem::Syncer
     import_item_data
 
     # Processes the raw Plaid data and updates internal domain objects
-    process_item_data
+    # process_item_data
 
     # All data is synced, so we can now run an account sync to calculate historical balances and more
-    plaid_item.reload.accounts.each do |account|
-      account.sync_later(
-        parent_sync: sync,
-        window_start_date: sync.window_start_date,
-        window_end_date: sync.window_end_date
-      )
-    end
+    # plaid_item.reload.accounts.each do |account|
+    #   account.sync_later(
+    #     parent_sync: sync,
+    #     window_start_date: sync.window_start_date,
+    #     window_end_date: sync.window_end_date
+    #   )
+    # end
   end
 
   def perform_post_sync
@@ -27,12 +27,12 @@ class PlaidItem::Syncer
   end
 
   private
-    def plaid
+    def plaid_provider
       plaid_item.plaid_provider
     end
 
     def import_item_data
-      PlaidItem::Importer.new(plaid_item).import_data
+      PlaidItem::Importer.new(plaid_item, plaid_provider: plaid_provider).import
     end
 
     def process_item_data
diff --git a/app/models/plaid_item/transactions_importer.rb b/app/models/plaid_item/transactions_importer.rb
deleted file mode 100644
index 45b9564f..00000000
--- a/app/models/plaid_item/transactions_importer.rb
+++ /dev/null
@@ -1,8 +0,0 @@
-class PlaidItem::TransactionsImporter
-  def initialize(plaid_item)
-    @plaid_item = plaid_item
-  end
-
-  def import_data
-  end
-end
diff --git a/db/migrate/20250518133020_add_raw_payload_to_plaid_entities.rb b/db/migrate/20250518133020_add_raw_payload_to_plaid_entities.rb
index a86bc1d9..90d9bfe3 100644
--- a/db/migrate/20250518133020_add_raw_payload_to_plaid_entities.rb
+++ b/db/migrate/20250518133020_add_raw_payload_to_plaid_entities.rb
@@ -1,4 +1,7 @@
 class AddRawPayloadToPlaidEntities < ActiveRecord::Migration[7.2]
   def change
+    add_column :plaid_accounts, :raw_payload, :jsonb, default: {}
+    add_column :plaid_items, :raw_payload, :jsonb, default: {}
+    add_column :plaid_items, :raw_institution_payload, :jsonb, default: {}
   end
 end
diff --git a/db/schema.rb b/db/schema.rb
index 5b9426c7..5cd7e03e 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_05_16_180846) do
+ActiveRecord::Schema[7.2].define(version: 2025_05_18_133020) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -429,6 +429,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_16_180846) do
     t.string "mask"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.jsonb "raw_payload", default: {}
     t.index ["plaid_item_id"], name: "index_plaid_accounts_on_plaid_item_id"
   end
 
@@ -448,6 +449,8 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_16_180846) do
     t.string "institution_id"
     t.string "institution_color"
     t.string "status", default: "good", null: false
+    t.jsonb "raw_payload", default: {}
+    t.jsonb "raw_institution_payload", default: {}
     t.index ["family_id"], name: "index_plaid_items_on_family_id"
   end
 
diff --git a/test/fixtures/plaid_items.yml b/test/fixtures/plaid_items.yml
index 21a0b460..4a3b3df8 100644
--- a/test/fixtures/plaid_items.yml
+++ b/test/fixtures/plaid_items.yml
@@ -2,4 +2,6 @@ one:
   family: dylan_family
   plaid_id: "1234567890"
   access_token: encrypted_token_1
-  name: "Test Bank"
\ No newline at end of file
+  name: "Test Bank"
+  billed_products: ["transactions", "investments", "liabilities"]
+  available_products: []
\ No newline at end of file
diff --git a/test/models/plaid_account/importer_test.rb b/test/models/plaid_account/importer_test.rb
new file mode 100644
index 00000000..19e60eb3
--- /dev/null
+++ b/test/models/plaid_account/importer_test.rb
@@ -0,0 +1,28 @@
+require "test_helper"
+
+class PlaidAccount::ImporterTest < ActiveSupport::TestCase
+  setup do
+    @mock_provider = mock("Provider::Plaid")
+    @plaid_account = plaid_accounts(:one)
+  end
+
+  test "imports account data" do
+    raw_payload = OpenStruct.new(
+      account_id: "123",
+      name: "Test Account",
+      mask: "1234",
+      type: "checking",
+      subtype: "checking",
+    )
+
+    PlaidAccount::Importer.new(@plaid_account, raw_payload, plaid_provider: @mock_provider).import
+
+    @plaid_account.reload
+
+    assert_equal "123", @plaid_account.plaid_id
+    assert_equal "Test Account", @plaid_account.name
+    assert_equal "1234", @plaid_account.mask
+    assert_equal "checking", @plaid_account.plaid_type
+    assert_equal "checking", @plaid_account.plaid_subtype
+  end
+end
diff --git a/test/models/plaid_item/importer_test.rb b/test/models/plaid_item/importer_test.rb
new file mode 100644
index 00000000..9a9d4ae1
--- /dev/null
+++ b/test/models/plaid_item/importer_test.rb
@@ -0,0 +1,62 @@
+require "test_helper"
+require "ostruct"
+
+class PlaidItem::ImporterTest < ActiveSupport::TestCase
+  setup do
+    @mock_provider = mock("Provider::Plaid")
+    @plaid_item = plaid_items(:one)
+  end
+
+  test "imports item metadata" do
+    mock_institution_id = "123"
+
+    raw_item_payload = OpenStruct.new(
+      item: OpenStruct.new(
+        available_products: [],
+        billed_products: %w[transactions investments liabilities],
+        institution_id: mock_institution_id
+      )
+    )
+
+    raw_institution_payload = OpenStruct.new(
+      institution: OpenStruct.new(
+        institution_id: mock_institution_id,
+        url: "https://example.com",
+        primary_color: "#000000"
+      )
+    )
+
+    raw_accounts_payload = OpenStruct.new(
+      accounts: [
+        OpenStruct.new(
+          account_id: "123",
+          name: "Test Account",
+          mask: "1234",
+          type: "checking",
+          subtype: "checking",
+        )
+      ]
+    )
+
+    @mock_provider.expects(:get_item).returns(raw_item_payload)
+    @mock_provider.expects(:get_institution).with(mock_institution_id).returns(raw_institution_payload)
+    @mock_provider.expects(:get_item_accounts).with(@plaid_item).returns(raw_accounts_payload)
+    @mock_provider.expects(:get_item_transactions).with(@plaid_item).returns(OpenStruct.new(transactions: []))
+    @mock_provider.expects(:get_item_investments).with(@plaid_item).returns(OpenStruct.new(investments: []))
+    @mock_provider.expects(:get_item_liabilities).with(@plaid_item).returns(OpenStruct.new(liabilities: []))
+
+    PlaidAccount::Importer.any_instance.expects(:import).times(raw_accounts_payload.accounts.count)
+
+    PlaidItem::Importer.new(@plaid_item, plaid_provider: @mock_provider).import
+
+    @plaid_item.reload
+
+    assert_equal mock_institution_id, @plaid_item.institution_id
+    assert_equal "https://example.com", @plaid_item.institution_url
+    assert_equal "#000000", @plaid_item.institution_color
+    assert_equal %w[transactions investments liabilities], @plaid_item.available_products
+    assert_equal %w[transactions investments liabilities], @plaid_item.billed_products
+    assert_not_nil @plaid_item.raw_payload
+    assert_not_nil @plaid_item.raw_institution_payload
+  end
+end