Data provider simplification, tests, and documentation (#1997)

* Ignore env.test from source control

* Simplification of providers interface

* Synth tests

* Update money to use new find rates method

* Remove unused issues code

* Additional issue feature removals

* Update price data fetching and tests

* Update documentation for providers

* Security test fixes

* Fix self host test

* Update synth usage data access

* Remove AI pr schema changes

app/models/account/convertible.rb

@@ -7,14 +7,13 @@ module Account::Convertible
       return
     end
 
-    rates = ExchangeRate.find_rates(
+    affected_row_count = ExchangeRate.sync_provider_rates(
       from: currency,
       to: target_currency,
       start_date: start_date,
-      cache: true # caches from provider to DB
     )
 
-    Rails.logger.info("Synced #{rates.count} exchange rates for account #{id}")
+    Rails.logger.info("Synced #{affected_row_count} exchange rates for account #{id}")
   end
 
   private

app/models/account/data_enricher.rb

@@ -1,66 +0,0 @@
-class Account::DataEnricher
-  attr_reader :account
-
-  def initialize(account)
-    @account = account
-  end
-
-  def run
-    total_unenriched = account.entries.account_transactions
-      .joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
-      .where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
-      .count
-
-    if total_unenriched > 0
-      batch_size = 50
-      batches = (total_unenriched.to_f / batch_size).ceil
-
-      batches.times do |batch|
-        EnrichTransactionBatchJob.perform_later(account, batch_size, batch * batch_size)
-      end
-    end
-  end
-
-  def enrich_transaction_batch(batch_size = 50, offset = 0)
-    candidates = account.entries.account_transactions
-      .includes(entryable: [ :merchant, :category ])
-      .joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
-      .where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
-      .offset(offset)
-      .limit(batch_size)
-
-    Rails.logger.info("Enriching batch of #{candidates.count} transactions for account #{account.id} (offset: #{offset})")
-
-    merchants = {}
-
-    candidates.each do |entry|
-      begin
-        info = entry.fetch_enrichment_info
-
-        next unless info.present?
-
-        if info.name.present?
-          merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
-
-          if info.icon_url.present?
-            merchant.icon_url = info.icon_url
-          end
-        end
-
-        entryable_attributes = { id: entry.entryable_id }
-        entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
-
-        Account.transaction do
-          merchant.save! if merchant.present?
-          entry.update!(
-            enriched_at: Time.current,
-            enriched_name: info.name,
-            entryable_attributes: entryable_attributes
-          )
-        end
-      rescue => e
-        Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
-      end
-    end
-  end
-end

app/models/account/enrichable.rb

@@ -2,11 +2,70 @@ module Account::Enrichable
   extend ActiveSupport::Concern
 
   def enrich_data
-    DataEnricher.new(self).run
+    total_unenriched = entries.account_transactions
+      .joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
+      .where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
+      .count
+
+    if total_unenriched > 0
+      batch_size = 50
+      batches = (total_unenriched.to_f / batch_size).ceil
+
+      batches.times do |batch|
+        EnrichTransactionBatchJob.perform_now(self, batch_size, batch * batch_size)
+        # EnrichTransactionBatchJob.perform_later(self, batch_size, batch * batch_size)
+      end
+    end
+  end
+
+  def enrich_transaction_batch(batch_size = 50, offset = 0)
+    transactions_batch = enrichable_transactions.offset(offset).limit(batch_size)
+
+    Rails.logger.info("Enriching batch of #{transactions_batch.count} transactions for account #{id} (offset: #{offset})")
+
+    merchants = {}
+
+    transactions_batch.each do |transaction|
+      begin
+        info = transaction.fetch_enrichment_info
+
+        next unless info.present?
+
+        if info.name.present?
+          merchant = merchants[info.name] ||= family.merchants.find_or_create_by(name: info.name)
+
+          if info.icon_url.present?
+            merchant.icon_url = info.icon_url
+          end
+        end
+
+        Account.transaction do
+          merchant.save! if merchant.present?
+          transaction.update!(merchant: merchant) if merchant.present? && transaction.merchant_id.nil?
+
+          transaction.entry.update!(
+            enriched_at: Time.current,
+            enriched_name: info.name,
+          )
+        end
+      rescue => e
+        Rails.logger.warn("Error enriching transaction #{transaction.id}: #{e.message}")
+      end
+    end
   end
 
   private
     def enrichable?
       family.data_enrichment_enabled? || (linked? && Rails.application.config.app_mode.hosted?)
     end
+
+    def enrichable_transactions
+      transactions.active
+                  .includes(:merchant, :category)
+                  .where(
+                    "account_entries.enriched_at IS NULL",
+                    "OR merchant_id IS NULL",
+                    "OR category_id IS NULL"
+                  )
+    end
 end

app/models/account/entry.rb

@@ -1,5 +1,5 @@
 class Account::Entry < ApplicationRecord
-  include Monetizable, Provided
+  include Monetizable
 
   monetize :amount
 

app/models/account/entry/provided.rb

@@ -1,11 +0,0 @@
-module Account::Entry::Provided
-  extend ActiveSupport::Concern
-
-  include Synthable
-
-  def fetch_enrichment_info
-    return nil unless synth_client.present?
-
-    synth_client.enrich_transaction(name).info
-  end
-end

app/models/account/holding/portfolio_cache.rb

@@ -79,12 +79,11 @@ class Account::Holding::PortfolioCache
       securities.each do |security|
         Rails.logger.info "Loading security: ID=#{security.id} Ticker=#{security.ticker}"
 
-        # Highest priority prices
-        db_or_provider_prices = Security::Price.find_prices(
-          security: security,
-          start_date: account.start_date,
-          end_date: Date.current
-        ).map do |price|
+        # Load prices from provider to DB
+        security.sync_provider_prices(start_date: account.start_date)
+
+        # High priority prices from DB (synced from provider)
+        db_prices = security.prices.where(date: account.start_date..Date.current).map do |price|
           PriceWithPriority.new(
             price: price,
             priority: 1
@@ -125,7 +124,7 @@ class Account::Holding::PortfolioCache
 
         @security_cache[security.id] = {
           security: security,
-          prices: db_or_provider_prices + trade_prices + holding_prices
+          prices: db_prices + trade_prices + holding_prices
         }
       end
     end

app/models/account/transaction.rb

@@ -1,5 +1,5 @@
 class Account::Transaction < ApplicationRecord
-  include Account::Entryable, Transferable
+  include Account::Entryable, Transferable, Provided
 
   belongs_to :category, optional: true
   belongs_to :merchant, optional: true

app/models/account/transaction/provided.rb

@@ -0,0 +1,15 @@
+module Account::Transaction::Provided
+  extend ActiveSupport::Concern
+
+  def fetch_enrichment_info
+    return nil unless Providers.synth # Only Synth can provide this data
+
+    response = Providers.synth.enrich_transaction(
+      entry.name,
+      amount: entry.amount,
+      date: entry.date
+    )
+
+    response.data
+  end
+end