Clarify backend data pipeline naming concepts (importers, processors, materializers, calculators, and syncers) (#2255)

* Rename MarketDataSyncer to MarketDataImporter

* Materializers

* Importers

* More reference replacements

test/models/account/market_data_importer_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 require "ostruct"
 
-class Account::MarketDataSyncerTest < ActiveSupport::TestCase
+class Account::MarketDataImporterTest < ActiveSupport::TestCase
   include ProviderTestHelper
 
   PROVIDER_BUFFER = 5.days
@@ -32,7 +32,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
       accountable: Depository.new
     )
 
-    # Seed a rate for the first required day so that the syncer only needs the next day forward
+    # Seed a rate for the first required day so that the importer only needs the next day forward
     existing_date = account.start_date
     ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: existing_date, rate: 2.0)
 
@@ -49,7 +49,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
              ]))
 
     before = ExchangeRate.count
-    Account::MarketDataSyncer.new(account).sync_market_data
+    Account::MarketDataImporter.new(account).import_all
     after  = ExchangeRate.count
 
     assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
@@ -100,7 +100,7 @@ class Account::MarketDataSyncerTest < ActiveSupport::TestCase
     # Ignore exchange-rate calls for this test
     @provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
 
-    Account::MarketDataSyncer.new(account).sync_market_data
+    Account::MarketDataImporter.new(account).import_all
 
     assert_equal 1, Security::Price.where(security: security, date: trade_date).count
   end

test/models/balance/materializer_test.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-class Balance::SyncerTest < ActiveSupport::TestCase
+class Balance::MaterializerTest < ActiveSupport::TestCase
   include EntriesTestHelper
 
   setup do
@@ -14,7 +14,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
   end
 
   test "syncs balances" do
-    Holding::Syncer.any_instance.expects(:sync_holdings).returns([]).once
+    Holding::Materializer.any_instance.expects(:materialize_holdings).returns([]).once
 
     @account.expects(:start_date).returns(2.days.ago.to_date)
 
@@ -26,7 +26,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
     )
 
     assert_difference "@account.balances.count", 2 do
-      Balance::Syncer.new(@account, strategy: :forward).sync_balances
+      Balance::Materializer.new(@account, strategy: :forward).materialize_balances
     end
   end
 
@@ -45,7 +45,7 @@ class Balance::SyncerTest < ActiveSupport::TestCase
     )
 
     assert_difference "@account.balances.count", 3 do
-      Balance::Syncer.new(@account, strategy: :forward).sync_balances
+      Balance::Materializer.new(@account, strategy: :forward).materialize_balances
     end
   end
 end

test/models/exchange_rate/importer_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 require "ostruct"
 
-class ExchangeRate::SyncerTest < ActiveSupport::TestCase
+class ExchangeRate::ImporterTest < ActiveSupport::TestCase
   include ProviderTestHelper
 
   setup do
@@ -21,13 +21,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
              .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    ExchangeRate::Syncer.new(
+    ExchangeRate::Importer.new(
       exchange_rate_provider: @provider,
       from: "USD",
       to: "EUR",
       start_date: 2.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_rates
+    ).import_provider_rates
 
     db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date..Date.current)
                            .order(:date)
@@ -53,13 +53,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
              .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    ExchangeRate::Syncer.new(
+    ExchangeRate::Importer.new(
       exchange_rate_provider: @provider,
       from: "USD",
       to: "EUR",
       start_date: 3.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_rates
+    ).import_provider_rates
 
     db_rates = ExchangeRate.order(:date)
     assert_equal 4, db_rates.count
@@ -75,13 +75,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
 
     @provider.expects(:fetch_exchange_rates).never
 
-    ExchangeRate::Syncer.new(
+    ExchangeRate::Importer.new(
       exchange_rate_provider: @provider,
       from: "USD",
       to: "EUR",
       start_date: 3.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_rates
+    ).import_provider_rates
   end
 
   # A helpful "reset" option for when we need to refresh provider data
@@ -103,14 +103,14 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
              .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    ExchangeRate::Syncer.new(
+    ExchangeRate::Importer.new(
       exchange_rate_provider: @provider,
       from: "USD",
       to: "EUR",
       start_date: 2.days.ago.to_date,
       end_date: Date.current,
       clear_cache: true
-    ).sync_provider_rates
+    ).import_provider_rates
 
     db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date)
     assert_equal [ 1.3, 1.4, 1.5 ], db_rates.map(&:rate)
@@ -129,13 +129,13 @@ class ExchangeRate::SyncerTest < ActiveSupport::TestCase
              .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
              .returns(provider_response)
 
-    ExchangeRate::Syncer.new(
+    ExchangeRate::Importer.new(
       exchange_rate_provider: @provider,
       from: "USD",
       to: "EUR",
       start_date: Date.current,
       end_date: future_date
-    ).sync_provider_rates
+    ).import_provider_rates
 
     assert_equal 1, ExchangeRate.count
   end

test/models/holding/materializer_test.rb

@@ -1,6 +1,6 @@
 require "test_helper"
 
-class Holding::SyncerTest < ActiveSupport::TestCase
+class Holding::MaterializerTest < ActiveSupport::TestCase
   include EntriesTestHelper
 
   setup do
@@ -14,7 +14,7 @@ class Holding::SyncerTest < ActiveSupport::TestCase
 
     # Should have yesterday's and today's holdings
     assert_difference "@account.holdings.count", 2 do
-      Holding::Syncer.new(@account, strategy: :forward).sync_holdings
+      Holding::Materializer.new(@account, strategy: :forward).materialize_holdings
     end
   end
 
@@ -23,7 +23,7 @@ class Holding::SyncerTest < ActiveSupport::TestCase
     Holding.create!(account: @account, security: @aapl, qty: 1, price: 100, amount: 100, currency: "USD", date: Date.current)
 
     assert_difference "Holding.count", -1 do
-      Holding::Syncer.new(@account, strategy: :forward).sync_holdings
+      Holding::Materializer.new(@account, strategy: :forward).materialize_holdings
     end
   end
 end

test/models/market_data_importer_test.rb

@@ -1,10 +1,10 @@
 require "test_helper"
 require "ostruct"
 
-class MarketDataSyncerTest < ActiveSupport::TestCase
+class MarketDataImporterTest < ActiveSupport::TestCase
   include ProviderTestHelper
 
-  SNAPSHOT_START_DATE = MarketDataSyncer::SNAPSHOT_DAYS.days.ago.to_date
+  SNAPSHOT_START_DATE = MarketDataImporter::SNAPSHOT_DAYS.days.ago.to_date
   PROVIDER_BUFFER     = 5.days
 
   setup do
@@ -47,7 +47,7 @@ class MarketDataSyncerTest < ActiveSupport::TestCase
              ]))
 
     before = ExchangeRate.count
-    MarketDataSyncer.new(mode: :snapshot).sync_exchange_rates
+    MarketDataImporter.new(mode: :snapshot).import_exchange_rates
     after  = ExchangeRate.count
 
     assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
@@ -78,7 +78,7 @@ class MarketDataSyncerTest < ActiveSupport::TestCase
     # Ignore exchange rate calls for this test
     @provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
 
-    MarketDataSyncer.new(mode: :snapshot).sync_prices
+    MarketDataImporter.new(mode: :snapshot).import_security_prices
 
     assert_equal 1, Security::Price.where(security: security, date: SNAPSHOT_START_DATE).count
   end

test/models/security/price/importer_test.rb

@@ -1,7 +1,7 @@
 require "test_helper"
 require "ostruct"
 
-class Security::Price::SyncerTest < ActiveSupport::TestCase
+class Security::Price::ImporterTest < ActiveSupport::TestCase
   include ProviderTestHelper
 
   setup do
@@ -23,12 +23,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
                    start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    Security::Price::Syncer.new(
+    Security::Price::Importer.new(
       security: @security,
       security_provider: @provider,
       start_date: 2.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_prices
+    ).import_provider_prices
 
     db_prices = Security::Price.where(security: @security, date: 2.days.ago.to_date..Date.current).order(:date)
 
@@ -52,12 +52,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
                    start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    Security::Price::Syncer.new(
+    Security::Price::Importer.new(
       security: @security,
       security_provider: @provider,
       start_date: 3.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_prices
+    ).import_provider_prices
 
     db_prices = Security::Price.where(security: @security).order(:date)
     assert_equal 4, db_prices.count
@@ -73,12 +73,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
 
     @provider.expects(:fetch_security_prices).never
 
-    Security::Price::Syncer.new(
+    Security::Price::Importer.new(
       security: @security,
       security_provider: @provider,
       start_date: 3.days.ago.to_date,
       end_date: Date.current
-    ).sync_provider_prices
+    ).import_provider_prices
   end
 
   test "full upsert if clear_cache is true" do
@@ -100,13 +100,13 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
                    start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
              .returns(provider_response)
 
-    Security::Price::Syncer.new(
+    Security::Price::Importer.new(
       security: @security,
       security_provider: @provider,
       start_date: 2.days.ago.to_date,
       end_date: Date.current,
       clear_cache: true
-    ).sync_provider_prices
+    ).import_provider_prices
 
     db_prices = Security::Price.where(security: @security).order(:date)
     assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
@@ -126,12 +126,12 @@ class Security::Price::SyncerTest < ActiveSupport::TestCase
                    start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
              .returns(provider_response)
 
-    Security::Price::Syncer.new(
+    Security::Price::Importer.new(
       security: @security,
       security_provider: @provider,
       start_date: Date.current,
       end_date: future_date
-    ).sync_provider_prices
+    ).import_provider_prices
 
     assert_equal 1, Security::Price.count
   end