Market data sync refinements (#2252)

* Exchange rate syncer implementation

* Security price syncer

* Fix issues with provider API

* Add back prod schedule

* Add back price and exchange rate syncs to account syncs

* Remove unused stock_exchanges table

app/jobs/sync_market_data_job.rb

@@ -1,7 +1,20 @@
+# This job runs daily at market close.  See config/schedule.yml for details.
+#
+# The primary purpose of this job is to:
+# 1. Determine what exchange rate pairs, security prices, and other market data all of our users need to view historical account balance data
+# 2. For each needed rate/price, fetch from our data provider and upsert to our database
+#
+# Each individual account sync will still fetch any missing market data that isn't yet synced, but by running
+# this job daily, we significantly reduce overlapping account syncs that both need the same market data (e.g. common security like `AAPL`)
+#
 class SyncMarketDataJob < ApplicationJob
   queue_as :scheduled
 
-  def perform
+  def perform(opts)
-    MarketDataSyncer.new.sync_all
+    opts = opts.symbolize_keys
+    mode = opts.fetch(:mode, :full)
+    clear_cache = opts.fetch(:clear_cache, false)
+
+    MarketDataSyncer.new(mode: mode, clear_cache: clear_cache).sync
   end
 end

app/models/account.rb

@@ -1,5 +1,5 @@
 class Account < ApplicationRecord
-  include Syncable, Monetizable, Chartable, Linkable, Convertible, Enrichable
+  include Syncable, Monetizable, Chartable, Linkable, Enrichable
 
   validates :name, :balance, :currency, presence: true
 

app/models/account/convertible.rb

@@ -1,27 +0,0 @@
-module Account::Convertible
-  extend ActiveSupport::Concern
-
-  def sync_required_exchange_rates
-    unless requires_exchange_rates?
-      Rails.logger.info("No exchange rate sync needed for account #{id}")
-      return
-    end
-
-    affected_row_count = ExchangeRate.sync_provider_rates(
-      from: currency,
-      to: target_currency,
-      start_date: start_date,
-    )
-
-    Rails.logger.info("Synced #{affected_row_count} exchange rates for account #{id}")
-  end
-
-  private
-    def target_currency
-      family.currency
-    end
-
-    def requires_exchange_rates?
-      currency != target_currency
-    end
-end

app/models/account/market_data_syncer.rb

@@ -0,0 +1,82 @@
+class Account::MarketDataSyncer
+  attr_reader :account
+
+  def initialize(account)
+    @account = account
+  end
+
+  def sync_market_data
+    sync_exchange_rates
+    sync_security_prices
+  end
+
+  private
+    def sync_exchange_rates
+      return unless needs_exchange_rates?
+      return unless ExchangeRate.provider
+
+      pair_dates = {}
+
+      # 1. ENTRY-BASED PAIRS – currencies that differ from the account currency
+      account.entries
+             .where.not(currency: account.currency)
+             .group(:currency)
+             .minimum(:date)
+             .each do |source_currency, date|
+        key = [ source_currency, account.currency ]
+        pair_dates[key] = [ pair_dates[key], date ].compact.min
+      end
+
+      # 2. ACCOUNT-BASED PAIR – convert the account currency to the family currency (if different)
+      if foreign_account?
+        key = [ account.currency, account.family.currency ]
+        pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min
+      end
+
+      pair_dates.each do |(source, target), start_date|
+        ExchangeRate.sync_provider_rates(
+          from: source,
+          to: target,
+          start_date: start_date,
+          end_date: Date.current
+        )
+      end
+    end
+
+    def sync_security_prices
+      return unless Security.provider
+
+      account_securities = account.trades.map(&:security).uniq
+
+      return if account_securities.empty?
+
+      account_securities.each do |security|
+        security.sync_provider_prices(
+          start_date: first_required_price_date(security),
+          end_date: Date.current
+        )
+
+        security.sync_provider_details
+      end
+    end
+
+    # Calculates the first date we require a price for the given security scoped to this account
+    def first_required_price_date(security)
+      account.trades.with_entry
+                    .where(security: security)
+                    .where(entries: { account_id: account.id })
+                    .minimum("entries.date")
+    end
+
+    def needs_exchange_rates?
+      has_multi_currency_entries? || foreign_account?
+    end
+
+    def has_multi_currency_entries?
+      account.entries.where.not(currency: account.currency).exists?
+    end
+
+    def foreign_account?
+      account.currency != account.family.currency
+    end
+end

app/models/account/syncer.rb

@@ -7,6 +7,7 @@ class Account::Syncer
 
   def perform_sync(sync)
     Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})")
+    sync_market_data
     sync_balances
   end
 
@@ -19,4 +20,18 @@ class Account::Syncer
       strategy = account.linked? ? :reverse : :forward
       Balance::Syncer.new(account, strategy: strategy).sync_balances
     end
+
+    # Syncs all the exchange rates + security prices this account needs to display historical chart data
+    #
+    # This is a *supplemental* sync.  The daily market data sync should have already populated
+    # a majority or all of this data, so this is often a no-op.
+    #
+    # We rescue errors here because if this operation fails, we don't want to fail the entire sync since
+    # we have reasonable fallbacks for missing market data.
+    def sync_market_data
+      Account::MarketDataSyncer.new(account).sync_market_data
+    rescue => e
+      Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}")
+      Sentry.capture_exception(e)
+    end
 end

app/models/balance/syncer.rb

@@ -19,8 +19,6 @@ class Balance::Syncer
       if strategy == :forward
         update_account_info
       end
-
-      account.sync_required_exchange_rates
     end
   end
 

app/models/exchange_rate/provided.rb

@@ -27,29 +27,21 @@ module ExchangeRate::Provided
       rate
     end
 
-    def sync_provider_rates(from:, to:, start_date:, end_date: Date.current)
+    # @return [Integer] The number of exchange rates synced
+    def sync_provider_rates(from:, to:, start_date:, end_date:, clear_cache: false)
       unless provider.present?
         Rails.logger.warn("No provider configured for ExchangeRate.sync_provider_rates")
         return 0
       end
 
-      fetched_rates = provider.fetch_exchange_rates(from: from, to: to, start_date: start_date, end_date: end_date)
+      ExchangeRate::Syncer.new(
-
+        exchange_rate_provider: provider,
-      unless fetched_rates.success?
+        from: from,
-        Rails.logger.error("Provider error for ExchangeRate.sync_provider_rates: #{fetched_rates.error}")
+        to: to,
-        return 0
+        start_date: start_date,
-      end
+        end_date: end_date,
-
+        clear_cache: clear_cache
-      rates_data = fetched_rates.data.map do |rate|
+      ).sync_provider_rates
-        {
-          from_currency: rate.from,
-          to_currency: rate.to,
-          date: rate.date,
-          rate: rate.rate
-        }
-      end
-
-      ExchangeRate.upsert_all(rates_data, unique_by: %i[from_currency to_currency date])
     end
   end
 end

app/models/exchange_rate/syncer.rb

@@ -0,0 +1,156 @@
+class ExchangeRate::Syncer
+  MissingExchangeRateError = Class.new(StandardError)
+  MissingStartRateError = Class.new(StandardError)
+
+  def initialize(exchange_rate_provider:, from:, to:, start_date:, end_date:, clear_cache: false)
+    @exchange_rate_provider = exchange_rate_provider
+    @from = from
+    @to = to
+    @start_date = start_date
+    @end_date = normalize_end_date(end_date)
+    @clear_cache = clear_cache
+  end
+
+  # Constructs a daily series of rates for the given currency pair for date range
+  def sync_provider_rates
+    if !clear_cache && all_rates_exist?
+      Rails.logger.info("No new rates to sync for #{from} to #{to} between #{start_date} and #{end_date}, skipping")
+      return
+    end
+
+    if clear_cache && provider_rates.empty?
+      Rails.logger.warn("Could not clear cache for #{from} to #{to} between #{start_date} and #{end_date} because provider returned no rates")
+      return
+    end
+
+    prev_rate_value = start_rate_value
+
+    unless prev_rate_value.present?
+      error = MissingStartRateError.new("Could not find a start rate for #{from} to #{to} between #{start_date} and #{end_date}")
+      Rails.logger.error(error.message)
+      Sentry.capture_exception(error)
+      return
+    end
+
+    gapfilled_rates = effective_start_date.upto(end_date).map do |date|
+      db_rate_value = db_rates[date]&.rate
+      provider_rate_value = provider_rates[date]&.rate
+
+      chosen_rate = if clear_cache
+        provider_rate_value || db_rate_value   # overwrite when possible
+      else
+        db_rate_value || provider_rate_value   # fill gaps
+      end
+
+      # Gapfill with LOCF strategy (last observation carried forward)
+      if chosen_rate.nil?
+        chosen_rate = prev_rate_value
+      end
+
+      prev_rate_value = chosen_rate
+
+      {
+        from_currency: from,
+        to_currency: to,
+        date: date,
+        rate: chosen_rate
+      }
+    end
+
+    upsert_rows(gapfilled_rates)
+  end
+
+  private
+    attr_reader :exchange_rate_provider, :from, :to, :start_date, :end_date, :clear_cache
+
+    def upsert_rows(rows)
+      batch_size = 200
+
+      total_upsert_count = 0
+
+      rows.each_slice(batch_size) do |batch|
+        upserted_ids = ExchangeRate.upsert_all(
+          batch,
+          unique_by: %i[from_currency to_currency date],
+          returning: [ "id" ]
+        )
+
+        total_upsert_count += upserted_ids.count
+      end
+
+      total_upsert_count
+    end
+
+    # Since provider may not return values on weekends and holidays, we grab the first rate from the provider that is on or before the start date
+    def start_rate_value
+      provider_rate_value = provider_rates.select { |date, _| date <= start_date }.max_by { |date, _| date }&.last
+      db_rate_value = db_rates[start_date]&.rate
+      provider_rate_value || db_rate_value
+    end
+
+    # No need to fetch/upsert rates for dates that we already have in the DB
+    def effective_start_date
+      return start_date if clear_cache
+
+      first_missing_date = nil
+
+      start_date.upto(end_date) do |date|
+        unless db_rates.key?(date)
+          first_missing_date = date
+          break
+        end
+      end
+
+      first_missing_date || end_date
+    end
+
+    def provider_rates
+      @provider_rates ||= begin
+        # Always fetch with a 5 day buffer to ensure we have a starting rate (for weekends and holidays)
+        provider_fetch_start_date = effective_start_date - 5.days
+
+        provider_response = exchange_rate_provider.fetch_exchange_rates(
+          from: from,
+          to: to,
+          start_date: provider_fetch_start_date,
+          end_date: end_date
+        )
+
+        if provider_response.success?
+          provider_response.data.index_by(&:date)
+        else
+          message = "#{exchange_rate_provider.class.name} could not fetch exchange rate pair from: #{from} to: #{to} between: #{effective_start_date} and: #{Date.current}.  Provider error: #{provider_response.error.message}"
+          Rails.logger.warn(message)
+          Sentry.capture_exception(MissingExchangeRateError.new(message))
+          {}
+        end
+      end
+    end
+
+    def all_rates_exist?
+      db_count == expected_count
+    end
+
+    def expected_count
+      (start_date..end_date).count
+    end
+
+    def db_count
+      db_rates.count
+    end
+
+    def db_rates
+      @db_rates ||= ExchangeRate.where(from_currency: from, to_currency: to, date: start_date..end_date)
+                  .order(:date)
+                  .to_a
+                  .index_by(&:date)
+    end
+
+    # Normalizes an end date so that it never exceeds today's date in the
+    # America/New_York timezone. If the caller passes a future date we clamp
+    # it to today so that upstream provider calls remain valid and predictable.
+    def normalize_end_date(requested_end_date)
+      today_est = Date.current.in_time_zone("America/New_York").to_date
+      [ requested_end_date, today_est ].min
+    end
+end

app/models/market_data_syncer.rb

@@ -1,196 +1,132 @@
 class MarketDataSyncer
-  DEFAULT_HISTORY_DAYS = 30
+  # By default, our graphs show 1M as the view, so by fetching 31 days,
-  RATE_PROVIDER_NAME = :synth
+  # we ensure we can always show an accurate default graph
-  PRICE_PROVIDER_NAME = :synth
+  SNAPSHOT_DAYS = 31
 
-  MissingExchangeRateError = Class.new(StandardError)
+  InvalidModeError = Class.new(StandardError)
-  InvalidExchangeRateDataError = Class.new(StandardError)
-  MissingSecurityPriceError = Class.new(StandardError)
-  InvalidSecurityPriceDataError = Class.new(StandardError)
 
-  class << self
+  def initialize(mode: :full, clear_cache: false)
-    def for(family: nil, account: nil)
+    @mode = set_mode!(mode)
-      new(family: family, account: account)
+    @clear_cache = clear_cache
-    end
   end
 
-  # Syncer can optionally be scoped.  Otherwise, it syncs all user data
+  def sync
-  def initialize(family: nil, account: nil)
+    sync_prices
-    @family = family
+    sync_exchange_rates
-    @account = account
   end
 
-  def sync_all(full_history: false)
+  # Syncs historical security prices (and details)
-    sync_exchange_rates(full_history: full_history)
+  def sync_prices
-    sync_prices(full_history: full_history)
+    unless Security.provider
-  end
+      Rails.logger.warn("No provider configured for MarketDataSyncer.sync_prices, skipping sync")
-
-  def sync_exchange_rates(full_history: false)
-    unless rate_provider
-      Rails.logger.warn("No rate provider configured for MarketDataSyncer.sync_exchange_rates, skipping sync")
       return
     end
 
-    # Finds distinct currency pairs
+    Security.where.not(exchange_operating_mic: nil).find_each do |security|
-    entry_pairs = entries_scope.joins(:account)
+      security.sync_provider_prices(
-                                  .where.not("entries.currency = accounts.currency")
+        start_date: get_first_required_price_date(security),
-                                  .select("entries.currency as source, accounts.currency as target")
+        end_date: end_date,
-                                  .distinct
+        clear_cache: clear_cache
+      )
 
-    # All accounts in currency not equal to the family currency require exchange rates to show a normalized historical graph
+      security.sync_provider_details(clear_cache: clear_cache)
-    account_pairs = accounts_scope.joins(:family)
-                                  .where.not("families.currency = accounts.currency")
-                                  .select("accounts.currency as source, families.currency as target")
-                                  .distinct
-
-    pairs = (entry_pairs + account_pairs).uniq
-
-    pairs.each do |pair|
-      sync_exchange_rate(from: pair.source, to: pair.target, full_history: full_history)
     end
   end
 
-  def sync_prices(full_history: false)
+  def sync_exchange_rates
-    unless price_provider
+    unless ExchangeRate.provider
-      Rails.logger.warn("No price provider configured for MarketDataSyncer.sync_prices, skipping sync")
+      Rails.logger.warn("No provider configured for MarketDataSyncer.sync_exchange_rates, skipping sync")
-      nil
+      return
     end
 
-    securities_scope.each do |security|
+    required_exchange_rate_pairs.each do |pair|
-      sync_security_price(security: security, full_history: full_history)
+      # pair is a Hash with keys :source, :target, and :start_date
+      start_date = snapshot? ? default_start_date : pair[:start_date]
+
+      ExchangeRate.sync_provider_rates(
+        from: pair[:source],
+        to: pair[:target],
+        start_date: start_date,
+        end_date: end_date,
+        clear_cache: clear_cache
+      )
     end
   end
 
   private
-    attr_reader :family, :account
+    attr_reader :mode, :clear_cache
 
-    def accounts_scope
+    def snapshot?
-      return Account.where(id: account.id) if account
+      mode.to_sym == :snapshot
-      return family.accounts if family
-      Account.all
     end
 
-    def entries_scope
+    # Builds a unique list of currency pairs with the earliest date we need
-      account&.entries || family&.entries || Entry.all
+    # exchange rates for.
+    #
+    # Returns: Array of Hashes – [{ source:, target:, start_date: }, ...]
+    def required_exchange_rate_pairs
+      pair_dates = {} # { [source, target] => earliest_date }
+
+      # 1. ENTRY-BASED PAIRS – we need rates from the first entry date
+      Entry.joins(:account)
+           .where.not("entries.currency = accounts.currency")
+           .group("entries.currency", "accounts.currency")
+           .minimum("entries.date")
+           .each do |(source, target), date|
+        key = [ source, target ]
+        pair_dates[key] = [ pair_dates[key], date ].compact.min
       end
 
-    def securities_scope
+      # 2. ACCOUNT-BASED PAIRS – use the account's oldest entry date
-      if account
+      account_first_entry_dates = Entry.group(:account_id).minimum(:date)
-        account.trades.joins(:security).where.not(securities: { exchange_operating_mic: nil })
+
-      elsif family
+      Account.joins(:family)
-        family.trades.joins(:security).where.not(securities: { exchange_operating_mic: nil })
+             .where.not("families.currency = accounts.currency")
-      else
+             .select("accounts.id, accounts.currency AS source, families.currency AS target")
-        Security.where.not(exchange_operating_mic: nil)
+             .find_each do |account|
+        earliest_entry_date = account_first_entry_dates[account.id]
+
+        chosen_date = [ earliest_entry_date, default_start_date ].compact.min
+
+        key = [ account.source, account.target ]
+        pair_dates[key] = [ pair_dates[key], chosen_date ].compact.min
+      end
+
+      # Convert to array of hashes for ease of use
+      pair_dates.map do |(source, target), date|
+        { source: source, target: target, start_date: date }
       end
     end
 
-    def sync_security_price(security:, full_history:)
+    def get_first_required_price_date(security)
-      start_date = full_history ? find_oldest_required_price(security: security) : default_start_date
+      return default_start_date if snapshot?
 
-      Rails.logger.info("Syncing security price for: #{security.ticker}, start_date: #{start_date}, end_date: #{end_date}")
+      Trade.with_entry.where(security: security).minimum(:date)
-
-      fetched_prices = price_provider.fetch_security_prices(
-        security,
-        start_date: start_date,
-        end_date: end_date
-      )
-
-      unless fetched_prices.success?
-        error = MissingSecurityPriceError.new(
-          "#{PRICE_PROVIDER_NAME} could not fetch security price for: #{security.ticker} between: #{start_date} and: #{Date.current}.  Provider error: #{fetched_prices.error.message}"
-        )
-
-        Rails.logger.warn(error.message)
-        Sentry.capture_exception(error, level: :warning)
-
-        return
     end
 
-      prices_for_upsert = fetched_prices.data.map do |price|
+    # An approximation that grabs more than we likely need, but simplifies the logic
-        if price.security.nil? || price.date.nil? || price.price.nil? || price.currency.nil?
+    def get_first_required_exchange_rate_date(from_currency:)
-          error = InvalidSecurityPriceDataError.new(
+      return default_start_date if snapshot?
-            "#{PRICE_PROVIDER_NAME} returned invalid price data for security: #{security.ticker} on: #{price.date}.  Price data: #{price.inspect}"
-          )
 
-          Rails.logger.warn(error.message)
+      Entry.where(currency: from_currency).minimum(:date)
-          Sentry.capture_exception(error, level: :warning)
-
-          next
-        end
-
-        {
-          security_id: price.security.id,
-          date: price.date,
-          price: price.price,
-          currency: price.currency
-        }
-      end.compact
-
-      Security::Price.upsert_all(
-        prices_for_upsert,
-        unique_by: %i[security_id date currency]
-      )
-    end
-
-    def sync_exchange_rate(from:, to:, full_history:)
-      start_date = full_history ? find_oldest_required_rate(from_currency: from) : default_start_date
-
-      Rails.logger.info("Syncing exchange rate from: #{from}, to: #{to}, start_date: #{start_date}, end_date: #{end_date}")
-
-      fetched_rates = rate_provider.fetch_exchange_rates(
-        from: from,
-        to: to,
-        start_date: start_date,
-        end_date: end_date
-      )
-
-      unless fetched_rates.success?
-        message = "#{RATE_PROVIDER_NAME} could not fetch exchange rate pair from: #{from} to: #{to} between: #{start_date} and: #{Date.current}.  Provider error: #{fetched_rates.error.message}"
-        Rails.logger.warn(message)
-        Sentry.capture_exception(MissingExchangeRateError.new(message))
-        return
-      end
-
-      rates_for_upsert = fetched_rates.data.map do |rate|
-        if rate.from.nil? || rate.to.nil? || rate.date.nil? || rate.rate.nil?
-          message = "#{RATE_PROVIDER_NAME} returned invalid rate data for pair from: #{from} to: #{to} on: #{rate.date}.  Rate data: #{rate.inspect}"
-          Rails.logger.warn(message)
-          Sentry.capture_exception(InvalidExchangeRateDataError.new(message))
-          next
-        end
-
-        {
-          from_currency: rate.from,
-          to_currency: rate.to,
-          date: rate.date,
-          rate: rate.rate
-        }
-      end.compact
-
-      ExchangeRate.upsert_all(
-        rates_for_upsert,
-        unique_by: %i[from_currency to_currency date]
-      )
-    end
-
-    def rate_provider
-      Provider::Registry.for_concept(:exchange_rates).get_provider(RATE_PROVIDER_NAME)
-    end
-
-    def price_provider
-      Provider::Registry.for_concept(:securities).get_provider(PRICE_PROVIDER_NAME)
-    end
-
-    def find_oldest_required_rate(from_currency:)
-      entries_scope.where(currency: from_currency).minimum(:date) || default_start_date
     end
 
     def default_start_date
-      DEFAULT_HISTORY_DAYS.days.ago.to_date
+      SNAPSHOT_DAYS.days.ago.to_date
     end
 
     # Since we're querying market data from a US-based API, end date should always be today (EST)
     def end_date
       Date.current.in_time_zone("America/New_York").to_date
     end
+
+    def set_mode!(mode)
+      valid_modes = [ :full, :snapshot ]
+
+      unless valid_modes.include?(mode.to_sym)
+        raise InvalidModeError, "Invalid mode for MarketDataSyncer, can only be :full or :snapshot, but was #{mode}"
+      end
+
+      mode.to_sym
+    end
 end

app/models/provider/security_concept.rb

@@ -2,22 +2,22 @@ module Provider::SecurityConcept
   extend ActiveSupport::Concern
 
   Security = Data.define(:symbol, :name, :logo_url, :exchange_operating_mic)
-  SecurityInfo = Data.define(:symbol, :name, :links, :logo_url, :description, :kind)
+  SecurityInfo = Data.define(:symbol, :name, :links, :logo_url, :description, :kind, :exchange_operating_mic)
-  Price = Data.define(:security, :date, :price, :currency)
+  Price = Data.define(:symbol, :date, :price, :currency, :exchange_operating_mic)
 
   def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
     raise NotImplementedError, "Subclasses must implement #search_securities"
   end
 
-  def fetch_security_info(security)
+  def fetch_security_info(symbol:, exchange_operating_mic:)
     raise NotImplementedError, "Subclasses must implement #fetch_security_info"
   end
 
-  def fetch_security_price(security, date:)
+  def fetch_security_price(symbol:, exchange_operating_mic:, date:)
     raise NotImplementedError, "Subclasses must implement #fetch_security_price"
   end
 
-  def fetch_security_prices(security, start_date:, end_date:)
+  def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:)
     raise NotImplementedError, "Subclasses must implement #fetch_security_prices"
   end
 end

app/models/provider/synth.rb

@@ -3,6 +3,8 @@ class Provider::Synth < Provider
 
   # Subclass so errors caught in this provider are raised as Provider::Synth::Error
   Error = Class.new(Provider::Error)
+  InvalidExchangeRateError = Class.new(Error)
+  InvalidSecurityPriceError = Class.new(Error)
 
   def initialize(api_key)
     @api_key = api_key
@@ -48,7 +50,7 @@ class Provider::Synth < Provider
 
       rates = JSON.parse(response.body).dig("data", "rates")
 
-      Rate.new(date:, from:, to:, rate: rates.dig(to))
+      Rate.new(date: date.to_date, from:, to:, rate: rates.dig(to))
     end
   end
 
@@ -65,8 +67,18 @@ class Provider::Synth < Provider
       end
 
       data.paginated.map do |rate|
-        Rate.new(date: rate.dig("date"), from:, to:, rate: rate.dig("rates", to))
+        date = rate.dig("date")
+        rate = rate.dig("rates", to)
+
+        if date.nil? || rate.nil?
+          message = "#{self.class.name} returned invalid rate data for pair from: #{from} to: #{to} on: #{date}.  Rate data: #{rate.inspect}"
+          Rails.logger.warn(message)
+          Sentry.capture_exception(InvalidExchangeRateError.new(message), level: :warning)
+          next
         end
+
+        Rate.new(date: date.to_date, from:, to:, rate:)
+      end.compact
     end
   end
 
@@ -97,65 +109,73 @@ class Provider::Synth < Provider
     end
   end
 
-  def fetch_security_info(security)
+  def fetch_security_info(symbol:, exchange_operating_mic:)
     with_provider_response do
-      response = client.get("#{base_url}/tickers/#{security.ticker}") do |req|
+      response = client.get("#{base_url}/tickers/#{symbol}") do |req|
-        req.params["mic_code"] = security.exchange_mic if security.exchange_mic.present?
+        req.params["operating_mic"] = exchange_operating_mic
-        req.params["operating_mic"] = security.exchange_operating_mic if security.exchange_operating_mic.present?
       end
 
       data = JSON.parse(response.body).dig("data")
 
       SecurityInfo.new(
-        symbol: data.dig("ticker"),
+        symbol: symbol,
         name: data.dig("name"),
         links: data.dig("links"),
         logo_url: data.dig("logo_url"),
         description: data.dig("description"),
-        kind: data.dig("kind")
+        kind: data.dig("kind"),
+        exchange_operating_mic: exchange_operating_mic
       )
     end
   end
 
-  def fetch_security_price(security, date:)
+  def fetch_security_price(symbol:, exchange_operating_mic:, date:)
     with_provider_response do
-      historical_data = fetch_security_prices(security, start_date: date, end_date: date)
+      historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
 
-      raise ProviderError, "No prices found for security #{security.ticker} on date #{date}" if historical_data.data.empty?
+      raise ProviderError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.empty?
 
       historical_data.data.first
     end
   end
 
-  def fetch_security_prices(security, start_date:, end_date:)
+  def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:)
     with_provider_response do
       params = {
         start_date: start_date,
-        end_date: end_date
+        end_date: end_date,
+        operating_mic_code: exchange_operating_mic
       }
 
-      params[:operating_mic_code] = security.exchange_operating_mic if security.exchange_operating_mic.present?
-
       data = paginate(
-        "#{base_url}/tickers/#{security.ticker}/open-close",
+        "#{base_url}/tickers/#{symbol}/open-close",
         params
       ) do |body|
         body.dig("prices")
       end
 
       currency = data.first_page.dig("currency")
-      country_code = data.first_page.dig("exchange", "country_code")
-      exchange_mic = data.first_page.dig("exchange", "mic_code")
       exchange_operating_mic = data.first_page.dig("exchange", "operating_mic_code")
 
       data.paginated.map do |price|
-        Price.new(
+        date = price.dig("date")
-          security: security,
+        price = price.dig("close") || price.dig("open")
-          date: price.dig("date"),
+
-          price: price.dig("close") || price.dig("open"),
+        if date.nil? || price.nil?
-          currency: currency
+          message = "#{self.class.name} returned invalid price data for security #{symbol} on: #{date}.  Price data: #{price.inspect}"
-        )
+          Rails.logger.warn(message)
+          Sentry.capture_exception(InvalidSecurityPriceError.new(message), level: :warning)
+          next
         end
+
+        Price.new(
+          symbol: symbol,
+          date: date.to_date,
+          price: price,
+          currency: currency,
+          exchange_operating_mic: exchange_operating_mic
+        )
+      end.compact
     end
   end
 

app/models/security/price/syncer.rb

@@ -0,0 +1,145 @@
+class Security::Price::Syncer
+  MissingSecurityPriceError = Class.new(StandardError)
+  MissingStartPriceError    = Class.new(StandardError)
+
+  def initialize(security:, security_provider:, start_date:, end_date:, clear_cache: false)
+    @security          = security
+    @security_provider = security_provider
+    @start_date        = start_date
+    @end_date          = normalize_end_date(end_date)
+    @clear_cache       = clear_cache
+  end
+
+  # Constructs a daily series of prices for a single security over the date range.
+  # Returns the number of rows upserted.
+  def sync_provider_prices
+    if !clear_cache && all_prices_exist?
+      Rails.logger.info("No new prices to sync for #{security.ticker} between #{start_date} and #{end_date}, skipping")
+      return 0
+    end
+
+    if clear_cache && provider_prices.empty?
+      Rails.logger.warn("Could not clear cache for #{security.ticker} between #{start_date} and #{end_date} because provider returned no prices")
+      return 0
+    end
+
+    prev_price_value = start_price_value
+
+    unless prev_price_value.present?
+      error = MissingStartPriceError.new("Could not find a start price for #{security.ticker} on or before #{start_date}")
+      Rails.logger.error(error.message)
+      Sentry.capture_exception(error)
+      return 0
+    end
+
+    gapfilled_prices = effective_start_date.upto(end_date).map do |date|
+      db_price_value       = db_prices[date]&.price
+      provider_price_value = provider_prices[date]&.price
+      provider_currency    = provider_prices[date]&.currency
+
+      chosen_price = if clear_cache
+        provider_price_value || db_price_value   # overwrite when possible
+      else
+        db_price_value || provider_price_value   # fill gaps
+      end
+
+      # Gap-fill using LOCF (last observation carried forward)
+      chosen_price ||= prev_price_value
+      prev_price_value = chosen_price
+
+      {
+        security_id: security.id,
+        date:        date,
+        price:       chosen_price,
+        currency:    provider_currency || prev_price_currency || db_price_currency || "USD"
+      }
+    end
+
+    upsert_rows(gapfilled_prices)
+  end
+
+  private
+    attr_reader :security, :security_provider, :start_date, :end_date, :clear_cache
+
+    def provider_prices
+      @provider_prices ||= begin
+        provider_fetch_start_date = effective_start_date - 5.days
+
+        response = security_provider.fetch_security_prices(
+          symbol: security.ticker,
+          exchange_operating_mic: security.exchange_operating_mic,
+          start_date: provider_fetch_start_date,
+          end_date:   end_date
+        )
+
+        if response.success?
+          response.data.index_by(&:date)
+        else
+          msg = "#{security_provider.class.name} could not fetch prices for #{security.ticker} between #{provider_fetch_start_date} and #{end_date}. Provider error: #{response.error.message}"
+          Rails.logger.warn(msg)
+          Sentry.capture_exception(MissingSecurityPriceError.new(msg))
+          {}
+        end
+      end
+    end
+
+    def db_prices
+      @db_prices ||= Security::Price.where(security_id: security.id, date: start_date..end_date)
+                                    .order(:date)
+                                    .to_a
+                                    .index_by(&:date)
+    end
+
+    def all_prices_exist?
+      db_prices.count == expected_count
+    end
+
+    def expected_count
+      (start_date..end_date).count
+    end
+
+    # Skip over ranges that already exist unless clearing cache
+    def effective_start_date
+      return start_date if clear_cache
+
+      (start_date..end_date).detect { |d| !db_prices.key?(d) } || end_date
+    end
+
+    def start_price_value
+      provider_price_value = provider_prices.select { |date, _| date <= start_date }
+                                            .max_by { |date, _| date }
+                                            &.last&.price
+      db_price_value       = db_prices[start_date]&.price
+      provider_price_value || db_price_value
+    end
+
+    def upsert_rows(rows)
+      batch_size         = 200
+      total_upsert_count = 0
+
+      rows.each_slice(batch_size) do |batch|
+        ids = Security::Price.upsert_all(
+          batch,
+          unique_by: %i[security_id date currency],
+          returning: [ "id" ]
+        )
+        total_upsert_count += ids.count
+      end
+
+      total_upsert_count
+    end
+
+    def db_price_currency
+      db_prices.values.first&.currency
+    end
+
+    def prev_price_currency
+      @prev_price_currency ||= provider_prices.values.first&.currency
+    end
+
+    # Clamp to today (EST) so we never call our price API for a future date (our API is in EST/EDT timezone)
+    def normalize_end_date(requested_end_date)
+      today_est = Date.current.in_time_zone("America/New_York").to_date
+      [ requested_end_date, today_est ].min
+    end
+end

app/models/security/provided.rb

@@ -49,6 +49,48 @@ module Security::Provided
     price
   end
 
+  def sync_provider_details(clear_cache: false)
+    unless provider.present?
+      Rails.logger.warn("No provider configured for Security.sync_provider_details")
+      return
+    end
+
+    if self.name.present? && self.logo_url.present? && !clear_cache
+      return
+    end
+
+    response = provider.fetch_security_info(
+      symbol: ticker,
+      exchange_operating_mic: exchange_operating_mic
+    )
+
+    if response.success?
+      update(
+        name: response.data.name,
+        logo_url: response.data.logo_url,
+      )
+    else
+      err = StandardError.new("Failed to fetch security info for #{ticker} from #{provider.class.name}: #{response.error.message}")
+      Rails.logger.warn(err.message)
+      Sentry.capture_exception(err, level: :warning)
+    end
+  end
+
+  def sync_provider_prices(start_date:, end_date:, clear_cache: false)
+    unless provider.present?
+      Rails.logger.warn("No provider configured for Security.sync_provider_prices")
+      return 0
+    end
+
+    Security::Price::Syncer.new(
+      security: self,
+      security_provider: provider,
+      start_date: start_date,
+      end_date: end_date,
+      clear_cache: clear_cache
+    ).sync_provider_prices
+  end
+
   private
     def provider
       self.class.provider

app/models/stock_exchange.rb

@@ -1,3 +0,0 @@
-class StockExchange < ApplicationRecord
-  scope :in_country, ->(country_code) { where(country_code: country_code) }
-end

app/models/trade_builder.rb

@@ -129,13 +129,9 @@ class TradeBuilder
     def security
       ticker_symbol, exchange_operating_mic = ticker.present? ? ticker.split("|") : [ manual_ticker, nil ]
 
-      security = Security.find_or_create_by!(
+      Security.find_or_create_by!(
         ticker: ticker_symbol,
         exchange_operating_mic: exchange_operating_mic
       )
-
-      FetchSecurityInfoJob.perform_later(security.id)
-
-      security
     end
 end

config/initializers/sidekiq.rb

@@ -1,5 +1,6 @@
 require "sidekiq/web"
 
+if Rails.env.production?
   Sidekiq::Web.use(Rack::Auth::Basic) do |username, password|
     configured_username = ::Digest::SHA256.hexdigest(ENV.fetch("SIDEKIQ_WEB_USERNAME", "maybe"))
     configured_password = ::Digest::SHA256.hexdigest(ENV.fetch("SIDEKIQ_WEB_PASSWORD", "maybe"))
@@ -7,6 +8,7 @@ Sidekiq::Web.use(Rack::Auth::Basic) do |username, password|
     ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(username), configured_username) &&
       ActiveSupport::SecurityUtils.secure_compare(::Digest::SHA256.hexdigest(password), configured_password)
   end
+end
 
 Sidekiq::Cron.configure do |config|
   # 10 min "catch-up" window in case worker process is re-deploying when cron tick occurs

config/schedule.yml

@@ -1,5 +1,8 @@
 sync_market_data:
-  cron: "0 17 * * 1-5" # 5:00 PM EST (1 hour after market close)
+  cron: "0 22 * * 1-5" # 5:00 PM EST / 6:00 PM EDT (NY time)
   class: "SyncMarketDataJob"
   queue: "scheduled"
   description: "Syncs market data daily at 5:00 PM EST (1 hour after market close)"
+  args:
+    mode: "full"
+    clear_cache: false

db/migrate/20250516180846_remove_stock_exchanges.rb

@@ -0,0 +1,11 @@
+class RemoveStockExchanges < ActiveRecord::Migration[7.2]
+  def change
+    drop_table :stock_exchanges do |t|
+      t.string :name, null: false
+      t.string :acronym
+      t.string :mic, null: false
+      t.string :country, null: false
+      t.string :country_code, null: false
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_05_14_214242) do
+ActiveRecord::Schema[7.2].define(version: 2025_05_16_180846) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -550,27 +550,6 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_14_214242) do
     t.index ["var"], name: "index_settings_on_var", unique: true
   end
 
-  create_table "stock_exchanges", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
-    t.string "name", null: false
-    t.string "acronym"
-    t.string "mic", null: false
-    t.string "country", null: false
-    t.string "country_code", null: false
-    t.string "city"
-    t.string "website"
-    t.string "timezone_name"
-    t.string "timezone_abbr"
-    t.string "timezone_abbr_dst"
-    t.string "currency_code"
-    t.string "currency_symbol"
-    t.string "currency_name"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.index ["country"], name: "index_stock_exchanges_on_country"
-    t.index ["country_code"], name: "index_stock_exchanges_on_country_code"
-    t.index ["currency_code"], name: "index_stock_exchanges_on_currency_code"
-  end
-
   create_table "subscriptions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.uuid "family_id", null: false
     t.string "status", null: false

test/fixtures/stock_exchanges.yml

@@ -1,13 +0,0 @@
-nasdaq:
-  name: NASDAQ
-  mic: XNAS
-  acronym: NASDAQ
-  country: USA
-  country_code: US
-
-nyse:
-  name: New York Stock Exchange
-  mic: XNYS
-  acronym: NYSE
-  country: USA
-  country_code: US

test/interfaces/exchange_rate_provider_interface_test.rb

@@ -15,6 +15,7 @@ module ExchangeRateProviderInterfaceTest
 
       assert_equal "USD", rate.from
       assert_equal "GBP", rate.to
+      assert rate.date.is_a?(Date)
       assert_in_delta 0.78, rate.rate, 0.01
     end
   end
@@ -26,6 +27,7 @@ module ExchangeRateProviderInterfaceTest
       )
 
       assert_equal 213, response.data.count # 213 days between 01.01.2024 and 31.07.2024
+      assert response.data.first.date.is_a?(Date)
     end
   end
 

test/interfaces/security_provider_interface_test.rb

@@ -7,7 +7,7 @@ module SecurityProviderInterfaceTest
     aapl = securities(:aapl)
 
     VCR.use_cassette("#{vcr_key_prefix}/security_price") do
-      response = @subject.fetch_security_price(aapl, date: Date.iso8601("2024-08-01"))
+      response = @subject.fetch_security_price(symbol: aapl.ticker, exchange_operating_mic: aapl.exchange_operating_mic, date: Date.iso8601("2024-08-01"))
 
       assert response.success?
       assert response.data.present?
@@ -19,12 +19,14 @@ module SecurityProviderInterfaceTest
 
     VCR.use_cassette("#{vcr_key_prefix}/security_prices") do
       response = @subject.fetch_security_prices(
-        aapl,
+        symbol: aapl.ticker,
+        exchange_operating_mic: aapl.exchange_operating_mic,
         start_date: Date.iso8601("2024-01-01"),
         end_date: Date.iso8601("2024-08-01")
       )
 
       assert response.success?
+      assert response.data.first.date.is_a?(Date)
       assert_equal 147, response.data.count # Synth won't return prices on weekends / holidays, so less than total day count of 213
     end
   end
@@ -44,7 +46,11 @@ module SecurityProviderInterfaceTest
     aapl = securities(:aapl)
 
     VCR.use_cassette("#{vcr_key_prefix}/security_info") do
-      response = @subject.fetch_security_info(aapl)
+      response = @subject.fetch_security_info(
+        symbol: aapl.ticker,
+        exchange_operating_mic: aapl.exchange_operating_mic
+      )
+
       info = response.data
 
       assert_equal "AAPL", info.symbol

test/models/account/convertible_test.rb

@@ -1,52 +0,0 @@
-require "test_helper"
-require "ostruct"
-
-class Account::ConvertibleTest < ActiveSupport::TestCase
-  include EntriesTestHelper, ProviderTestHelper
-
-  setup do
-    @family = families(:empty)
-    @family.update!(currency: "USD")
-
-    # Foreign account (currency is not in the family's primary currency, so it will require exchange rates for net worth rollups)
-    @account = @family.accounts.create!(name: "Test Account", currency: "EUR", balance: 10000, accountable: Depository.new)
-
-    @provider = mock
-    ExchangeRate.stubs(:provider).returns(@provider)
-  end
-
-  test "syncs required exchange rates for an account" do
-    create_valuation(account: @account, date: 1.day.ago.to_date, amount: 9500, currency: "EUR")
-
-    # Since we had a valuation 1 day ago, this account starts 2 days ago and needs daily exchange rates looking forward
-    assert_equal 2.days.ago.to_date, @account.start_date
-
-    ExchangeRate.delete_all
-
-    provider_response = provider_success_response(
-      [
-        OpenStruct.new(from: "EUR", to: "USD", date: 2.days.ago.to_date, rate: 1.1),
-        OpenStruct.new(from: "EUR", to: "USD", date: 1.day.ago.to_date, rate: 1.2),
-        OpenStruct.new(from: "EUR", to: "USD", date: Date.current, rate: 1.3)
-      ]
-    )
-
-    @provider.expects(:fetch_exchange_rates)
-             .with(from: "EUR", to: "USD", start_date: 2.days.ago.to_date, end_date: Date.current)
-             .returns(provider_response)
-
-    assert_difference "ExchangeRate.count", 3 do
-      @account.sync_required_exchange_rates
-    end
-  end
-
-  test "does not sync rates for a domestic account" do
-    @account.update!(currency: "USD")
-
-    @provider.expects(:fetch_exchange_rates).never
-
-    assert_no_difference "ExchangeRate.count" do
-      @account.sync_required_exchange_rates
-    end
-  end
-end

test/models/account/market_data_syncer_test.rb

@@ -0,0 +1,107 @@
+require "test_helper"
+require "ostruct"
+
+class Account::MarketDataSyncerTest < ActiveSupport::TestCase
+  include ProviderTestHelper
+
+  PROVIDER_BUFFER = 5.days
+
+  setup do
+    # Ensure a clean slate for deterministic assertions
+    Security::Price.delete_all
+    ExchangeRate.delete_all
+    Trade.delete_all
+    Holding.delete_all
+    Security.delete_all
+    Entry.delete_all
+
+    @provider = mock("provider")
+    Provider::Registry.any_instance
+                      .stubs(:get_provider)
+                      .with(:synth)
+                      .returns(@provider)
+  end
+
+  test "syncs required exchange rates for a foreign-currency account" do
+    family = Family.create!(name: "Smith", currency: "USD")
+
+    account = family.accounts.create!(
+      name: "Chequing",
+      currency: "CAD",
+      balance: 100,
+      accountable: Depository.new
+    )
+
+    # Seed a rate for the first required day so that the syncer only needs the next day forward
+    existing_date = account.start_date
+    ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: existing_date, rate: 2.0)
+
+    expected_start_date = (existing_date + 1.day) - PROVIDER_BUFFER
+    end_date            = Date.current.in_time_zone("America/New_York").to_date
+
+    @provider.expects(:fetch_exchange_rates)
+             .with(from: "CAD",
+                   to: "USD",
+                   start_date: expected_start_date,
+                   end_date: end_date)
+             .returns(provider_success_response([
+               OpenStruct.new(from: "CAD", to: "USD", date: existing_date, rate: 1.5)
+             ]))
+
+    before = ExchangeRate.count
+    Account::MarketDataSyncer.new(account).sync_market_data
+    after  = ExchangeRate.count
+
+    assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
+  end
+
+  test "syncs security prices for securities traded by the account" do
+    family = Family.create!(name: "Smith", currency: "USD")
+
+    account = family.accounts.create!(
+      name: "Brokerage",
+      currency: "USD",
+      balance: 0,
+      accountable: Investment.new
+    )
+
+    security = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
+
+    trade_date = 10.days.ago.to_date
+    trade      = Trade.new(security: security, qty: 1, price: 100, currency: "USD")
+
+    account.entries.create!(
+      name: "Buy AAPL",
+      date: trade_date,
+      amount: 100,
+      currency: "USD",
+      entryable: trade
+    )
+
+    expected_start_date = trade_date - PROVIDER_BUFFER
+    end_date            = Date.current.in_time_zone("America/New_York").to_date
+
+    @provider.expects(:fetch_security_prices)
+             .with(symbol: security.ticker,
+                   exchange_operating_mic: security.exchange_operating_mic,
+                   start_date: expected_start_date,
+                   end_date: end_date)
+             .returns(provider_success_response([
+               OpenStruct.new(security: security,
+                              date: trade_date,
+                              price: 100,
+                              currency: "USD")
+             ]))
+
+    @provider.stubs(:fetch_security_info)
+             .with(symbol: security.ticker, exchange_operating_mic: security.exchange_operating_mic)
+             .returns(provider_success_response(OpenStruct.new(name: "Apple", logo_url: "logo")))
+
+    # Ignore exchange-rate calls for this test
+    @provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
+
+    Account::MarketDataSyncer.new(account).sync_market_data
+
+    assert_equal 1, Security::Price.where(security: security, date: trade_date).count
+  end
+end

test/models/exchange_rate/syncer_test.rb

@@ -0,0 +1,148 @@
+require "test_helper"
+require "ostruct"
+
+class ExchangeRate::SyncerTest < ActiveSupport::TestCase
+  include ProviderTestHelper
+
+  setup do
+    @provider = mock
+  end
+
+  test "syncs missing rates from provider" do
+    ExchangeRate.delete_all
+
+    provider_response = provider_success_response([
+      OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.3),
+      OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
+      OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.5)
+    ])
+
+    @provider.expects(:fetch_exchange_rates)
+             .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    ExchangeRate::Syncer.new(
+      exchange_rate_provider: @provider,
+      from: "USD",
+      to: "EUR",
+      start_date: 2.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_rates
+
+    db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date..Date.current)
+                           .order(:date)
+
+    assert_equal 3, db_rates.count
+    assert_equal 1.3, db_rates[0].rate
+    assert_equal 1.4, db_rates[1].rate
+    assert_equal 1.5, db_rates[2].rate
+  end
+
+  test "syncs diff when some rates already exist" do
+    ExchangeRate.delete_all
+
+    # Pre-populate DB with the first two days
+    ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date: 3.days.ago.to_date, rate: 1.2)
+    ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date, rate: 1.25)
+
+    provider_response = provider_success_response([
+      OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.3)
+    ])
+
+    @provider.expects(:fetch_exchange_rates)
+             .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    ExchangeRate::Syncer.new(
+      exchange_rate_provider: @provider,
+      from: "USD",
+      to: "EUR",
+      start_date: 3.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_rates
+
+    db_rates = ExchangeRate.order(:date)
+    assert_equal 4, db_rates.count
+    assert_equal [ 1.2, 1.25, 1.3, 1.3 ], db_rates.map(&:rate)
+  end
+
+  test "no provider calls when all rates exist" do
+    ExchangeRate.delete_all
+
+    (3.days.ago.to_date..Date.current).each_with_index do |date, idx|
+      ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date:, rate: 1.2 + idx * 0.01)
+    end
+
+    @provider.expects(:fetch_exchange_rates).never
+
+    ExchangeRate::Syncer.new(
+      exchange_rate_provider: @provider,
+      from: "USD",
+      to: "EUR",
+      start_date: 3.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_rates
+  end
+
+  # A helpful "reset" option for when we need to refresh provider data
+  test "full upsert if clear_cache is true" do
+    ExchangeRate.delete_all
+
+    # Seed DB with stale data
+    (2.days.ago.to_date..Date.current).each do |date|
+      ExchangeRate.create!(from_currency: "USD", to_currency: "EUR", date:, rate: 1.0)
+    end
+
+    provider_response = provider_success_response([
+      OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.3),
+      OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
+      OpenStruct.new(from: "USD", to: "EUR", date: Date.current,        rate: 1.5)
+    ])
+
+    @provider.expects(:fetch_exchange_rates)
+             .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    ExchangeRate::Syncer.new(
+      exchange_rate_provider: @provider,
+      from: "USD",
+      to: "EUR",
+      start_date: 2.days.ago.to_date,
+      end_date: Date.current,
+      clear_cache: true
+    ).sync_provider_rates
+
+    db_rates = ExchangeRate.where(from_currency: "USD", to_currency: "EUR").order(:date)
+    assert_equal [ 1.3, 1.4, 1.5 ], db_rates.map(&:rate)
+  end
+
+  test "clamps end_date to today when future date is provided" do
+    ExchangeRate.delete_all
+
+    future_date = Date.current + 3.days
+
+    provider_response = provider_success_response([
+      OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.6)
+    ])
+
+    @provider.expects(:fetch_exchange_rates)
+             .with(from: "USD", to: "EUR", start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
+             .returns(provider_response)
+
+    ExchangeRate::Syncer.new(
+      exchange_rate_provider: @provider,
+      from: "USD",
+      to: "EUR",
+      start_date: Date.current,
+      end_date: future_date
+    ).sync_provider_rates
+
+    assert_equal 1, ExchangeRate.count
+  end
+
+  private
+    def get_provider_fetch_start_date(start_date)
+      # We fetch with a 5 day buffer to account for weekends and holidays
+      start_date - 5.days
+    end
+end

test/models/exchange_rate_test.rb

@@ -67,26 +67,4 @@ class ExchangeRateTest < ActiveSupport::TestCase
 
     assert_nil ExchangeRate.find_or_fetch_rate(from: "USD", to: "EUR", date: Date.current, cache: true)
   end
-
-  test "upserts rates for currency pair and date range" do
-    ExchangeRate.delete_all
-
-    ExchangeRate.create!(date: 1.day.ago.to_date, from_currency: "USD", to_currency: "EUR", rate: 0.9)
-
-    provider_response = provider_success_response([
-      OpenStruct.new(from: "USD", to: "EUR", date: Date.current, rate: 1.3),
-      OpenStruct.new(from: "USD", to: "EUR", date: 1.day.ago.to_date, rate: 1.4),
-      OpenStruct.new(from: "USD", to: "EUR", date: 2.days.ago.to_date, rate: 1.5)
-    ])
-
-    @provider.expects(:fetch_exchange_rates)
-             .with(from: "USD", to: "EUR", start_date: 2.days.ago.to_date, end_date: Date.current)
-             .returns(provider_response)
-
-    ExchangeRate.sync_provider_rates(from: "USD", to: "EUR", start_date: 2.days.ago.to_date)
-
-    assert_equal 1.3, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: Date.current).rate
-    assert_equal 1.4, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: 1.day.ago.to_date).rate
-    assert_equal 1.5, ExchangeRate.find_by(from_currency: "USD", to_currency: "EUR", date: 2.days.ago.to_date).rate
-  end
 end

test/models/market_data_syncer_test.rb

@@ -2,70 +2,84 @@ require "test_helper"
 require "ostruct"
 
 class MarketDataSyncerTest < ActiveSupport::TestCase
-  include EntriesTestHelper, ProviderTestHelper
+  include ProviderTestHelper
 
-  test "syncs exchange rates with upsert" do
+  SNAPSHOT_START_DATE = MarketDataSyncer::SNAPSHOT_DAYS.days.ago.to_date
-    empty_db
+  PROVIDER_BUFFER     = 5.days
 
-    family1 = Family.create!(name: "Family 1", currency: "USD")
+  setup do
-    account1 = family1.accounts.create!(name: "Account 1", currency: "USD", balance: 100, accountable: Depository.new)
+    Security::Price.delete_all
-    account2 = family1.accounts.create!(name: "Account 2", currency: "CAD", balance: 100, accountable: Depository.new)
+    ExchangeRate.delete_all
+    Trade.delete_all
+    Holding.delete_all
+    Security.delete_all
 
-    family2 = Family.create!(name: "Family 2", currency: "EUR")
+    @provider = mock("provider")
-    account3 = family2.accounts.create!(name: "Account 3", currency: "EUR", balance: 100, accountable: Depository.new)
+    Provider::Registry.any_instance
-    account4 = family2.accounts.create!(name: "Account 4", currency: "USD", balance: 100, accountable: Depository.new)
+                      .stubs(:get_provider)
+                      .with(:synth)
+                      .returns(@provider)
+  end
 
-    mock_provider = mock
+  test "syncs required exchange rates" do
-    Provider::Registry.any_instance.expects(:get_provider).with(:synth).returns(mock_provider).at_least_once
+    family = Family.create!(name: "Smith", currency: "USD")
+    family.accounts.create!(name: "Chequing",
+                            currency: "CAD",
+                            balance: 100,
+                            accountable: Depository.new)
 
-    start_date = 1.month.ago.to_date
+    # Seed stale rate so only the next missing day is fetched
+    ExchangeRate.create!(from_currency: "CAD",
+                         to_currency: "USD",
+                         date: SNAPSHOT_START_DATE,
+                         rate: 2.0)
+
+    expected_start_date = (SNAPSHOT_START_DATE + 1.day) - PROVIDER_BUFFER
     end_date            = Date.current.in_time_zone("America/New_York").to_date
 
-    # Put an existing rate in DB to test upsert
+    @provider.expects(:fetch_exchange_rates)
-    ExchangeRate.create!(from_currency: "CAD", to_currency: "USD", date: start_date, rate: 2.0)
+             .with(from: "CAD",
+                   to: "USD",
+                   start_date: expected_start_date,
+                   end_date: end_date)
+             .returns(provider_success_response([
+               OpenStruct.new(from: "CAD", to: "USD", date: SNAPSHOT_START_DATE, rate: 1.5)
+             ]))
 
-    mock_provider.expects(:fetch_exchange_rates)
+    before = ExchangeRate.count
-                 .with(from: "CAD", to: "USD", start_date: start_date, end_date: end_date)
+    MarketDataSyncer.new(mode: :snapshot).sync_exchange_rates
-                 .returns(provider_success_response([ OpenStruct.new(from: "CAD", to: "USD", date: start_date, rate: 1.0) ]))
+    after  = ExchangeRate.count
 
-    mock_provider.expects(:fetch_exchange_rates)
+    assert_operator after, :>, before, "Should insert at least one new exchange-rate row"
-                 .with(from: "USD", to: "EUR", start_date: start_date, end_date: end_date)
-                 .returns(provider_success_response([ OpenStruct.new(from: "USD", to: "EUR", date: start_date, rate: 1.0) ]))
-
-    assert_difference "ExchangeRate.count", 1 do
-      MarketDataSyncer.new.sync_exchange_rates
   end
 
-    assert_equal 1.0, ExchangeRate.where(from_currency: "CAD", to_currency: "USD", date: start_date).first.rate
+  test "syncs security prices" do
-  end
+    security = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
 
-  test "syncs security prices with upsert" do
+    expected_start_date = SNAPSHOT_START_DATE - PROVIDER_BUFFER
-    empty_db
-
-    aapl = Security.create!(ticker: "AAPL", exchange_operating_mic: "XNAS")
-
-    family = Family.create!(name: "Family 1", currency: "USD")
-    account = family.accounts.create!(name: "Account 1", currency: "USD", balance: 100, accountable: Investment.new)
-
-    mock_provider = mock
-    Provider::Registry.any_instance.expects(:get_provider).with(:synth).returns(mock_provider).at_least_once
-
-    start_date = 1.month.ago.to_date
     end_date            = Date.current.in_time_zone("America/New_York").to_date
 
-    mock_provider.expects(:fetch_security_prices)
+    @provider.expects(:fetch_security_prices)
-                 .with(aapl, start_date: start_date, end_date: end_date)
+             .with(symbol: security.ticker,
-                 .returns(provider_success_response([ OpenStruct.new(security: aapl, date: start_date, price: 100, currency: "USD") ]))
+                   exchange_operating_mic: security.exchange_operating_mic,
+                   start_date: expected_start_date,
+                   end_date: end_date)
+             .returns(provider_success_response([
+               OpenStruct.new(security: security,
+                              date: SNAPSHOT_START_DATE,
+                              price: 100,
+                              currency: "USD")
+             ]))
 
-    assert_difference "Security::Price.count", 1 do
+    @provider.stubs(:fetch_security_info)
-      MarketDataSyncer.new.sync_prices
+             .with(symbol: "AAPL", exchange_operating_mic: "XNAS")
-    end
+             .returns(provider_success_response(OpenStruct.new(name: "Apple", logo_url: "logo")))
-  end
 
-  private
+    # Ignore exchange rate calls for this test
-    def empty_db
+    @provider.stubs(:fetch_exchange_rates).returns(provider_success_response([]))
-      Invitation.destroy_all
+
-      Family.destroy_all
+    MarketDataSyncer.new(mode: :snapshot).sync_prices
-      Security.destroy_all
+
+    assert_equal 1, Security::Price.where(security: security, date: SNAPSHOT_START_DATE).count
   end
 end

test/models/security/price/syncer_test.rb

@@ -0,0 +1,143 @@
+require "test_helper"
+require "ostruct"
+
+class Security::Price::SyncerTest < ActiveSupport::TestCase
+  include ProviderTestHelper
+
+  setup do
+    @provider = mock
+    @security = Security.create!(ticker: "AAPL")
+  end
+
+  test "syncs missing prices from provider" do
+    Security::Price.delete_all
+
+    provider_response = provider_success_response([
+      OpenStruct.new(security: @security, date: 2.days.ago.to_date, price: 150, currency: "USD"),
+      OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 155, currency: "USD"),
+      OpenStruct.new(security: @security, date: Date.current, price: 160, currency: "USD")
+    ])
+
+    @provider.expects(:fetch_security_prices)
+             .with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
+                   start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    Security::Price::Syncer.new(
+      security: @security,
+      security_provider: @provider,
+      start_date: 2.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_prices
+
+    db_prices = Security::Price.where(security: @security, date: 2.days.ago.to_date..Date.current).order(:date)
+
+    assert_equal 3, db_prices.count
+    assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
+  end
+
+  test "syncs diff when some prices already exist" do
+    Security::Price.delete_all
+
+    # Pre-populate DB with first two days
+    Security::Price.create!(security: @security, date: 3.days.ago.to_date, price: 140, currency: "USD")
+    Security::Price.create!(security: @security, date: 2.days.ago.to_date, price: 145, currency: "USD")
+
+    provider_response = provider_success_response([
+      OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 150, currency: "USD")
+    ])
+
+    @provider.expects(:fetch_security_prices)
+             .with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
+                   start_date: get_provider_fetch_start_date(1.day.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    Security::Price::Syncer.new(
+      security: @security,
+      security_provider: @provider,
+      start_date: 3.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_prices
+
+    db_prices = Security::Price.where(security: @security).order(:date)
+    assert_equal 4, db_prices.count
+    assert_equal [ 140, 145, 150, 150 ], db_prices.map(&:price)
+  end
+
+  test "no provider calls when all prices exist" do
+    Security::Price.delete_all
+
+    (3.days.ago.to_date..Date.current).each_with_index do |date, idx|
+      Security::Price.create!(security: @security, date:, price: 100 + idx, currency: "USD")
+    end
+
+    @provider.expects(:fetch_security_prices).never
+
+    Security::Price::Syncer.new(
+      security: @security,
+      security_provider: @provider,
+      start_date: 3.days.ago.to_date,
+      end_date: Date.current
+    ).sync_provider_prices
+  end
+
+  test "full upsert if clear_cache is true" do
+    Security::Price.delete_all
+
+    # Seed DB with stale prices
+    (2.days.ago.to_date..Date.current).each do |date|
+      Security::Price.create!(security: @security, date:, price: 100, currency: "USD")
+    end
+
+    provider_response = provider_success_response([
+      OpenStruct.new(security: @security, date: 2.days.ago.to_date, price: 150, currency: "USD"),
+      OpenStruct.new(security: @security, date: 1.day.ago.to_date, price: 155, currency: "USD"),
+      OpenStruct.new(security: @security, date: Date.current,        price: 160, currency: "USD")
+    ])
+
+    @provider.expects(:fetch_security_prices)
+             .with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
+                   start_date: get_provider_fetch_start_date(2.days.ago.to_date), end_date: Date.current)
+             .returns(provider_response)
+
+    Security::Price::Syncer.new(
+      security: @security,
+      security_provider: @provider,
+      start_date: 2.days.ago.to_date,
+      end_date: Date.current,
+      clear_cache: true
+    ).sync_provider_prices
+
+    db_prices = Security::Price.where(security: @security).order(:date)
+    assert_equal [ 150, 155, 160 ], db_prices.map(&:price)
+  end
+
+  test "clamps end_date to today when future date is provided" do
+    Security::Price.delete_all
+
+    future_date = Date.current + 3.days
+
+    provider_response = provider_success_response([
+      OpenStruct.new(security: @security, date: Date.current, price: 165, currency: "USD")
+    ])
+
+    @provider.expects(:fetch_security_prices)
+             .with(symbol: @security.ticker, exchange_operating_mic: @security.exchange_operating_mic,
+                   start_date: get_provider_fetch_start_date(Date.current), end_date: Date.current)
+             .returns(provider_response)
+
+    Security::Price::Syncer.new(
+      security: @security,
+      security_provider: @provider,
+      start_date: Date.current,
+      end_date: future_date
+    ).sync_provider_prices
+
+    assert_equal 1, Security::Price.count
+  end
+
+  private
+    def get_provider_fetch_start_date(start_date)
+      start_date - 5.days
+    end
+end

test/vcr_cassettes/synth/exchange_rate.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sat, 15 Mar 2025 22:18:46 GMT
+      - Fri, 16 May 2025 13:01:38 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -35,11 +35,11 @@ http_interactions:
       Cache-Control:
       - max-age=0, private, must-revalidate
       Etag:
-      - W/"b0b21c870fe53492404cc5ac258fa465"
+      - W/"0c93a67d0c68e6f206e2954a41aa2933"
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - 44367fcb-e5b4-457d
+      - 146e30b2-e03b-47e3
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - 8ce9dc85-afbd-437c-b18d-ec788b712334
+      - 3cf7ade1-8066-422a-97c7-5f8b99e24296
       X-Runtime:
-      - '0.031963'
+      - '0.024284'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=SwRPS1vBsrKtk%2Ftb7Ix8j%2FCWYw9tZgbJxR1FCmotWn%2FIZAE3Ri%2FUwHtvkOSqBq6HN5pLVetfem5hp%2BkqWmD5GRCVho0mp3VgRr3J1tBMwrVK2p50tfpmb3X22Jj%2BOfapq1C22PnN"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=ih8sEFqAOWyINqAEtKGKPKO2lr1qAYSVeipyB5F8g2umPODXvCD4hN3G6wTTs2Q7H8CDWsqiOlYkmVvmr%2BWvl2ojOtBwO25Ahk9TbhlcgRO9nT6mEIXOSdVXJpzpRn5Ov%2FMGigpQ"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,13 +69,13 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 920f6378fe582237-ORD
+      - 940b109b5df1a3d7-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=26670&min_rtt=26569&rtt_var=10167&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2829&recv_bytes=922&delivery_rate=105759&cwnd=181&unsent_bytes=0&cid=f0a872e0b2909c59&ts=188&x=0"
+      - cfL4;desc="?proto=TCP&rtt=25865&min_rtt=25683&rtt_var=9996&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2827&recv_bytes=922&delivery_rate=106690&cwnd=219&unsent_bytes=0&cid=e48ae188d1f86721&ts=190&x=0"
     body:
       encoding: ASCII-8BIT
-      string: '{"data":{"date":"2024-01-01","source":"USD","rates":{"GBP":0.785476}},"meta":{"total_records":1,"credits_used":1,"credits_remaining":249830,"date":"2024-01-01"}}'
+      string: '{"data":{"date":"2024-01-01","source":"USD","rates":{"GBP":0.785476}},"meta":{"total_records":1,"credits_used":1,"credits_remaining":249734,"date":"2024-01-01"}}'
-  recorded_at: Sat, 15 Mar 2025 22:18:46 GMT
+  recorded_at: Fri, 16 May 2025 13:01:38 GMT
 recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/health.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sat, 15 Mar 2025 22:18:47 GMT
+      - Fri, 16 May 2025 13:01:39 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -35,11 +35,11 @@ http_interactions:
       Cache-Control:
       - max-age=0, private, must-revalidate
       Etag:
-      - W/"4ec3e0a20895d90b1e1241ca67f10ca3"
+      - W/"c5c1d51b68b499d00936c9eb1e8bfdbb"
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - 0cab64c9-e312-4bec
+      - 3abc1256-5517-44a7
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - 1958563c-7c18-4201-a03c-a4b343dc68ab
+      - aaf85301-dd16-4b9b-a3a4-c4fbcf1d3f55
       X-Runtime:
-      - '0.014938'
+      - '0.014386'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=P3OWn4c8LFFWI0Dwr2CSYwHLaNhf9iD9TfAhqdx5PtLoWZ0pSImebfUsh00ZbOmh4r2cRJEQOmvy67wAwl6p0W%2Fx9017EkCnCaXibBBCKqJTBOdGnsSuV%2B45LrHsQmg%2BGeBwrw4b"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=OaVSdNPSl6CQ8gbhnDzkCisX2ILOEWAwweMW3rXXP5rBKuxZoDT024srQWmHKGLsCEhpt4G9mqCthDwlHu2%2BuZ3AyTJQcnBONtE%2FNQ7fKT9x8nLz4mnqL8iyynLuRWQSUJ8SWMj5"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,14 +69,14 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 920f637aa8cf1152-ORD
+      - 940b109d086eb4b8-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=25627&min_rtt=25594&rtt_var=9664&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2827&recv_bytes=878&delivery_rate=111991&cwnd=248&unsent_bytes=0&cid=c8e4c4e269114d14&ts=263&x=0"
+      - cfL4;desc="?proto=TCP&rtt=32457&min_rtt=26792&rtt_var=14094&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2826&recv_bytes=878&delivery_rate=108091&cwnd=229&unsent_bytes=0&cid=a6f330e4d5f16682&ts=309&x=0"
     body:
       encoding: ASCII-8BIT
-      string: '{"id":"user_3208c49393f54b3e974795e4bea5b864","email":"test@maybe.co","name":"Test
+      string: '{"id":"user_3208c49393f54b3e974795e4bea5b864","email":"zach@maybe.co","name":"Zach
-        User","plan":"Business","api_calls_remaining":249830,"api_limit":250000,"credits_reset_at":"2025-04-01T00:00:00.000-04:00","current_period_start":"2025-03-01T00:00:00.000-05:00"}'
+        Gollwitzer","plan":"Business","api_calls_remaining":249733,"api_limit":250000,"credits_reset_at":"2025-06-01T00:00:00.000-04:00","current_period_start":"2025-05-01T00:00:00.000-04:00"}'
-  recorded_at: Sat, 15 Mar 2025 22:18:47 GMT
+  recorded_at: Fri, 16 May 2025 13:01:39 GMT
 recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/security_info.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sun, 16 Mar 2025 12:04:12 GMT
+      - Fri, 16 May 2025 13:01:37 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -35,11 +35,11 @@ http_interactions:
       Cache-Control:
       - max-age=0, private, must-revalidate
       Etag:
-      - W/"a9deeb6437d359f080be449b9b2c547b"
+      - W/"75f336ad88e262c72044e8b865265298"
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - 1e77ae49-050a-45fc
+      - ba973abf-7d96-4a9a
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - 222dacf1-37f3-4eb8-91d5-edf13d732d46
+      - 76cb13a6-0d7e-4c36-8df9-bb63110d9e2a
       X-Runtime:
-      - '0.059222'
+      - '0.099716'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=%2BLW%2Fd%2BbcNg4%2FleO6ECyB4RJBMbm6vWG3%2FX4oKQXfn1ROSPVrISc3ZFVlXfITGW4XYJSPyUDF%2FXrrRF6p3Wzow07QamOrsux7sxBMvtWmcubgpCMFI4zgnhESklW6KcmAefwrgj9i"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=aDn7ApAO9Ma86gZ%2BJKCUCFjH2Re%2BtXdB5gcqYj2KTGXJKNpgf5TNgzbrp5%2Bw%2FGL5nTvtp%2B7cxT8MMcLWjAV6Ne1r6z5YBFq1K4W7Zw5m1lhMiqYLnTnEs2Oq85TjzOvpsE%2BmC33d"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,11 +69,11 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 92141c97bfd9124c-ORD
+      - 940b10910abdd2ec-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=27459&min_rtt=26850&rtt_var=11288&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2828&recv_bytes=905&delivery_rate=91272&cwnd=104&unsent_bytes=0&cid=ccd6aa7e48e4b0eb&ts=287&x=0"
+      - cfL4;desc="?proto=TCP&rtt=28163&min_rtt=27237&rtt_var=12066&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2827&recv_bytes=905&delivery_rate=83590&cwnd=239&unsent_bytes=0&cid=7ef62bd693b52ccd&ts=240&x=0"
     body:
       encoding: ASCII-8BIT
       string: '{"data":{"ticker":"AAPL","name":"Apple Inc.","links":{"homepage_url":"https://www.apple.com"},"logo_url":"https://logo.synthfinance.com/ticker/AAPL","description":"Apple
@@ -100,6 +100,6 @@ http_interactions:
         Apple Park Way","city":"Cupertino","state":"CA","postal_code":"95014"},"exchange":{"name":"Nasdaq/Ngs
         (Global Select Market)","mic_code":"XNGS","operating_mic_code":"XNAS","acronym":"NGS","country":"United
         States","country_code":"US","timezone":"America/New_York"},"ceo":"Mr. Timothy
-        D. Cook","founding_year":1976,"industry":"Consumer Electronics","sector":"Technology","phone":"408-996-1010","total_employees":161000,"composite_figi":"BBG000B9Y5X2","market_data":{"high_today":213.95,"low_today":209.58,"open_today":211.25,"close_today":213.49,"volume_today":60060200.0,"fifty_two_week_high":260.1,"fifty_two_week_low":164.08,"average_volume":62848099.37313433,"price_change":0.0,"percent_change":0.0}},"meta":{"credits_used":1,"credits_remaining":249808}}'
+        D. Cook","founding_year":1976,"industry":"Consumer Electronics","sector":"Technology","phone":"408-996-1010","total_employees":161000,"composite_figi":"BBG000B9Y5X2","market_data":{"high_today":212.96,"low_today":209.54,"open_today":210.95,"close_today":211.45,"volume_today":44979900.0,"fifty_two_week_high":260.1,"fifty_two_week_low":169.21,"average_volume":61769396.875,"price_change":0.0,"percent_change":0.0}},"meta":{"credits_used":1,"credits_remaining":249737}}'
-  recorded_at: Sun, 16 Mar 2025 12:04:12 GMT
+  recorded_at: Fri, 16 May 2025 13:01:37 GMT
 recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/security_price.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sun, 16 Mar 2025 12:08:00 GMT
+      - Fri, 16 May 2025 13:01:36 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -35,11 +35,11 @@ http_interactions:
       Cache-Control:
       - max-age=0, private, must-revalidate
       Etag:
-      - W/"cdf04c2cd77e230c03117dd13d0921f9"
+      - W/"72340d82266397447b865407dda15492"
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - e74b3425-0b7c-447d
+      - 4c3462aa-2471-40b4
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - b906c5e1-18cc-44cc-9085-313ff066a6ce
+      - bdbc757d-2528-44c3-ae08-9788e8ee15f7
       X-Runtime:
-      - '0.544708'
+      - '0.034898'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=dZNe6qCGGI2XGXgByLr69%2FYrDQdy2FLtnXafxJnlsvyVjrRFiCvmbbIzgF5CDgtj9HZ8RC5Rh9jbuEI6hPokpa3Al4FEIAZB5AbfZ9toP%2Bc5muG%2FuBgHR%2FnIZpsWG%2BQKmBPu9MBa"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=2Mu4PK4XTsAq%2Bn1%2F2yxy%2Blj7kz3ZCiQ9t8ikr2m19BrhQhrqfeUQfPwxbLc1WIgGMIxpPInKYtDVIX3En%2FGpTNQLAeu%2FpuLKv%2BRmCx%2B7u28od5L%2F9%2BLmEhFWqJjs8Y6C1O2a3SKv"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,15 +69,15 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 921422292d0feacc-ORD
+      - 940b108f29129d03-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=30826&min_rtt=26727&rtt_var=12950&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2827&recv_bytes=970&delivery_rate=108354&cwnd=219&unsent_bytes=0&cid=43c717161effdc57&ts=695&x=0"
+      - cfL4;desc="?proto=TCP&rtt=27793&min_rtt=26182&rtt_var=13041&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2827&recv_bytes=970&delivery_rate=74111&cwnd=244&unsent_bytes=0&cid=9bcc030369a615fb&ts=210&x=0"
     body:
       encoding: ASCII-8BIT
       string: '{"ticker":"AAPL","currency":"USD","exchange":{"name":"Nasdaq/Ngs (Global
         Select Market)","mic_code":"XNGS","operating_mic_code":"XNAS","acronym":"NGS","country":"United
-        States","country_code":"US","timezone":"America/New_York"},"prices":[{"date":"2024-08-01","open":224.37,"close":218.36,"high":224.48,"low":217.02,"volume":62501000}],"paging":{"prev":"/tickers/AAPL/open-close?end_date=2024-08-01\u0026operating_mic_code=XNAS\u0026page=\u0026start_date=2024-08-01","next":"/tickers/AAPL/open-close?end_date=2024-08-01\u0026operating_mic_code=XNAS\u0026page=\u0026start_date=2024-08-01","total_records":1,"current_page":1,"per_page":100,"total_pages":1},"meta":{"credits_used":1,"credits_remaining":249807}}'
+        States","country_code":"US","timezone":"America/New_York"},"prices":[{"date":"2024-08-01","open":224.37,"close":218.36,"high":224.48,"low":217.02,"volume":62501000}],"paging":{"prev":"/tickers/AAPL/open-close?end_date=2024-08-01\u0026operating_mic_code=XNAS\u0026page=\u0026start_date=2024-08-01","next":"/tickers/AAPL/open-close?end_date=2024-08-01\u0026operating_mic_code=XNAS\u0026page=\u0026start_date=2024-08-01","total_records":1,"current_page":1,"per_page":100,"total_pages":1},"meta":{"total_records":1,"credits_used":1,"credits_remaining":249738}}'
-  recorded_at: Sun, 16 Mar 2025 12:08:00 GMT
+  recorded_at: Fri, 16 May 2025 13:01:36 GMT
 recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/security_search.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sun, 16 Mar 2025 12:01:58 GMT
+      - Fri, 16 May 2025 13:01:38 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -39,7 +39,7 @@ http_interactions:
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - 2effb56b-f67f-402d
+      - 701ae22a-18c8-4e62
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - 33470619-5119-4923-b4e0-e9a0eeb532a1
+      - edb55bc6-e3ea-470b-b7af-9b4d9883420b
       X-Runtime:
-      - '0.453770'
+      - '0.355152'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=ayZOlXkCwLgUl%2FrB2%2BlqtqR5HCllubf4HLDipEt3klWKyHS4nilHi9XZ1fiEQWx7xwiRMJZ5EW0Xzm7ISoHWTtEbkgMQHWYQwSTeg30ahFFHK1pkOOnET1fuW1UxiZwlJtq1XZGB"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=QGeBWdYED%2F%2FgT9BzborFAnM%2FG6UiNmI0ej212XGHWdFwYXUvTJ2GyqA9hMJrpYIvgbHdQ9Ed0MsQUv3KFb57VXQq0T6UXTNPa%2BFRPepK0hsXeGDLxch04v6KnkTATqcw2M8HuYHS"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,11 +69,11 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 921419514e0a6399-ORD
+      - 940b1097a830f856-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=25809&min_rtt=25801&rtt_var=9692&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2829&recv_bytes=939&delivery_rate=111952&cwnd=121&unsent_bytes=0&cid=2beb787f15cd8ab9&ts=610&x=0"
+      - cfL4;desc="?proto=TCP&rtt=26401&min_rtt=25556&rtt_var=11273&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2825&recv_bytes=939&delivery_rate=89615&cwnd=244&unsent_bytes=0&cid=cf6d0758d165295d&ts=500&x=0"
     body:
       encoding: ASCII-8BIT
       string: '{"data":[{"symbol":"AAPL","name":"Apple Inc.","logo_url":"https://logo.synthfinance.com/ticker/AAPL","currency":"USD","exchange":{"name":"Nasdaq/Ngs
@@ -100,5 +100,5 @@ http_interactions:
         Inc.","logo_url":"https://logo.synthfinance.com/ticker/AAPJ","currency":"USD","exchange":{"name":"Otc
         Pink Marketplace","mic_code":"PINX","operating_mic_code":"OTCM","acronym":"","country":"United
         States","country_code":"US","timezone":"America/New_York"}}]}'
-  recorded_at: Sun, 16 Mar 2025 12:01:58 GMT
+  recorded_at: Fri, 16 May 2025 13:01:38 GMT
 recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/transaction_enrich.yml

@@ -1,82 +0,0 @@
----
-http_interactions:
-- request:
-    method: get
-    uri: https://api.synthfinance.com/enrich?amount=25.5&city=San%20Francisco&country=US&date=2025-03-16&description=UBER%20EATS&state=CA
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Authorization:
-      - Bearer <SYNTH_API_KEY>
-      X-Source:
-      - maybe_app
-      X-Source-Type:
-      - managed
-      User-Agent:
-      - Faraday v2.12.2
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Date:
-      - Sun, 16 Mar 2025 12:09:33 GMT
-      Content-Type:
-      - application/json; charset=utf-8
-      Transfer-Encoding:
-      - chunked
-      Connection:
-      - keep-alive
-      Cache-Control:
-      - max-age=0, private, must-revalidate
-      Etag:
-      - W/"00411c83cfeaade519bcc3e57d9e461e"
-      Referrer-Policy:
-      - strict-origin-when-cross-origin
-      Rndr-Id:
-      - 56a8791d-85ed-4342
-      Strict-Transport-Security:
-      - max-age=63072000; includeSubDomains
-      Vary:
-      - Accept-Encoding
-      X-Content-Type-Options:
-      - nosniff
-      X-Frame-Options:
-      - SAMEORIGIN
-      X-Permitted-Cross-Domain-Policies:
-      - none
-      X-Render-Origin-Server:
-      - Render
-      X-Request-Id:
-      - 1b35b9c1-0092-40b1-8b70-2bce7c5796af
-      X-Runtime:
-      - '0.884634'
-      X-Xss-Protection:
-      - '0'
-      Cf-Cache-Status:
-      - DYNAMIC
-      Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=qUtB0aWbK%2Fh5W7cV%2FugsUGbWKtJzsf%2FXd5i8cm8KlepEtLyuVPH7XX0fqwzHp43OCWQkGr9r8hRBBSEcx9LWW5vS7%2B1kXCJaKPaTRn%2BWtsEymHg78OHqDcMahwSuy%2FkpSGLWo0or"}],"group":"cf-nel","max_age":604800}'
-      Nel:
-      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
-      Speculation-Rules:
-      - '"/cdn-cgi/speculation"'
-      Server:
-      - cloudflare
-      Cf-Ray:
-      - 921424681aa4acab-ORD
-      Alt-Svc:
-      - h3=":443"; ma=86400
-      Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=26975&min_rtt=26633&rtt_var=10231&sent=4&recv=6&lost=0&retrans=0&sent_bytes=2829&recv_bytes=969&delivery_rate=108737&cwnd=210&unsent_bytes=0&cid=318ff675628918e1&ts=1035&x=0"
-    body:
-      encoding: ASCII-8BIT
-      string: '{"merchant":"Uber Eats","merchant_id":"mer_aea41e7f29ce47b5873f3caf49d5972d","category":"Dining
-        Out","website":"ubereats.com","icon":"https://logo.synthfinance.com/ubereats.com","meta":{"credits_used":1,"credits_remaining":249806}}'
-  recorded_at: Sun, 16 Mar 2025 12:09:33 GMT
-recorded_with: VCR 6.3.1

test/vcr_cassettes/synth/usage.yml

@@ -14,7 +14,7 @@ http_interactions:
       X-Source-Type:
       - managed
       User-Agent:
-      - Faraday v2.12.2
+      - Faraday v2.13.1
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -25,7 +25,7 @@ http_interactions:
       message: OK
     headers:
       Date:
-      - Sat, 15 Mar 2025 22:18:47 GMT
+      - Fri, 16 May 2025 13:01:36 GMT
       Content-Type:
       - application/json; charset=utf-8
       Transfer-Encoding:
@@ -35,11 +35,11 @@ http_interactions:
       Cache-Control:
       - max-age=0, private, must-revalidate
       Etag:
-      - W/"4ec3e0a20895d90b1e1241ca67f10ca3"
+      - W/"7b8c2bf0cba54bc26b78bdc6e611dcbd"
       Referrer-Policy:
       - strict-origin-when-cross-origin
       Rndr-Id:
-      - 54c8ecf9-6858-4db6
+      - 1b53adf6-b391-45b2
       Strict-Transport-Security:
       - max-age=63072000; includeSubDomains
       Vary:
@@ -53,15 +53,15 @@ http_interactions:
       X-Render-Origin-Server:
       - Render
       X-Request-Id:
-      - a4112cfb-0eac-4e3e-a880-7536d90dcba0
+      - f88670a2-81d2-48b6-8d73-a911c846e330
       X-Runtime:
-      - '0.007036'
+      - '0.018749'
       X-Xss-Protection:
       - '0'
       Cf-Cache-Status:
       - DYNAMIC
       Report-To:
-      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=Rt0BTtrgXzYjWOQFgb%2Bg6N4xKvXtPI66Q251bq9nWtqUhGHo17GmVVAPkutwN7Gisw1RmvYfxYUiMCCxlc4%2BjuHxbU1%2BXr9KHy%2F5pUpLhgLNNrtkqqKOCW4GduODnDbw2I38Rocu"}],"group":"cf-nel","max_age":604800}'
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=oH4OsWB6itK0jpi%2FPs%2BswVyCZIbkJGPfyJaoR4TKFtTAfmnqa8Lp6aZhv22WKzotXJuAKbh99VdYdZIOkeIPWbYTc6j4rGw%2BkQB3Hw%2Fc44QxDBJFdIo6wJNe8TGiPAZ%2BvgoBVHWn"}],"group":"cf-nel","max_age":604800}'
       Nel:
       - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
       Speculation-Rules:
@@ -69,14 +69,14 @@ http_interactions:
       Server:
       - cloudflare
       Cf-Ray:
-      - 920f637d1fe8eb68-ORD
+      - 940b108c38f66392-ORD
       Alt-Svc:
       - h3=":443"; ma=86400
       Server-Timing:
-      - cfL4;desc="?proto=TCP&rtt=28779&min_rtt=27036&rtt_var=11384&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2828&recv_bytes=878&delivery_rate=107116&cwnd=203&unsent_bytes=0&cid=52bc39ad09dd9eff&ts=145&x=0"
+      - cfL4;desc="?proto=TCP&rtt=33369&min_rtt=25798&rtt_var=15082&sent=5&recv=6&lost=0&retrans=0&sent_bytes=2826&recv_bytes=878&delivery_rate=112256&cwnd=205&unsent_bytes=0&cid=1b13324eb0768fd3&ts=285&x=0"
     body:
       encoding: ASCII-8BIT
-      string: '{"id":"user_3208c49393f54b3e974795e4bea5b864","email":"test@maybe.co","name":"Test
+      string: '{"id":"user_3208c49393f54b3e974795e4bea5b864","email":"zach@maybe.co","name":"Zach
-        User","plan":"Business","api_calls_remaining":1200,"api_limit":5000,"credits_reset_at":"2025-04-01T00:00:00.000-04:00","current_period_start":"2025-03-01T00:00:00.000-05:00"}'
+        Gollwitzer","plan":"Business","api_calls_remaining":249738,"api_limit":250000,"credits_reset_at":"2025-06-01T00:00:00.000-04:00","current_period_start":"2025-05-01T00:00:00.000-04:00"}'
-  recorded_at: Sat, 15 Mar 2025 22:18:47 GMT
+  recorded_at: Fri, 16 May 2025 13:01:36 GMT
 recorded_with: VCR 6.3.1