Security resolver and health checker (#2281)

* Setup health check

* Security health checker cron

* Use resolver throughout codebase

* Use resolver for trade builder

* Add security health checks to schedule

* Handle no provider

* Lint fixes

app/models/security/health_checker.rb

@@ -0,0 +1,120 @@
+# There are hundreds of thousands of market securities that Maybe must handle.
+# Due to the always-changing nature of the market, the health checker is responsible
+# for periodically checking active securities to ensure we can still fetch prices for them.
+#
+# Each security goes through some basic health checks.  If failed, this class is responsible for:
+# - Marking failed attempts and incrementing the failed attempts counter
+# - Marking the security offline if enough consecutive failed checks occur
+# - When we move a security "offline", delete all prices for that security as we assume they are bad data
+#
+# The health checker is run daily through SecurityHealthCheckJob (see config/schedule.yml), but not all
+# securities will be checked every day (we run in batches)
+class Security::HealthChecker
+  MAX_CONSECUTIVE_FAILURES = 5
+  HEALTH_CHECK_INTERVAL = 7.days
+  DAILY_BATCH_SIZE = 1000
+
+  class << self
+    def check_all
+      # No daily limit for unchecked securities (they are prioritized)
+      never_checked_scope.find_each do |security|
+        new(security).run_check
+      end
+
+      # Daily limit for checked securities
+      due_for_check_scope.limit(DAILY_BATCH_SIZE).each do |security|
+        new(security).run_check
+      end
+    end
+
+    private
+      # If a security has never had a health check, we prioritize it, regardless of batch size
+      def never_checked_scope
+        Security.where(last_health_check_at: nil)
+      end
+
+      # Any securities not checked for 30 days are due
+      # We only process the batch size, which means some "due" securities will not be checked today
+      # This is by design, to prevent all securities from coming due at the same time
+      def due_for_check_scope
+        Security.where(last_health_check_at: ..HEALTH_CHECK_INTERVAL.ago)
+                .order(last_health_check_at: :asc)
+      end
+  end
+
+  def initialize(security)
+    @security = security
+  end
+
+  def run_check
+    Rails.logger.info("Running health check for #{security.ticker}")
+
+    if latest_provider_price
+      handle_success
+    else
+      handle_failure
+    end
+  rescue => e
+    Sentry.capture_exception(e) do |scope|
+      scope.set_tags(security_id: @security.id)
+    end
+  ensure
+    security.update!(last_health_check_at: Time.current)
+  end
+
+  private
+    attr_reader :security
+
+    def provider
+      Security.provider
+    end
+
+    def latest_provider_price
+      return nil unless provider.present?
+
+      response = provider.fetch_security_price(
+        symbol: security.ticker,
+        exchange_operating_mic: security.exchange_operating_mic,
+        date: Date.current
+      )
+
+      return nil unless response.success?
+
+      response.data.price
+    end
+
+    # On success, reset any failure counters and ensure it is "online"
+    def handle_success
+      security.update!(
+        offline: false,
+        failed_fetch_count: 0,
+        failed_fetch_at: nil
+      )
+    end
+
+    def handle_failure
+      new_failure_count = security.failed_fetch_count.to_i + 1
+      new_failure_at = Time.current
+
+      if new_failure_count > MAX_CONSECUTIVE_FAILURES
+        convert_to_offline_security!
+      else
+        security.update!(
+          failed_fetch_count: new_failure_count,
+          failed_fetch_at: new_failure_at
+        )
+      end
+    end
+
+    # The "offline" state tells our MarketDataImporter (daily cron) to skip this security when fetching prices
+    def convert_to_offline_security!
+      Security.transaction do
+        security.update!(
+          offline: true,
+          failed_fetch_count: MAX_CONSECUTIVE_FAILURES + 1,
+          failed_fetch_at: Time.current
+        )
+        security.prices.delete_all
+      end
+    end
+end

app/models/security/provided.rb

@@ -10,9 +10,14 @@ module Security::Provided
     end
 
     def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)
-      return [] if symbol.blank? || symbol.length < 2
+      return [] if provider.nil? || symbol.blank?
 
-      response = provider.search_securities(symbol, country_code: country_code, exchange_operating_mic: exchange_operating_mic)
+      params = {
+        country_code: country_code,
+        exchange_operating_mic: exchange_operating_mic
+      }.compact_blank
+
+      response = provider.search_securities(symbol, **params)
 
       if response.success?
         response.data.map do |provider_security|
@@ -38,7 +43,11 @@ module Security::Provided
 
     # Make sure we have a data provider before fetching
     return nil unless provider.present?
-    response = provider.fetch_security_price(self, date: date)
+    response = provider.fetch_security_price(
+      symbol: ticker,
+      exchange_operating_mic: exchange_operating_mic,
+      date: date
+    )
 
     return nil unless response.success? # Provider error
 

app/models/security/resolver.rb

@@ -0,0 +1,156 @@
+class Security::Resolver
+  def initialize(symbol, exchange_operating_mic: nil, country_code: nil)
+    @symbol = validate_symbol!(symbol)
+    @exchange_operating_mic = exchange_operating_mic
+    @country_code = country_code
+  end
+
+  # Attempts several paths to resolve a security:
+  # 1. Exact match in DB
+  # 2. Search provider for an exact match
+  # 3. Search provider for close match, ranked by relevance
+  # 4. Create offline security if no match is found in either DB or provider
+  def resolve
+    return nil if symbol.blank?
+
+    exact_match_from_db ||
+      exact_match_from_provider ||
+      close_match_from_provider ||
+      offline_security
+  end
+
+  private
+    attr_reader :symbol, :exchange_operating_mic, :country_code
+
+    def validate_symbol!(symbol)
+      raise ArgumentError, "Symbol is required and cannot be blank" if symbol.blank?
+      symbol.strip.upcase
+    end
+
+    def offline_security
+      security = Security.find_or_initialize_by(
+        ticker: symbol,
+        exchange_operating_mic: exchange_operating_mic,
+      )
+
+      security.assign_attributes(
+        country_code: country_code,
+        offline: true # This tells us that we shouldn't try to fetch prices later
+      )
+
+      security.save!
+
+      security
+    end
+
+    def exact_match_from_db
+      Security.find_by(
+        {
+          ticker: symbol,
+          exchange_operating_mic: exchange_operating_mic,
+          country_code: country_code.presence
+        }.compact
+      )
+    end
+
+    # If provided a ticker + exchange (and optionally, a country code), we can find exact matches
+    def exact_match_from_provider
+      # Without an exchange, we can never know if we have an exact match
+      return nil unless exchange_operating_mic.present?
+
+      match = provider_search_result.find do |s|
+        ticker_matches = s.ticker.upcase.to_s == symbol.upcase.to_s
+        exchange_matches = s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s
+
+        if country_code && exchange_operating_mic
+          ticker_matches && exchange_matches && s.country_code.upcase.to_s == country_code.upcase.to_s
+        else
+          ticker_matches && exchange_matches
+        end
+      end
+
+      return nil unless match
+
+      find_or_create_provider_match!(match)
+    end
+
+    def close_match_from_provider
+      filtered_candidates = provider_search_result
+
+      # If a country code is specified, we MUST find a match with the same code
+      if country_code.present?
+        filtered_candidates = filtered_candidates.select { |s| s.country_code.upcase.to_s == country_code.upcase.to_s }
+      end
+
+      # 1. Prefer exact exchange_operating_mic matches (if one was provided)
+      # 2. Rank by country relevance (lower index in the list is more relevant)
+      # 3. Rank by exchange_operating_mic relevance (lower index in the list is more relevant)
+      sorted_candidates = filtered_candidates.sort_by do |s|
+        [
+          exchange_operating_mic.present? && s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
+          sorted_country_codes_by_relevance.index(s.country_code&.upcase.to_s) || sorted_country_codes_by_relevance.length,
+          sorted_exchange_operating_mics_by_relevance.index(s.exchange_operating_mic&.upcase.to_s) || sorted_exchange_operating_mics_by_relevance.length
+        ]
+      end
+
+      match = sorted_candidates.first
+
+      return nil unless match
+
+      find_or_create_provider_match!(match)
+    end
+
+    def find_or_create_provider_match!(match)
+      security = Security.find_or_initialize_by(
+        ticker: match.ticker,
+        exchange_operating_mic: match.exchange_operating_mic,
+      )
+
+      security.country_code = match.country_code
+      security.save!
+
+      security
+    end
+
+    def provider_search_result
+      params = {
+        exchange_operating_mic: exchange_operating_mic,
+        country_code: country_code
+      }.compact_blank
+
+      @provider_search_result ||= Security.search_provider(symbol, **params)
+    end
+
+    # Non-exhaustive list of common country codes for help in choosing "close" matches
+    # These are generally sorted by market cap.
+    def sorted_country_codes_by_relevance
+      %w[US CN JP IN GB CA FR DE CH SA TW AU NL SE KR IE ES AE IT HK BR DK SG MX RU IL ID BE TH NO]
+    end
+
+    # Non-exhaustive list of common exchange operating MICs for help in choosing "close" matches
+    # This is very US-centric since our prices provider and user base is a majority US-based
+    def sorted_exchange_operating_mics_by_relevance
+      [
+        "XNYS",  # New York Stock Exchange
+        "XNAS",  # NASDAQ Stock Market
+        "XOTC",  # OTC Markets Group (OTC Link)
+        "OTCM",  # OTC Markets Group
+        "OTCN",  # OTC Bulletin Board
+        "OTCI",  # OTC International
+        "OPRA",  # Options Price Reporting Authority
+        "MEMX",  # Members Exchange
+        "IEXA",  # IEX All-Market
+        "IEXG",  # IEX Growth Market
+        "EDXM",  # Cboe EDGX Exchange (Equities)
+        "XCME",  # CME Group (Derivatives)
+        "XCBT",  # Chicago Board of Trade
+        "XPUS",  # Nasdaq PSX (U.S.)
+        "XPSE",  # Nasdaq PHLX (U.S.)
+        "XTRD",  # Nasdaq TRF (Trade Reporting Facility)
+        "XTXD",  # FINRA TRACE (Trade Reporting)
+        "XARC",  # NYSE Arca
+        "XBOX",  # BOX Options Exchange
+        "XBXO"  # BZX Options (Cboe)
+      ]
+    end
+end