Security resolver and health checker (#2281)

* Setup health check

* Security health checker cron

* Use resolver throughout codebase

* Use resolver for trade builder

* Add security health checks to schedule

* Handle no provider

* Lint fixes

app/jobs/security_health_check_job.rb

@@ -0,0 +1,7 @@
+class SecurityHealthCheckJob < ApplicationJob
+  queue_as :scheduled
+
+  def perform
+    Security::HealthChecker.check_all
+  end
+end

app/models/market_data_importer.rb

@@ -22,7 +22,8 @@ class MarketDataImporter
       return
     end
 
-    Security.where.not(exchange_operating_mic: nil).find_each do |security|
+    # Import all securities that aren't marked as "offline" (i.e. they're available from the provider)
+    Security.online.find_each do |security|
       security.import_provider_prices(
         start_date: get_first_required_price_date(security),
         end_date: end_date,

app/models/provider/synth.rb

@@ -94,7 +94,8 @@ class Provider::Synth < Provider
         req.params["name"] = symbol
         req.params["dataset"] = "limited"
         req.params["country_code"] = country_code if country_code.present?
-        req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present?
+        # Synth uses mic_code, which encompasses both exchange_mic AND exchange_operating_mic (union)
+        req.params["mic_code"] = exchange_operating_mic if exchange_operating_mic.present?
         req.params["limit"] = 25
       end
 
@@ -132,7 +133,7 @@ class Provider::Synth < Provider
     end
   end
 
-  def fetch_security_price(symbol:, exchange_operating_mic:, date:)
+  def fetch_security_price(symbol:, exchange_operating_mic: nil, date:)
     with_provider_response do
       historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date)
 
@@ -142,13 +143,13 @@ class Provider::Synth < Provider
     end
   end
 
-  def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:)
+  def fetch_security_prices(symbol:, exchange_operating_mic: nil, start_date:, end_date:)
     with_provider_response do
       params = {
         start_date: start_date,
         end_date: end_date,
         operating_mic_code: exchange_operating_mic
-      }
+      }.compact
 
       data = paginate(
         "#{base_url}/tickers/#{symbol}/open-close",

app/models/security.rb

@@ -9,6 +9,8 @@ class Security < ApplicationRecord
   validates :ticker, presence: true
   validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }
 
+  scope :online, -> { where(offline: false) }
+
   def current_price
     @current_price ||= find_or_fetch_price
     return nil if @current_price.nil?
@@ -25,10 +27,6 @@ class Security < ApplicationRecord
     )
   end
 
-  def has_prices?
-    exchange_operating_mic.present?
-  end
-
   private
     def upcase_symbols
       self.ticker = ticker.upcase

app/models/security/health_checker.rb

@@ -0,0 +1,120 @@
+# There are hundreds of thousands of market securities that Maybe must handle.
+# Due to the always-changing nature of the market, the health checker is responsible
+# for periodically checking active securities to ensure we can still fetch prices for them.
+#
+# Each security goes through some basic health checks.  If failed, this class is responsible for:
+# - Marking failed attempts and incrementing the failed attempts counter
+# - Marking the security offline if enough consecutive failed checks occur
+# - When we move a security "offline", delete all prices for that security as we assume they are bad data
+#
+# The health checker is run daily through SecurityHealthCheckJob (see config/schedule.yml), but not all
+# securities will be checked every day (we run in batches)
+class Security::HealthChecker
+  MAX_CONSECUTIVE_FAILURES = 5
+  HEALTH_CHECK_INTERVAL = 7.days
+  DAILY_BATCH_SIZE = 1000
+
+  class << self
+    def check_all
+      # No daily limit for unchecked securities (they are prioritized)
+      never_checked_scope.find_each do |security|
+        new(security).run_check
+      end
+
+      # Daily limit for checked securities
+      due_for_check_scope.limit(DAILY_BATCH_SIZE).each do |security|
+        new(security).run_check
+      end
+    end
+
+    private
+      # If a security has never had a health check, we prioritize it, regardless of batch size
+      def never_checked_scope
+        Security.where(last_health_check_at: nil)
+      end
+
+      # Any securities not checked for 30 days are due
+      # We only process the batch size, which means some "due" securities will not be checked today
+      # This is by design, to prevent all securities from coming due at the same time
+      def due_for_check_scope
+        Security.where(last_health_check_at: ..HEALTH_CHECK_INTERVAL.ago)
+                .order(last_health_check_at: :asc)
+      end
+  end
+
+  def initialize(security)
+    @security = security
+  end
+
+  def run_check
+    Rails.logger.info("Running health check for #{security.ticker}")
+
+    if latest_provider_price
+      handle_success
+    else
+      handle_failure
+    end
+  rescue => e
+    Sentry.capture_exception(e) do |scope|
+      scope.set_tags(security_id: @security.id)
+    end
+  ensure
+    security.update!(last_health_check_at: Time.current)
+  end
+
+  private
+    attr_reader :security
+
+    def provider
+      Security.provider
+    end
+
+    def latest_provider_price
+      return nil unless provider.present?
+
+      response = provider.fetch_security_price(
+        symbol: security.ticker,
+        exchange_operating_mic: security.exchange_operating_mic,
+        date: Date.current
+      )
+
+      return nil unless response.success?
+
+      response.data.price
+    end
+
+    # On success, reset any failure counters and ensure it is "online"
+    def handle_success
+      security.update!(
+        offline: false,
+        failed_fetch_count: 0,
+        failed_fetch_at: nil
+      )
+    end
+
+    def handle_failure
+      new_failure_count = security.failed_fetch_count.to_i + 1
+      new_failure_at = Time.current
+
+      if new_failure_count > MAX_CONSECUTIVE_FAILURES
+        convert_to_offline_security!
+      else
+        security.update!(
+          failed_fetch_count: new_failure_count,
+          failed_fetch_at: new_failure_at
+        )
+      end
+    end
+
+    # The "offline" state tells our MarketDataImporter (daily cron) to skip this security when fetching prices
+    def convert_to_offline_security!
+      Security.transaction do
+        security.update!(
+          offline: true,
+          failed_fetch_count: MAX_CONSECUTIVE_FAILURES + 1,
+          failed_fetch_at: Time.current
+        )
+        security.prices.delete_all
+      end
+    end
+end

app/models/security/provided.rb

@@ -10,9 +10,14 @@ module Security::Provided
     end
 
     def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)
-      return [] if symbol.blank? || symbol.length < 2
+      return [] if provider.nil? || symbol.blank?
 
-      response = provider.search_securities(symbol, country_code: country_code, exchange_operating_mic: exchange_operating_mic)
+      params = {
+        country_code: country_code,
+        exchange_operating_mic: exchange_operating_mic
+      }.compact_blank
+
+      response = provider.search_securities(symbol, **params)
 
       if response.success?
         response.data.map do |provider_security|
@@ -38,7 +43,11 @@ module Security::Provided
 
     # Make sure we have a data provider before fetching
     return nil unless provider.present?
-    response = provider.fetch_security_price(self, date: date)
+    response = provider.fetch_security_price(
+      symbol: ticker,
+      exchange_operating_mic: exchange_operating_mic,
+      date: date
+    )
 
     return nil unless response.success? # Provider error
 

app/models/security/resolver.rb

@@ -0,0 +1,156 @@
+class Security::Resolver
+  def initialize(symbol, exchange_operating_mic: nil, country_code: nil)
+    @symbol = validate_symbol!(symbol)
+    @exchange_operating_mic = exchange_operating_mic
+    @country_code = country_code
+  end
+
+  # Attempts several paths to resolve a security:
+  # 1. Exact match in DB
+  # 2. Search provider for an exact match
+  # 3. Search provider for close match, ranked by relevance
+  # 4. Create offline security if no match is found in either DB or provider
+  def resolve
+    return nil if symbol.blank?
+
+    exact_match_from_db ||
+      exact_match_from_provider ||
+      close_match_from_provider ||
+      offline_security
+  end
+
+  private
+    attr_reader :symbol, :exchange_operating_mic, :country_code
+
+    def validate_symbol!(symbol)
+      raise ArgumentError, "Symbol is required and cannot be blank" if symbol.blank?
+      symbol.strip.upcase
+    end
+
+    def offline_security
+      security = Security.find_or_initialize_by(
+        ticker: symbol,
+        exchange_operating_mic: exchange_operating_mic,
+      )
+
+      security.assign_attributes(
+        country_code: country_code,
+        offline: true # This tells us that we shouldn't try to fetch prices later
+      )
+
+      security.save!
+
+      security
+    end
+
+    def exact_match_from_db
+      Security.find_by(
+        {
+          ticker: symbol,
+          exchange_operating_mic: exchange_operating_mic,
+          country_code: country_code.presence
+        }.compact
+      )
+    end
+
+    # If provided a ticker + exchange (and optionally, a country code), we can find exact matches
+    def exact_match_from_provider
+      # Without an exchange, we can never know if we have an exact match
+      return nil unless exchange_operating_mic.present?
+
+      match = provider_search_result.find do |s|
+        ticker_matches = s.ticker.upcase.to_s == symbol.upcase.to_s
+        exchange_matches = s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s
+
+        if country_code && exchange_operating_mic
+          ticker_matches && exchange_matches && s.country_code.upcase.to_s == country_code.upcase.to_s
+        else
+          ticker_matches && exchange_matches
+        end
+      end
+
+      return nil unless match
+
+      find_or_create_provider_match!(match)
+    end
+
+    def close_match_from_provider
+      filtered_candidates = provider_search_result
+
+      # If a country code is specified, we MUST find a match with the same code
+      if country_code.present?
+        filtered_candidates = filtered_candidates.select { |s| s.country_code.upcase.to_s == country_code.upcase.to_s }
+      end
+
+      # 1. Prefer exact exchange_operating_mic matches (if one was provided)
+      # 2. Rank by country relevance (lower index in the list is more relevant)
+      # 3. Rank by exchange_operating_mic relevance (lower index in the list is more relevant)
+      sorted_candidates = filtered_candidates.sort_by do |s|
+        [
+          exchange_operating_mic.present? && s.exchange_operating_mic.upcase.to_s == exchange_operating_mic.upcase.to_s ? 0 : 1,
+          sorted_country_codes_by_relevance.index(s.country_code&.upcase.to_s) || sorted_country_codes_by_relevance.length,
+          sorted_exchange_operating_mics_by_relevance.index(s.exchange_operating_mic&.upcase.to_s) || sorted_exchange_operating_mics_by_relevance.length
+        ]
+      end
+
+      match = sorted_candidates.first
+
+      return nil unless match
+
+      find_or_create_provider_match!(match)
+    end
+
+    def find_or_create_provider_match!(match)
+      security = Security.find_or_initialize_by(
+        ticker: match.ticker,
+        exchange_operating_mic: match.exchange_operating_mic,
+      )
+
+      security.country_code = match.country_code
+      security.save!
+
+      security
+    end
+
+    def provider_search_result
+      params = {
+        exchange_operating_mic: exchange_operating_mic,
+        country_code: country_code
+      }.compact_blank
+
+      @provider_search_result ||= Security.search_provider(symbol, **params)
+    end
+
+    # Non-exhaustive list of common country codes for help in choosing "close" matches
+    # These are generally sorted by market cap.
+    def sorted_country_codes_by_relevance
+      %w[US CN JP IN GB CA FR DE CH SA TW AU NL SE KR IE ES AE IT HK BR DK SG MX RU IL ID BE TH NO]
+    end
+
+    # Non-exhaustive list of common exchange operating MICs for help in choosing "close" matches
+    # This is very US-centric since our prices provider and user base is a majority US-based
+    def sorted_exchange_operating_mics_by_relevance
+      [
+        "XNYS",  # New York Stock Exchange
+        "XNAS",  # NASDAQ Stock Market
+        "XOTC",  # OTC Markets Group (OTC Link)
+        "OTCM",  # OTC Markets Group
+        "OTCN",  # OTC Bulletin Board
+        "OTCI",  # OTC International
+        "OPRA",  # Options Price Reporting Authority
+        "MEMX",  # Members Exchange
+        "IEXA",  # IEX All-Market
+        "IEXG",  # IEX Growth Market
+        "EDXM",  # Cboe EDGX Exchange (Equities)
+        "XCME",  # CME Group (Derivatives)
+        "XCBT",  # Chicago Board of Trade
+        "XPUS",  # Nasdaq PSX (U.S.)
+        "XPSE",  # Nasdaq PHLX (U.S.)
+        "XTRD",  # Nasdaq TRF (Trade Reporting Facility)
+        "XTXD",  # FINRA TRACE (Trade Reporting)
+        "XARC",  # NYSE Arca
+        "XBOX",  # BOX Options Exchange
+        "XBXO"  # BZX Options (Cboe)
+      ]
+    end
+end

app/models/trade_builder.rb

@@ -129,9 +129,9 @@ class TradeBuilder
     def security
       ticker_symbol, exchange_operating_mic = ticker.present? ? ticker.split("|") : [ manual_ticker, nil ]
 
-      Security.find_or_create_by!(
-        ticker: ticker_symbol,
+      Security::Resolver.new(
+        ticker_symbol,
         exchange_operating_mic: exchange_operating_mic
-      )
+      ).resolve
     end
 end

app/models/trade_import.rb

@@ -76,42 +76,25 @@ class TradeImport < Import
   end
 
   private
-    def find_or_create_security(ticker:, exchange_operating_mic:)
-      # Normalize empty string to nil for consistency
-      exchange_operating_mic = nil if exchange_operating_mic.blank?
+    def find_or_create_security(ticker: nil, exchange_operating_mic: nil)
+      return nil unless ticker.present?
 
-      # First try to find an exact match in our DB, or if no exchange_operating_mic is provided, find by ticker only
-      internal_security = if exchange_operating_mic.present?
-        Security.find_by(ticker:, exchange_operating_mic:)
-      else
-        Security.find_by(ticker:)
-      end
+      # Avoids resolving the same security over and over again (resolver potentially makes network calls)
+      @security_cache ||= {}
 
-      return internal_security if internal_security.present?
+      cache_key = [ ticker, exchange_operating_mic ].compact.join(":")
 
-      # If security prices provider isn't properly configured or available, create with nil exchange_operating_mic
-      return Security.find_or_create_by!(ticker: ticker&.upcase, exchange_operating_mic: nil) unless Security.provider.present?
+      security = @security_cache[cache_key]
 
-      # Cache provider responses so that when we're looping through rows and importing,
-      # we only hit our provider for the unique combinations of ticker / exchange_operating_mic
-      cache_key = [ ticker, exchange_operating_mic ]
-      @provider_securities_cache ||= {}
+      return security if security.present?
 
-      provider_security = @provider_securities_cache[cache_key] ||= begin
-        Security.search_provider(
-          ticker,
-          exchange_operating_mic: exchange_operating_mic
-        ).first
-      end
+      security = Security::Resolver.new(
+        ticker,
+        exchange_operating_mic: exchange_operating_mic.presence
+      ).resolve
 
-      return Security.find_or_create_by!(ticker: ticker&.upcase, exchange_operating_mic: nil) if provider_security.nil?
+      @security_cache[cache_key] = security
 
-      Security.find_or_create_by!(ticker: provider_security[:ticker]&.upcase, exchange_operating_mic: provider_security[:exchange_operating_mic]&.upcase) do |security|
-        security.name = provider_security[:name]
-        security.country_code = provider_security[:country_code]
-        security.logo_url = provider_security[:logo_url]
-        security.exchange_acronym = provider_security[:exchange_acronym]
-        security.exchange_mic = provider_security[:exchange_mic]
-      end
+      security
     end
 end

app/views/pages/dashboard/_cashflow_sankey.html.erb

@@ -41,4 +41,4 @@
       </div>
     </div>
   <% end %>
-</div> 
+</div>

app/views/rules/_rule.html.erb

@@ -1,5 +1,5 @@
 <%# locals: (rule:) %>
-<div id="<%= dom_id(rule) %>" class="flex justify-between items-center p-4 <%= rule.active? ? 'text-primary' : 'text-secondary' %>">
+<div id="<%= dom_id(rule) %>" class="flex justify-between items-center p-4 <%= rule.active? ? "text-primary" : "text-secondary" %>">
 
   <div class="text-sm space-y-1.5">
     <% if rule.name.present? %>

app/views/shared/_ruler.html.erb

@@ -1,2 +1,2 @@
 <%# locals: (classes: nil) %>
-<hr class="border-divider <%= classes || 'mx-4' %>">
+<hr class="border-divider <%= classes || "mx-4" %>">