Security resolver and health checker (#2281)

* Setup health check

* Security health checker cron

* Use resolver throughout codebase

* Use resolver for trade builder

* Add security health checks to schedule

* Handle no provider

* Lint fixes

app/models/security/health_checker.rb

@@ -0,0 +1,120 @@
+# There are hundreds of thousands of market securities that Maybe must handle.
+# Due to the always-changing nature of the market, the health checker is responsible
+# for periodically checking active securities to ensure we can still fetch prices for them.
+#
+# Each security goes through some basic health checks.  If failed, this class is responsible for:
+# - Marking failed attempts and incrementing the failed attempts counter
+# - Marking the security offline if enough consecutive failed checks occur
+# - When we move a security "offline", delete all prices for that security as we assume they are bad data
+#
+# The health checker is run daily through SecurityHealthCheckJob (see config/schedule.yml), but not all
+# securities will be checked every day (we run in batches)
+class Security::HealthChecker
+  MAX_CONSECUTIVE_FAILURES = 5
+  HEALTH_CHECK_INTERVAL = 7.days
+  DAILY_BATCH_SIZE = 1000
+
+  class << self
+    def check_all
+      # No daily limit for unchecked securities (they are prioritized)
+      never_checked_scope.find_each do |security|
+        new(security).run_check
+      end
+
+      # Daily limit for checked securities
+      due_for_check_scope.limit(DAILY_BATCH_SIZE).each do |security|
+        new(security).run_check
+      end
+    end
+
+    private
+      # If a security has never had a health check, we prioritize it, regardless of batch size
+      def never_checked_scope
+        Security.where(last_health_check_at: nil)
+      end
+
+      # Any securities not checked for 30 days are due
+      # We only process the batch size, which means some "due" securities will not be checked today
+      # This is by design, to prevent all securities from coming due at the same time
+      def due_for_check_scope
+        Security.where(last_health_check_at: ..HEALTH_CHECK_INTERVAL.ago)
+                .order(last_health_check_at: :asc)
+      end
+  end
+
+  def initialize(security)
+    @security = security
+  end
+
+  def run_check
+    Rails.logger.info("Running health check for #{security.ticker}")
+
+    if latest_provider_price
+      handle_success
+    else
+      handle_failure
+    end
+  rescue => e
+    Sentry.capture_exception(e) do |scope|
+      scope.set_tags(security_id: @security.id)
+    end
+  ensure
+    security.update!(last_health_check_at: Time.current)
+  end
+
+  private
+    attr_reader :security
+
+    def provider
+      Security.provider
+    end
+
+    def latest_provider_price
+      return nil unless provider.present?
+
+      response = provider.fetch_security_price(
+        symbol: security.ticker,
+        exchange_operating_mic: security.exchange_operating_mic,
+        date: Date.current
+      )
+
+      return nil unless response.success?
+
+      response.data.price
+    end
+
+    # On success, reset any failure counters and ensure it is "online"
+    def handle_success
+      security.update!(
+        offline: false,
+        failed_fetch_count: 0,
+        failed_fetch_at: nil
+      )
+    end
+
+    def handle_failure
+      new_failure_count = security.failed_fetch_count.to_i + 1
+      new_failure_at = Time.current
+
+      if new_failure_count > MAX_CONSECUTIVE_FAILURES
+        convert_to_offline_security!
+      else
+        security.update!(
+          failed_fetch_count: new_failure_count,
+          failed_fetch_at: new_failure_at
+        )
+      end
+    end
+
+    # The "offline" state tells our MarketDataImporter (daily cron) to skip this security when fetching prices
+    def convert_to_offline_security!
+      Security.transaction do
+        security.update!(
+          offline: true,
+          failed_fetch_count: MAX_CONSECUTIVE_FAILURES + 1,
+          failed_fetch_at: Time.current
+        )
+        security.prices.delete_all
+      end
+    end
+end