From 3767ecb562e22cee76b6170c1cb9f3bd8b17f79f Mon Sep 17 00:00:00 2001 From: Zach Gollwitzer Date: Wed, 21 May 2025 08:44:05 -0400 Subject: [PATCH] Setup health check --- app/jobs/security_health_check_job.rb | 7 +++ app/models/security/health_checker.rb | 58 +++++++++++++++++++ app/models/security/resolver.rb | 17 ++++++ ...0521112347_add_security_resolver_fields.rb | 8 +++ db/schema.rb | 8 ++- test/models/security/health_checker_test.rb | 42 ++++++++++++++ 6 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 app/jobs/security_health_check_job.rb create mode 100644 app/models/security/health_checker.rb create mode 100644 app/models/security/resolver.rb create mode 100644 db/migrate/20250521112347_add_security_resolver_fields.rb create mode 100644 test/models/security/health_checker_test.rb diff --git a/app/jobs/security_health_check_job.rb b/app/jobs/security_health_check_job.rb new file mode 100644 index 00000000..f413f388 --- /dev/null +++ b/app/jobs/security_health_check_job.rb @@ -0,0 +1,7 @@ +class SecurityHealthCheckJob < ApplicationJob + queue_as :default + + def perform + Security::HealthChecker.new.perform + end +end diff --git a/app/models/security/health_checker.rb b/app/models/security/health_checker.rb new file mode 100644 index 00000000..7fb090c6 --- /dev/null +++ b/app/models/security/health_checker.rb @@ -0,0 +1,58 @@ +# There are hundreds of thousands of market securities that Maybe must handle. +# Due to the always-changing nature of the market, the health checker is responsible +# for periodically checking active securities to ensure we can still fetch prices for them. +# +# Securities that cannot fetch prices are marked "offline" and will not run price updates. +# +# The health checker is run daily through SecurityHealthCheckJob (see config/schedule.yml) +class Security::HealthChecker + HEALTH_CHECK_INTERVAL = 30.days + DAILY_BATCH_SIZE = 1000 + + class << self + def check_all + # All securities that have never been checked run, regardless of daily batch size + never_checked_scope.find_each do |security| + new(security).run_check + end + + due_for_check_scope.find_each do |security| + new(security).run_check + end + end + + private + # If a security has never had a health check, we prioritize it, regardless of batch size + def never_checked_scope + Security.where(last_health_check_at: nil) + end + + # Any securities not checked for 30 days are due + # We only process the batch size, which means some "due" securities will not be checked today + # This is by design, to prevent all securities from coming due at the same time + def due_for_check_scope + Security.where(last_health_check_at: ..HEALTH_CHECK_INTERVAL.ago) + .order(last_health_check_at: :asc) + .limit(DAILY_BATCH_SIZE) + end + end + + def initialize(security) + @security = security + end + + def run_check + end + + private + def scope + Security.where(last_health_check_at: nil) + .or(Security.where(last_health_check_at: ..7.days.ago)) + end + + def can_fetch_from_provider? + end + + def has_daily_prices? + end +end diff --git a/app/models/security/resolver.rb b/app/models/security/resolver.rb new file mode 100644 index 00000000..31c3f40b --- /dev/null +++ b/app/models/security/resolver.rb @@ -0,0 +1,17 @@ +class Security::Resolver + def initialize(symbol, exchange_operating_mic: nil, country_code: nil) + @symbol = symbol + @exchange_operating_mic = exchange_operating_mic + @country_code = country_code + end + + def resolve + end + + private + attr_reader :symbol, :exchange_operating_mic, :country_code + + def provider + Security.provider + end +end diff --git a/db/migrate/20250521112347_add_security_resolver_fields.rb b/db/migrate/20250521112347_add_security_resolver_fields.rb new file mode 100644 index 00000000..29391512 --- /dev/null +++ b/db/migrate/20250521112347_add_security_resolver_fields.rb @@ -0,0 +1,8 @@ +class AddSecurityResolverFields < ActiveRecord::Migration[7.2] + def change + add_column :securities, :offline, :boolean, default: false, null: false + add_column :securities, :failed_fetch_at, :datetime + add_column :securities, :failed_fetch_count, :integer, default: 0, null: false + add_column :securities, :last_health_check_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index fe602824..9f079956 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_05_18_181619) do +ActiveRecord::Schema[7.2].define(version: 2025_05_21_112347) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -30,7 +30,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_18_181619) do t.decimal "balance", precision: 19, scale: 4 t.string "currency" t.boolean "is_active", default: true, null: false - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.boolean "scheduled_for_deletion", default: false @@ -513,6 +513,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_05_18_181619) do t.string "exchange_acronym" t.string "logo_url" t.string "exchange_operating_mic" + t.boolean "offline", default: false, null: false + t.datetime "failed_fetch_at" + t.integer "failed_fetch_count", default: 0, null: false + t.datetime "last_health_check_at" t.index ["country_code"], name: "index_securities_on_country_code" t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic" t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: true diff --git a/test/models/security/health_checker_test.rb b/test/models/security/health_checker_test.rb new file mode 100644 index 00000000..61acab15 --- /dev/null +++ b/test/models/security/health_checker_test.rb @@ -0,0 +1,42 @@ +require "test_helper" + +class Security::HealthCheckerTest < ActiveSupport::TestCase + test "checks all securities that don't have a health check" do + # Setup + unchecked_security = Security.create!(ticker: "AAA") + recent_security = Security.create!(ticker: "BBB", last_health_check_at: 5.days.ago) + due_security = Security.create!(ticker: "CCC", last_health_check_at: Security::HealthChecker::HEALTH_CHECK_INTERVAL.ago - 1.day) + + scope = Security::HealthChecker.send(:never_checked_scope) + + assert_includes scope, unchecked_security + refute_includes scope, recent_security + refute_includes scope, due_security + end + + # We don't intend to check all securities every day + test "checks oldest DAILY_BATCH_SIZE securities that haven't been checked in HEALTH_CHECK_INTERVAL days" do + batch_size = Security::HealthChecker::DAILY_BATCH_SIZE + + # Create batch_size + 2 securities that are all past the health check interval so that + # the scope needs to apply the LIMIT and ordering. + (batch_size + 2).times do |i| + # Spread the dates so we can assert ordering (older first) + days_past_interval = Security::HealthChecker::HEALTH_CHECK_INTERVAL + i.days + 1.day + Security.create!(ticker: "SEC#{i}", last_health_check_at: days_past_interval.ago) + end + + scoped = Security::HealthChecker.send(:due_for_check_scope).to_a + + # 1. Only DAILY_BATCH_SIZE records are returned + assert_equal batch_size, scoped.size + + # 2. Records are ordered oldest -> newest by last_health_check_at + ordered_dates = scoped.map(&:last_health_check_at) + assert_equal ordered_dates.sort, ordered_dates, "due_for_check_scope should return oldest records first" + + # 3. The newest (least old) security should have been excluded due to the LIMIT + newest_excluded_date = Security.order(last_health_check_at: :desc).where(last_health_check_at: ..Security::HealthChecker::HEALTH_CHECK_INTERVAL.ago).first.last_health_check_at + refute_includes scoped.map(&:last_health_check_at), newest_excluded_date + end +end