Use Redis for ActiveJob and ActionCable (#2004)

* Use Redis for ActiveJob and ActionCable

* Fix alwaysApply setting

* Update queue names and weights

* Tweak weights

* Update job queues

* Update docker setup guide

* Remove deprecated upgrade columns from users table

* Refactor Redis configuration for Sidekiq and caching in production environment

* Add Sidekiq Sentry monitoring

* queue naming fix

* Clean up schema

app/jobs/application_job.rb

@@ -1,7 +1,5 @@
 class ApplicationJob < ActiveJob::Base
-  # Automatically retry jobs that encountered a deadlock
-  # retry_on ActiveRecord::Deadlocked
-
-  # Most jobs are safe to ignore if the underlying records are no longer available
-  # discard_on ActiveJob::DeserializationError
+  retry_on ActiveRecord::Deadlocked
+  discard_on ActiveJob::DeserializationError
+  queue_as :low_priority # default queue
 end

app/jobs/auto_upgrade_job.rb

@@ -1,31 +0,0 @@
-class AutoUpgradeJob < ApplicationJob
-  queue_as :latency_low
-
-  def perform(*args)
-    raise_if_disabled
-
-    return Rails.logger.info "Skipping auto-upgrades because app is set to manual upgrades.  Please set UPGRADES_MODE=auto to enable auto-upgrades" if Setting.upgrades_mode == "manual"
-
-    Rails.logger.info "Searching for available auto-upgrades..."
-
-    candidate = Upgrader.available_upgrade_by_type(Setting.upgrades_target)
-
-    if candidate
-      if Rails.cache.read("last_auto_upgrade_commit_sha") == candidate.commit_sha
-        Rails.logger.info "Skipping auto upgrade: #{candidate.type} #{candidate.commit_sha} deploy in progress"
-        return
-      end
-
-      Rails.logger.info "Auto upgrading to #{candidate.type} #{candidate.commit_sha}..."
-      Upgrader.upgrade_to(candidate)
-      Rails.cache.write("last_auto_upgrade_commit_sha", candidate.commit_sha, expires_in: 1.day)
-    else
-      Rails.logger.info "No auto upgrade available at this time"
-    end
-  end
-
-  private
-    def raise_if_disabled
-      raise "Upgrades module is disabled.  Please set UPGRADES_ENABLED=true to enable upgrade features" unless ENV["UPGRADES_ENABLED"] == "true"
-    end
-end

app/jobs/data_cache_clear_job.rb

@@ -1,5 +1,5 @@
 class DataCacheClearJob < ApplicationJob
-  queue_as :default
+  queue_as :low_priority
 
   def perform(family)
     ActiveRecord::Base.transaction do

app/jobs/destroy_job.rb

@@ -1,5 +1,5 @@
 class DestroyJob < ApplicationJob
-  queue_as :latency_low
+  queue_as :low_priority
 
   def perform(model)
     model.destroy

app/jobs/enrich_transaction_batch_job.rb

@@ -1,5 +1,5 @@
 class EnrichTransactionBatchJob < ApplicationJob
-  queue_as :latency_high
+  queue_as :low_priority
 
   def perform(account, batch_size = 100, offset = 0)
     account.enrich_transaction_batch(batch_size, offset)

app/jobs/family_reset_job.rb

@@ -1,5 +1,5 @@
 class FamilyResetJob < ApplicationJob
-  queue_as :default
+  queue_as :low_priority
 
   def perform(family)
     # Delete all family data except users

app/jobs/fetch_security_info_job.rb

@@ -1,5 +1,5 @@
 class FetchSecurityInfoJob < ApplicationJob
-  queue_as :latency_low
+  queue_as :low_priority
 
   def perform(security_id)
     return unless Security.provider.present?

app/jobs/import_job.rb

@@ -1,5 +1,5 @@
 class ImportJob < ApplicationJob
-  queue_as :latency_medium
+  queue_as :high_priority
 
   def perform(import)
     import.publish

app/jobs/revert_import_job.rb

@@ -1,5 +1,5 @@
 class RevertImportJob < ApplicationJob
-  queue_as :latency_low
+  queue_as :medium_priority
 
   def perform(import)
     import.revert

app/jobs/sync_job.rb

@@ -1,7 +1,8 @@
 class SyncJob < ApplicationJob
-  queue_as :latency_medium
+  queue_as :high_priority
 
   def perform(sync)
+    sleep 1 # simulate work for faster jobs
     sync.perform
   end
 end

app/jobs/user_purge_job.rb

@@ -1,5 +1,5 @@
 class UserPurgeJob < ApplicationJob
-  queue_as :latency_low
+  queue_as :low_priority
 
   def perform(user)
     user.purge