From a7667ba29aa71fa4c491182a79d08f6011dae15c Mon Sep 17 00:00:00 2001 From: Claude Ayitey Date: Sat, 6 Jan 2024 14:47:35 +0000 Subject: [PATCH] Rename app/sidekiq to app/jobs --- app/{sidekiq => jobs}/ask_question_job.rb | 7 +++---- app/{sidekiq => jobs}/enrich_transactions_job.rb | 1 - app/{sidekiq => jobs}/generate_balance_job.rb | 3 +-- .../generate_categorical_metrics_job.rb | 1 - app/{sidekiq => jobs}/generate_metrics_job.rb | 3 +-- app/{sidekiq => jobs}/real_time_sync_job.rb | 1 - app/{sidekiq => jobs}/sync_plaid_holdings_job.rb | 7 +++---- .../sync_plaid_institutions_job.rb | 7 +++---- .../sync_plaid_investment_transactions_job.rb | 13 ++++++------- .../sync_plaid_item_accounts_job.rb | 7 +++---- .../sync_plaid_transactions_job.rb | 7 +++---- app/{sidekiq => jobs}/sync_property_values_job.rb | 5 ++--- app/{sidekiq => jobs}/sync_security_details_job.rb | 3 +-- app/{sidekiq => jobs}/sync_security_history_job.rb | 3 +-- 14 files changed, 27 insertions(+), 41 deletions(-) rename app/{sidekiq => jobs}/ask_question_job.rb (99%) rename app/{sidekiq => jobs}/enrich_transactions_job.rb (99%) rename app/{sidekiq => jobs}/generate_balance_job.rb (98%) rename app/{sidekiq => jobs}/generate_categorical_metrics_job.rb (98%) rename app/{sidekiq => jobs}/generate_metrics_job.rb (98%) rename app/{sidekiq => jobs}/real_time_sync_job.rb (95%) rename app/{sidekiq => jobs}/sync_plaid_holdings_job.rb (96%) rename app/{sidekiq => jobs}/sync_plaid_institutions_job.rb (93%) rename app/{sidekiq => jobs}/sync_plaid_investment_transactions_job.rb (97%) rename app/{sidekiq => jobs}/sync_plaid_item_accounts_job.rb (89%) rename app/{sidekiq => jobs}/sync_plaid_transactions_job.rb (96%) rename app/{sidekiq => jobs}/sync_property_values_job.rb (97%) rename app/{sidekiq => jobs}/sync_security_details_job.rb (99%) rename app/{sidekiq => jobs}/sync_security_history_job.rb (94%) diff --git a/app/sidekiq/ask_question_job.rb b/app/jobs/ask_question_job.rb similarity index 99% rename from app/sidekiq/ask_question_job.rb rename to app/jobs/ask_question_job.rb index ecce1e42..e64d42ea 100644 --- a/app/sidekiq/ask_question_job.rb +++ b/app/jobs/ask_question_job.rb @@ -1,5 +1,4 @@ class AskQuestionJob - include Sidekiq::Job def perform(conversation_id, reply_id) conversation = Conversation.find(conversation_id) @@ -163,7 +162,7 @@ class AskQuestionJob 5. investing: the user has a specific question about investing and needs real-time data 6. accounts: the user has a specific question about their accounts 7. system: the user wants to know how to do something within the product - + Second, remember to keep these things in mind regarding how to resolve: - We have access to both historical and real-time data we can query, so we can answer questions about the user's accounts. But if we need to get that data, then content_contains_answer should be false. - If the user is asking for metrics, then resolution should be to query the metrics table. @@ -188,7 +187,7 @@ class AskQuestionJob ) raw_response = response.dig("choices", 0, "message", "content") - + justification_reply = conversation.messages.new justification_reply.log = raw_response justification_reply.user = nil @@ -226,7 +225,7 @@ class AskQuestionJob resolve_value = last_log_json["resolve"] text_string = '' - + response = openai.chat( parameters: { model: "gpt-4-1106-preview", diff --git a/app/sidekiq/enrich_transactions_job.rb b/app/jobs/enrich_transactions_job.rb similarity index 99% rename from app/sidekiq/enrich_transactions_job.rb rename to app/jobs/enrich_transactions_job.rb index ead67a66..debf630c 100644 --- a/app/sidekiq/enrich_transactions_job.rb +++ b/app/jobs/enrich_transactions_job.rb @@ -1,5 +1,4 @@ class EnrichTransactionsJob - include Sidekiq::Job def perform enrichment = Faraday.new( diff --git a/app/sidekiq/generate_balance_job.rb b/app/jobs/generate_balance_job.rb similarity index 98% rename from app/sidekiq/generate_balance_job.rb rename to app/jobs/generate_balance_job.rb index 3be0b8ec..1bd3df57 100644 --- a/app/sidekiq/generate_balance_job.rb +++ b/app/jobs/generate_balance_job.rb @@ -1,5 +1,4 @@ class GenerateBalanceJob - include Sidekiq::Job def perform(account_id) account = Account.find(account_id) @@ -11,7 +10,7 @@ class GenerateBalanceJob # Get current balance and save it to Balance model. Update based on account and date. Don't add last_balance if it's nil. Balance.find_or_initialize_by(account_id: account_id, security_id: nil, date: Date.today, kind: 'account', family_id: account.family.id).update(balance: account.current_balance, change: last_balance.nil? ? 0 : account.current_balance - last_balance) - + # Check if there holdings if account.holdings.any? # Get current holdings value and save it to Balance model. Update based on account, security and date. diff --git a/app/sidekiq/generate_categorical_metrics_job.rb b/app/jobs/generate_categorical_metrics_job.rb similarity index 98% rename from app/sidekiq/generate_categorical_metrics_job.rb rename to app/jobs/generate_categorical_metrics_job.rb index 2f39f10c..620c615b 100644 --- a/app/sidekiq/generate_categorical_metrics_job.rb +++ b/app/jobs/generate_categorical_metrics_job.rb @@ -1,5 +1,4 @@ class GenerateCategoricalMetricsJob - include Sidekiq::Job def perform(family_id) family = Family.find(family_id) diff --git a/app/sidekiq/generate_metrics_job.rb b/app/jobs/generate_metrics_job.rb similarity index 98% rename from app/sidekiq/generate_metrics_job.rb rename to app/jobs/generate_metrics_job.rb index 4ca4d5a3..d0a999bd 100644 --- a/app/sidekiq/generate_metrics_job.rb +++ b/app/jobs/generate_metrics_job.rb @@ -1,5 +1,4 @@ class GenerateMetricsJob - include Sidekiq::Job def perform(family_id) family = Family.find(family_id) @@ -15,7 +14,7 @@ class GenerateMetricsJob property_accounts_balance = accounts.property.sum { |account| account.current_balance } total_assets = depository_accounts_balance + investment_accounts_balance + property_accounts_balance - + total_debts = credit_accounts_balance net_worth = total_assets - total_debts diff --git a/app/sidekiq/real_time_sync_job.rb b/app/jobs/real_time_sync_job.rb similarity index 95% rename from app/sidekiq/real_time_sync_job.rb rename to app/jobs/real_time_sync_job.rb index 9dc20875..ba0e0635 100644 --- a/app/sidekiq/real_time_sync_job.rb +++ b/app/jobs/real_time_sync_job.rb @@ -1,5 +1,4 @@ class RealTimeSyncJob - include Sidekiq::Job def perform(security_id) security = Security.find(security_id) diff --git a/app/sidekiq/sync_plaid_holdings_job.rb b/app/jobs/sync_plaid_holdings_job.rb similarity index 96% rename from app/sidekiq/sync_plaid_holdings_job.rb rename to app/jobs/sync_plaid_holdings_job.rb index de3740e7..314cc5b7 100644 --- a/app/sidekiq/sync_plaid_holdings_job.rb +++ b/app/jobs/sync_plaid_holdings_job.rb @@ -1,5 +1,4 @@ class SyncPlaidHoldingsJob - include Sidekiq::Job def perform(item_id) connection = Connection.find_by(source: 'plaid', item_id: item_id) @@ -31,7 +30,7 @@ class SyncPlaidHoldingsJob # Process all securities first securities = holdings_response.securities - + # upsert_all securities all_securities = [] @@ -59,7 +58,7 @@ class SyncPlaidHoldingsJob holdings.each do |holding| next if account_ids[holding.account_id].nil? next if holding.quantity <= 0 - + all_holdings << { account_id: account_ids[holding.account_id], security_id: Security.find_by(source_id: holding.security_id).id, @@ -74,6 +73,6 @@ class SyncPlaidHoldingsJob Holding.upsert_all(all_holdings, unique_by: :index_holdings_on_account_id_and_security_id) - SyncPlaidInvestmentTransactionsJob.perform_async(item_id) + SyncPlaidInvestmentTransactionsJob.perform(item_id) end end diff --git a/app/sidekiq/sync_plaid_institutions_job.rb b/app/jobs/sync_plaid_institutions_job.rb similarity index 93% rename from app/sidekiq/sync_plaid_institutions_job.rb rename to app/jobs/sync_plaid_institutions_job.rb index ea5b0acf..2537cee0 100644 --- a/app/sidekiq/sync_plaid_institutions_job.rb +++ b/app/jobs/sync_plaid_institutions_job.rb @@ -1,12 +1,11 @@ -class SyncPlaidInstitutionsJob - include Sidekiq::Job +class SyncPlaidInstitutionsJob < ApplicationJob - def perform + def perform_now # Get all institutions from Plaid, which includes paginating through all pages offset = 0 while true institutions = [] - + institutions_get_request = Plaid::InstitutionsGetRequest.new({ offset: offset, count: 500, diff --git a/app/sidekiq/sync_plaid_investment_transactions_job.rb b/app/jobs/sync_plaid_investment_transactions_job.rb similarity index 97% rename from app/sidekiq/sync_plaid_investment_transactions_job.rb rename to app/jobs/sync_plaid_investment_transactions_job.rb index 4819ca73..ae3e80ca 100644 --- a/app/sidekiq/sync_plaid_investment_transactions_job.rb +++ b/app/jobs/sync_plaid_investment_transactions_job.rb @@ -1,5 +1,4 @@ class SyncPlaidInvestmentTransactionsJob - include Sidekiq::Job def perform(item_id) connection = Connection.find_by(source: 'plaid', item_id: item_id) @@ -37,11 +36,11 @@ class SyncPlaidInvestmentTransactionsJob return end end - + # Process all securities first securities = response.securities - + # upsert_all securities all_securities = [] @@ -70,7 +69,7 @@ class SyncPlaidInvestmentTransactionsJob access_token: access_token, start_date: start_date, end_date: Date.today, - options: { + options: { count: 500, offset: investmentTransactions.length() } @@ -83,7 +82,7 @@ class SyncPlaidInvestmentTransactionsJob if investmentTransactions.any? investmentTransactions_hash = investmentTransactions.map do |transaction| security = Security.find_by(source_id: transaction.security_id) - + next if security.blank? { account_id: account_ids[transaction.account_id], @@ -118,9 +117,9 @@ class SyncPlaidInvestmentTransactionsJob # Update investments_last_synced_at to the current time connection.update(investments_last_synced_at: DateTime.now) end - + accounts.each do |account| - GenerateBalanceJob.perform_async(account.id) + GenerateBalanceJob.perform(account.id) end end end diff --git a/app/sidekiq/sync_plaid_item_accounts_job.rb b/app/jobs/sync_plaid_item_accounts_job.rb similarity index 89% rename from app/sidekiq/sync_plaid_item_accounts_job.rb rename to app/jobs/sync_plaid_item_accounts_job.rb index 7eca98e4..d10787f2 100644 --- a/app/sidekiq/sync_plaid_item_accounts_job.rb +++ b/app/jobs/sync_plaid_item_accounts_job.rb @@ -1,5 +1,4 @@ class SyncPlaidItemAccountsJob - include Sidekiq::Job def perform(item_id) connection = Connection.find_by(source: 'plaid', item_id: item_id) @@ -38,12 +37,12 @@ class SyncPlaidItemAccountsJob ) connection_account.save - #GenerateBalanceJob.perform_async(connection_account.id) + #GenerateBalanceJob.perform(connection_account.id) end connection.update(sync_status: 'idle') - SyncPlaidTransactionsJob.perform_async(item_id) - SyncPlaidHoldingsJob.perform_async(item_id) + SyncPlaidTransactionsJob.perform(item_id) + SyncPlaidHoldingsJob.perform(item_id) end end diff --git a/app/sidekiq/sync_plaid_transactions_job.rb b/app/jobs/sync_plaid_transactions_job.rb similarity index 96% rename from app/sidekiq/sync_plaid_transactions_job.rb rename to app/jobs/sync_plaid_transactions_job.rb index 77690671..04461053 100644 --- a/app/sidekiq/sync_plaid_transactions_job.rb +++ b/app/jobs/sync_plaid_transactions_job.rb @@ -1,5 +1,4 @@ class SyncPlaidTransactionsJob - include Sidekiq::Job def perform(item_id) connection = Connection.find_by(source: 'plaid', item_id: item_id) @@ -27,7 +26,7 @@ class SyncPlaidTransactionsJob include_personal_finance_category: true } }) - + transactions_response = $plaid_api_client.transactions_sync(transactions_request) added_transactions += transactions_response.added @@ -92,10 +91,10 @@ class SyncPlaidTransactionsJob Transaction.where(source_transaction_id: removed_transactions).destroy_all end - EnrichTransactionsJob.perform_async + EnrichTransactionsJob.perform accounts.each do |account| - GenerateBalanceJob.perform_async(account.id) + GenerateBalanceJob.perform(account.id) end end end diff --git a/app/sidekiq/sync_property_values_job.rb b/app/jobs/sync_property_values_job.rb similarity index 97% rename from app/sidekiq/sync_property_values_job.rb rename to app/jobs/sync_property_values_job.rb index a72a5f38..2f6bd790 100644 --- a/app/sidekiq/sync_property_values_job.rb +++ b/app/jobs/sync_property_values_job.rb @@ -1,5 +1,4 @@ class SyncPropertyValuesJob - include Sidekiq::Job def perform(account_id) account = Account.find(account_id) @@ -8,11 +7,11 @@ class SyncPropertyValuesJob if account.auto_valuation && (account.current_balance_date.nil? || account.current_balance_date < 30.days.ago) url_formatted_address = "#{account.property_details['line_1'].gsub(' ','-')}-#{account.property_details['city']}-#{account.property_details['state_abbreviation']}-#{account.property_details['zip_code']}_rb" scraper = Faraday.get("https://app.scrapingbee.com/api/v1/?api_key=#{ENV['SCRAPING_BEE_KEY']}&url=https%3A%2F%2Fwww.zillow.com%2Fhomes%2F#{url_formatted_address}%2F&render_js=false&extract_rules=%7B%22value%22%3A'%2F%2Fspan%5B%40data-testid%3D%22zestimate-text%22%5D%2Fspan%2Fspan'%7D") - + # If the scraper returns a 200 status code, parse the response body and update the account if scraper.status == 200 and JSON.parse(scraper.body)['value'].present? account.update(current_balance: JSON.parse(scraper.body)['value'].gsub('$','').gsub(',','').to_i, current_balance_date: Date.today) end end end -end \ No newline at end of file +end diff --git a/app/sidekiq/sync_security_details_job.rb b/app/jobs/sync_security_details_job.rb similarity index 99% rename from app/sidekiq/sync_security_details_job.rb rename to app/jobs/sync_security_details_job.rb index 524573af..95277a2e 100644 --- a/app/sidekiq/sync_security_details_job.rb +++ b/app/jobs/sync_security_details_job.rb @@ -1,5 +1,4 @@ class SyncSecurityDetailsJob - include Sidekiq::Job def perform(security_id) security = Security.find_by(id: security_id) @@ -80,4 +79,4 @@ class SyncSecurityDetailsJob ["https://logo.clearbit.com/#{domain}", 'clearbit'] end -end \ No newline at end of file +end diff --git a/app/sidekiq/sync_security_history_job.rb b/app/jobs/sync_security_history_job.rb similarity index 94% rename from app/sidekiq/sync_security_history_job.rb rename to app/jobs/sync_security_history_job.rb index 2edfb2a7..7a8f52ac 100644 --- a/app/sidekiq/sync_security_history_job.rb +++ b/app/jobs/sync_security_history_job.rb @@ -1,5 +1,4 @@ class SyncSecurityHistoryJob - include Sidekiq::Job def perform(security_id) security = Security.find(security_id) @@ -38,7 +37,7 @@ class SyncSecurityHistoryJob end # remove duplicate dates - all_prices.uniq! { |price| price[:date] } + all_prices.uniq! { |price| price[:date] } SecurityPrice.upsert_all(all_prices, unique_by: :index_security_prices_on_security_id_and_date)