diff --git a/app/jobs/fetch_security_info_job.rb b/app/jobs/fetch_security_info_job.rb index 5eaafa43..aa64c169 100644 --- a/app/jobs/fetch_security_info_job.rb +++ b/app/jobs/fetch_security_info_job.rb @@ -6,10 +6,13 @@ class FetchSecurityInfoJob < ApplicationJob security = Security.find(security_id) - security_info_response = Security.security_info_provider.fetch_security_info( - ticker: security.ticker, - mic_code: security.exchange_mic - ) + params = { + ticker: security.ticker + } + params[:mic_code] = security.exchange_mic if security.exchange_mic.present? + params[:operating_mic] = security.exchange_operating_mic if security.exchange_operating_mic.present? + + security_info_response = Security.security_info_provider.fetch_security_info(**params) security.update( name: security_info_response.info.dig("name") diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rb index 8e0f9ad7..704d851f 100644 --- a/app/models/account/trade_builder.rb +++ b/app/models/account/trade_builder.rb @@ -111,13 +111,10 @@ class Account::TradeBuilder end def security - ticker_symbol, exchange_mic, exchange_acronym, exchange_country_code = ticker.split("|") + ticker_symbol, exchange_operating_mic = ticker.split("|") - security = Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic, country_code: exchange_country_code) - security.update(exchange_acronym: exchange_acronym) - - FetchSecurityInfoJob.perform_later(security.id) - - security + Security.find_or_create_by(ticker: ticker_symbol, exchange_operating_mic: exchange_operating_mic) do |s| + FetchSecurityInfoJob.perform_later(s.id) + end end end diff --git a/app/models/demo/generator.rb b/app/models/demo/generator.rb index 22038857..283ae35b 100644 --- a/app/models/demo/generator.rb +++ b/app/models/demo/generator.rb @@ -231,12 +231,12 @@ class Demo::Generator def load_securities! # Create an unknown security to simulate edge cases - Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock", exchange_mic: "UNKNOWN" + Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock" securities = [ - { ticker: "AAPL", exchange_mic: "NASDAQ", name: "Apple Inc.", reference_price: 210 }, - { ticker: "TM", exchange_mic: "NYSE", name: "Toyota Motor Corporation", reference_price: 202 }, - { ticker: "MSFT", exchange_mic: "NASDAQ", name: "Microsoft Corporation", reference_price: 455 } + { ticker: "AAPL", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Apple Inc.", reference_price: 210 }, + { ticker: "TM", exchange_mic: "XNYS", exchange_operating_mic: "XNYS", name: "Toyota Motor Corporation", reference_price: 202 }, + { ticker: "MSFT", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Microsoft Corporation", reference_price: 455 } ] securities.each do |security_attributes| diff --git a/app/models/plaid_investment_sync.rb b/app/models/plaid_investment_sync.rb index 8c1d9805..bcb1f330 100644 --- a/app/models/plaid_investment_sync.rb +++ b/app/models/plaid_investment_sync.rb @@ -82,12 +82,15 @@ class PlaidInvestmentSync end return [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank? + return [ nil, plaid_security ] if plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately + operating_mic = plaid_security.market_identifier_code + + # Find any matching security security = Security.find_or_create_by!( ticker: plaid_security.ticker_symbol, - exchange_mic: plaid_security.market_identifier_code || "XNAS", - country_code: "US" - ) unless plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately + exchange_operating_mic: operating_mic + ) [ security, plaid_security ] end diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb index a5356a0f..65deaae5 100644 --- a/app/models/provider/synth.rb +++ b/app/models/provider/synth.rb @@ -145,6 +145,7 @@ class Provider::Synth logo_url: security.dig("logo_url"), exchange_acronym: security.dig("exchange", "acronym"), exchange_mic: security.dig("exchange", "mic_code"), + exchange_operating_mic: security.dig("exchange", "operating_mic_code"), country_code: security.dig("exchange", "country_code") } end @@ -155,9 +156,10 @@ class Provider::Synth raw_response: response end - def fetch_security_info(ticker:, mic_code:) + def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil) response = client.get("#{base_url}/tickers/#{ticker}") do |req| - req.params["mic_code"] = mic_code + req.params["mic_code"] = mic_code if mic_code.present? + req.params["operating_mic"] = operating_mic if operating_mic.present? end parsed = JSON.parse(response.body) diff --git a/app/models/security.rb b/app/models/security.rb index d2ce6387..4a2bd1a7 100644 --- a/app/models/security.rb +++ b/app/models/security.rb @@ -6,7 +6,7 @@ class Security < ApplicationRecord has_many :prices, dependent: :destroy validates :ticker, presence: true - validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false } + validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false } class << self def search(query) @@ -30,11 +30,15 @@ class Security < ApplicationRecord name: name, logo_url: logo_url, exchange_acronym: exchange_acronym, - exchange_mic: exchange_mic, + exchange_operating_mic: exchange_operating_mic, exchange_country_code: country_code ) end + def has_prices? + exchange_operating_mic.present? + end + private def upcase_ticker diff --git a/app/models/security/price/provided.rb b/app/models/security/price/provided.rb index 02e5ce14..e2a99774 100644 --- a/app/models/security/price/provided.rb +++ b/app/models/security/price/provided.rb @@ -8,6 +8,7 @@ module Security::Price::Provided def fetch_price_from_provider(security:, date:, cache: false) return nil unless security_prices_provider.present? + return nil unless security.has_prices? response = security_prices_provider.fetch_security_prices \ ticker: security.ticker, @@ -32,6 +33,7 @@ module Security::Price::Provided def fetch_prices_from_provider(security:, start_date:, end_date:, cache: false) return [] unless security_prices_provider.present? return [] unless security + return [] unless security.has_prices? response = security_prices_provider.fetch_security_prices \ ticker: security.ticker, diff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rb index d3b4437d..9c2336df 100644 --- a/app/models/security/synth_combobox_option.rb +++ b/app/models/security/synth_combobox_option.rb @@ -1,13 +1,14 @@ class Security::SynthComboboxOption include ActiveModel::Model - attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic, :exchange_country_code + attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_country_code, :exchange_operating_mic def id - "#{symbol}|#{exchange_mic}|#{exchange_acronym}|#{exchange_country_code}" # submitted by combobox as value + "#{symbol}|#{exchange_operating_mic}" # submitted by combobox as value end def to_combobox_display - "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected + display_code = exchange_acronym.presence || exchange_operating_mic + "#{symbol} - #{name} (#{display_code})" # shown in combobox input when selected end end diff --git a/app/views/securities/_combobox_security.turbo_stream.erb b/app/views/securities/_combobox_security.turbo_stream.erb index cc0667a3..b14708c7 100644 --- a/app/views/securities/_combobox_security.turbo_stream.erb +++ b/app/views/securities/_combobox_security.turbo_stream.erb @@ -5,7 +5,7 @@ <%= combobox_security.name.presence || combobox_security.symbol %> - <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym})" %> + <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym.presence || combobox_security.exchange_operating_mic})" %> diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb new file mode 100644 index 00000000..cdd424f5 --- /dev/null +++ b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb @@ -0,0 +1,6 @@ +class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2] + def change + add_column :securities, :exchange_operating_mic, :string + add_index :securities, :exchange_operating_mic + end +end diff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rb new file mode 100644 index 00000000..b04d8de1 --- /dev/null +++ b/db/migrate/20250207194638_adjust_securities_indexes.rb @@ -0,0 +1,6 @@ +class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2] + def change + remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic" + add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true + end +end diff --git a/db/migrate/20250211161238_make_ticker_not_null.rb b/db/migrate/20250211161238_make_ticker_not_null.rb new file mode 100644 index 00000000..ee22e6da --- /dev/null +++ b/db/migrate/20250211161238_make_ticker_not_null.rb @@ -0,0 +1,5 @@ +class MakeTickerNotNull < ActiveRecord::Migration[7.2] + def change + change_column_null :securities, :ticker, false + end +end diff --git a/db/schema.rb b/db/schema.rb index f44329c6..c2d3815b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do +ActiveRecord::Schema[7.2].define(version: 2025_02_11_161238) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -542,7 +542,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do end create_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.string "ticker" + t.string "ticker", null: false t.string "name" t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -550,8 +550,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do t.string "exchange_mic" t.string "exchange_acronym" t.string "logo_url" + t.string "exchange_operating_mic" t.index ["country_code"], name: "index_securities_on_country_code" - t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true + t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic" + t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: true end create_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| diff --git a/lib/tasks/securities.rake b/lib/tasks/securities.rake new file mode 100644 index 00000000..d1cbcb5a --- /dev/null +++ b/lib/tasks/securities.rake @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +namespace :securities do + desc "Backfill exchange_operating_mic for securities using Synth API" + task backfill_exchange_mic: :environment do + puts "Starting exchange_operating_mic backfill..." + + api_key = Rails.application.config.app_mode.self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"] + unless api_key.present? + puts "ERROR: No Synth API key found. Please set SYNTH_API_KEY env var or configure it in Settings for self-hosted mode." + exit 1 + end + + securities = Security.where(exchange_operating_mic: nil).where.not(ticker: nil) + total = securities.count + processed = 0 + errors = [] + + securities.find_each do |security| + processed += 1 + print "\rProcessing #{processed}/#{total} (#{(processed.to_f/total * 100).round(1)}%)" + + begin + response = Faraday.get("https://api.synthfinance.com/tickers/#{security.ticker}") do |req| + req.params["country_code"] = security.country_code if security.country_code.present? + req.headers["Authorization"] = "Bearer #{api_key}" + end + + if response.success? + data = JSON.parse(response.body).dig("data") + exchange_data = data["exchange"] + + # Update security with exchange info and other metadata + security.update!( + exchange_operating_mic: exchange_data["operating_mic_code"], + exchange_mic: exchange_data["mic_code"], + exchange_acronym: exchange_data["acronym"], + name: data["name"], + logo_url: data["logo_url"], + country_code: exchange_data["country_code"] + ) + else + errors << "#{security.ticker}: HTTP #{response.status} - #{response.body}" + end + rescue => e + errors << "#{security.ticker}: #{e.message}" + end + + # Add a small delay to not overwhelm the API + sleep(0.1) + end + + puts "\n\nBackfill complete!" + puts "Processed #{processed} securities" + + if errors.any? + puts "\nErrors encountered:" + errors.each { |error| puts " - #{error}" } + end + end + + desc "De-duplicate securities based on ticker + exchange_operating_mic" + task :deduplicate, [ :dry_run ] => :environment do |_t, args| + # First check if we have any securities without exchange_operating_mic + missing_mic_count = Security.where(exchange_operating_mic: nil).where.not(ticker: nil).count + + if missing_mic_count > 0 + puts "ERROR: Found #{missing_mic_count} securities without exchange_operating_mic." + puts "Please run 'rails securities:backfill_exchange_mic' first to ensure all securities have exchange_operating_mic values." + exit 1 + end + + dry_run = args[:dry_run].present? + puts "Starting securities de-duplication... #{dry_run ? '(DRY RUN)' : ''}" + + # Find all duplicate securities (same ticker + exchange_operating_mic) + duplicates = Security + .where.not(ticker: nil) + .where.not(exchange_operating_mic: nil) + .group(:ticker, :exchange_operating_mic) + .having("COUNT(*) > 1") + .pluck(:ticker, :exchange_operating_mic) + + puts "Found #{duplicates.length} sets of duplicate securities" + total_holdings = 0 + total_trades = 0 + total_prices = 0 + + duplicates.each do |ticker, exchange_operating_mic| + securities = Security.where(ticker: ticker, exchange_operating_mic: exchange_operating_mic) + .order(created_at: :asc) + + canonical = securities.first + duplicates = securities[1..] + + puts "\nProcessing #{ticker} (#{exchange_operating_mic}):" + puts " Canonical: #{canonical.id} (created: #{canonical.created_at})" + puts " Duplicates: #{duplicates.map(&:id).join(', ')}" + + # Count affected records + holdings_count = Account::Holding.where(security_id: duplicates).count + trades_count = Account::Trade.where(security_id: duplicates).count + prices_count = Security::Price.where(security_id: duplicates).count + + total_holdings += holdings_count + total_trades += trades_count + total_prices += prices_count + + puts " Would update:" + puts " - #{holdings_count} holdings" + puts " - #{trades_count} trades" + puts " - #{prices_count} prices" + + unless dry_run + begin + ActiveRecord::Base.transaction do + # Update all references to point to the canonical security + Account::Holding.where(security_id: duplicates).update_all(security_id: canonical.id) + Account::Trade.where(security_id: duplicates).update_all(security_id: canonical.id) + Security::Price.where(security_id: duplicates).update_all(security_id: canonical.id) + + # Delete the duplicates + duplicates.each(&:destroy!) + end + puts " ✓ Successfully merged and removed duplicates" + rescue => e + puts " ✗ Error processing #{ticker}: #{e.message}" + end + end + end + + puts "\nSummary:" + puts " Total duplicate sets: #{duplicates.length}" + puts " Total affected records:" + puts " - #{total_holdings} holdings" + puts " - #{total_trades} trades" + puts " - #{total_prices} prices" + puts " Mode: #{dry_run ? 'Dry run (no changes made)' : 'Live run (changes applied)'}" + puts "\nDe-duplication complete!" + end +end diff --git a/test/fixtures/securities.yml b/test/fixtures/securities.yml index e5e38687..9133f008 100644 --- a/test/fixtures/securities.yml +++ b/test/fixtures/securities.yml @@ -1,12 +1,12 @@ aapl: ticker: AAPL name: Apple - exchange_mic: XNAS + exchange_operating_mic: XNAS country_code: US msft: ticker: MSFT name: Microsoft - exchange_mic: XNAS + exchange_operating_mic: XNAS country_code: US