Enhance security information retrieval and handling (#1826)

* Enhance security information retrieval and handling

- Add support for operating MIC codes in security info fetching
- Update security uniqueness validation to handle unknown securities
- Improve security creation and update logic in Plaid investment sync
- Update combobox and view components to handle operating MIC codes
- Add unknown flag for securities with incomplete information

* Update schema.rb

* Refactor the need for mic codes

* Don't fetch prices unless a security has the necessary mic code

* Deduplication

* Lint

* Update Securities and Plaid Investment Sync

- Modify PlaidInvestmentSync to return plaid_security for USD cash
- Add non-null constraint to Securities ticker column
- Update Securities fixture to use exchange_operating_mic instead of exchange_mic

---------

Signed-off-by: Josh Pigford <josh@joshpigford.com>

app/jobs/fetch_security_info_job.rb

@@ -6,10 +6,13 @@ class FetchSecurityInfoJob < ApplicationJob
 
     security = Security.find(security_id)
 
-    security_info_response = Security.security_info_provider.fetch_security_info(
-      ticker: security.ticker,
-      mic_code: security.exchange_mic
-    )
+    params = {
+      ticker: security.ticker
+    }
+    params[:mic_code] = security.exchange_mic if security.exchange_mic.present?
+    params[:operating_mic] = security.exchange_operating_mic if security.exchange_operating_mic.present?
+
+    security_info_response = Security.security_info_provider.fetch_security_info(**params)
 
     security.update(
       name: security_info_response.info.dig("name")

app/models/account/trade_builder.rb

@@ -111,13 +111,10 @@ class Account::TradeBuilder
     end
 
     def security
-      ticker_symbol, exchange_mic, exchange_acronym, exchange_country_code = ticker.split("|")
+      ticker_symbol, exchange_operating_mic = ticker.split("|")
 
-      security = Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic, country_code: exchange_country_code)
-      security.update(exchange_acronym: exchange_acronym)
-
-      FetchSecurityInfoJob.perform_later(security.id)
-
-      security
+      Security.find_or_create_by(ticker: ticker_symbol, exchange_operating_mic: exchange_operating_mic) do |s|
+        FetchSecurityInfoJob.perform_later(s.id)
+      end
     end
 end

app/models/demo/generator.rb

@@ -231,12 +231,12 @@ class Demo::Generator
 
     def load_securities!
       # Create an unknown security to simulate edge cases
-      Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock", exchange_mic: "UNKNOWN"
+      Security.create! ticker: "UNKNOWN", name: "Unknown Demo Stock"
 
       securities = [
-        { ticker: "AAPL", exchange_mic: "NASDAQ", name: "Apple Inc.", reference_price: 210 },
-        { ticker: "TM", exchange_mic: "NYSE", name: "Toyota Motor Corporation", reference_price: 202 },
-        { ticker: "MSFT", exchange_mic: "NASDAQ", name: "Microsoft Corporation", reference_price: 455 }
+        { ticker: "AAPL", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Apple Inc.", reference_price: 210 },
+        { ticker: "TM", exchange_mic: "XNYS", exchange_operating_mic: "XNYS", name: "Toyota Motor Corporation", reference_price: 202 },
+        { ticker: "MSFT", exchange_mic: "XNGS", exchange_operating_mic: "XNAS", name: "Microsoft Corporation", reference_price: 455 }
       ]
 
       securities.each do |security_attributes|

app/models/plaid_investment_sync.rb

@@ -82,12 +82,15 @@ class PlaidInvestmentSync
       end
 
       return [ nil, nil ] if plaid_security.nil? || plaid_security.ticker_symbol.blank?
+      return [ nil, plaid_security ] if plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
 
+      operating_mic = plaid_security.market_identifier_code
+
+      # Find any matching security
       security = Security.find_or_create_by!(
         ticker: plaid_security.ticker_symbol,
-        exchange_mic: plaid_security.market_identifier_code || "XNAS",
-        country_code: "US"
-      ) unless plaid_security.ticker_symbol == "CUR:USD" # internally, we do not consider cash a security and track it separately
+        exchange_operating_mic: operating_mic
+      )
 
       [ security, plaid_security ]
     end

app/models/provider/synth.rb

@@ -145,6 +145,7 @@ class Provider::Synth
         logo_url: security.dig("logo_url"),
         exchange_acronym: security.dig("exchange", "acronym"),
         exchange_mic: security.dig("exchange", "mic_code"),
+        exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
         country_code: security.dig("exchange", "country_code")
       }
     end
@@ -155,9 +156,10 @@ class Provider::Synth
       raw_response: response
   end
 
-  def fetch_security_info(ticker:, mic_code:)
+  def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil)
     response = client.get("#{base_url}/tickers/#{ticker}") do |req|
-      req.params["mic_code"] = mic_code
+      req.params["mic_code"] = mic_code if mic_code.present?
+      req.params["operating_mic"] = operating_mic if operating_mic.present?
     end
 
     parsed = JSON.parse(response.body)

app/models/security.rb

@@ -6,7 +6,7 @@ class Security < ApplicationRecord
   has_many :prices, dependent: :destroy
 
   validates :ticker, presence: true
-  validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }
+  validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }
 
   class << self
     def search(query)
@@ -30,11 +30,15 @@ class Security < ApplicationRecord
       name: name,
       logo_url: logo_url,
       exchange_acronym: exchange_acronym,
-      exchange_mic: exchange_mic,
+      exchange_operating_mic: exchange_operating_mic,
       exchange_country_code: country_code
     )
   end
 
+  def has_prices?
+    exchange_operating_mic.present?
+  end
+
   private
 
     def upcase_ticker

app/models/security/price/provided.rb

@@ -8,6 +8,7 @@ module Security::Price::Provided
 
       def fetch_price_from_provider(security:, date:, cache: false)
         return nil unless security_prices_provider.present?
+        return nil unless security.has_prices?
 
         response = security_prices_provider.fetch_security_prices \
           ticker: security.ticker,
@@ -32,6 +33,7 @@ module Security::Price::Provided
       def fetch_prices_from_provider(security:, start_date:, end_date:, cache: false)
         return [] unless security_prices_provider.present?
         return [] unless security
+        return [] unless security.has_prices?
 
         response = security_prices_provider.fetch_security_prices \
           ticker: security.ticker,

app/models/security/synth_combobox_option.rb

@@ -1,13 +1,14 @@
 class Security::SynthComboboxOption
   include ActiveModel::Model
 
-  attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic, :exchange_country_code
+  attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_country_code, :exchange_operating_mic
 
   def id
-    "#{symbol}|#{exchange_mic}|#{exchange_acronym}|#{exchange_country_code}" # submitted by combobox as value
+    "#{symbol}|#{exchange_operating_mic}" # submitted by combobox as value
   end
 
   def to_combobox_display
-    "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected
+    display_code = exchange_acronym.presence || exchange_operating_mic
+    "#{symbol} - #{name} (#{display_code})" # shown in combobox input when selected
   end
 end

app/views/securities/_combobox_security.turbo_stream.erb

@@ -5,7 +5,7 @@
       <%= combobox_security.name.presence || combobox_security.symbol %>
     </span>
     <span class="text-xs text-gray-500">
-      <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym})" %>
+      <%= "#{combobox_security.symbol} (#{combobox_security.exchange_acronym.presence || combobox_security.exchange_operating_mic})" %>
     </span>
   </div>
 </div>

db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb

@@ -0,0 +1,6 @@
+class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2]
+  def change
+    add_column :securities, :exchange_operating_mic, :string
+    add_index :securities, :exchange_operating_mic
+  end
+end

db/migrate/20250207194638_adjust_securities_indexes.rb

@@ -0,0 +1,6 @@
+class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2]
+  def change
+    remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic"
+    add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true
+  end
+end

db/migrate/20250211161238_make_ticker_not_null.rb

@@ -0,0 +1,5 @@
+class MakeTickerNotNull < ActiveRecord::Migration[7.2]
+  def change
+    change_column_null :securities, :ticker, false
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
+ActiveRecord::Schema[7.2].define(version: 2025_02_11_161238) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -542,7 +542,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
   end
 
   create_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
-    t.string "ticker"
+    t.string "ticker", null: false
     t.string "name"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -550,8 +550,10 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_07_014022) do
     t.string "exchange_mic"
     t.string "exchange_acronym"
     t.string "logo_url"
+    t.string "exchange_operating_mic"
     t.index ["country_code"], name: "index_securities_on_country_code"
-    t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true
+    t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic"
+    t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: true
   end
 
   create_table "security_prices", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|

lib/tasks/securities.rake

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+namespace :securities do
+  desc "Backfill exchange_operating_mic for securities using Synth API"
+  task backfill_exchange_mic: :environment do
+    puts "Starting exchange_operating_mic backfill..."
+
+    api_key = Rails.application.config.app_mode.self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+    unless api_key.present?
+      puts "ERROR: No Synth API key found. Please set SYNTH_API_KEY env var or configure it in Settings for self-hosted mode."
+      exit 1
+    end
+
+    securities = Security.where(exchange_operating_mic: nil).where.not(ticker: nil)
+    total = securities.count
+    processed = 0
+    errors = []
+
+    securities.find_each do |security|
+      processed += 1
+      print "\rProcessing #{processed}/#{total} (#{(processed.to_f/total * 100).round(1)}%)"
+
+      begin
+        response = Faraday.get("https://api.synthfinance.com/tickers/#{security.ticker}") do |req|
+          req.params["country_code"] = security.country_code if security.country_code.present?
+          req.headers["Authorization"] = "Bearer #{api_key}"
+        end
+
+        if response.success?
+          data = JSON.parse(response.body).dig("data")
+          exchange_data = data["exchange"]
+
+          # Update security with exchange info and other metadata
+          security.update!(
+            exchange_operating_mic: exchange_data["operating_mic_code"],
+            exchange_mic: exchange_data["mic_code"],
+            exchange_acronym: exchange_data["acronym"],
+            name: data["name"],
+            logo_url: data["logo_url"],
+            country_code: exchange_data["country_code"]
+          )
+        else
+          errors << "#{security.ticker}: HTTP #{response.status} - #{response.body}"
+        end
+      rescue => e
+        errors << "#{security.ticker}: #{e.message}"
+      end
+
+      # Add a small delay to not overwhelm the API
+      sleep(0.1)
+    end
+
+    puts "\n\nBackfill complete!"
+    puts "Processed #{processed} securities"
+
+    if errors.any?
+      puts "\nErrors encountered:"
+      errors.each { |error| puts "  - #{error}" }
+    end
+  end
+
+  desc "De-duplicate securities based on ticker + exchange_operating_mic"
+  task :deduplicate, [ :dry_run ] => :environment do |_t, args|
+    # First check if we have any securities without exchange_operating_mic
+    missing_mic_count = Security.where(exchange_operating_mic: nil).where.not(ticker: nil).count
+
+    if missing_mic_count > 0
+      puts "ERROR: Found #{missing_mic_count} securities without exchange_operating_mic."
+      puts "Please run 'rails securities:backfill_exchange_mic' first to ensure all securities have exchange_operating_mic values."
+      exit 1
+    end
+
+    dry_run = args[:dry_run].present?
+    puts "Starting securities de-duplication... #{dry_run ? '(DRY RUN)' : ''}"
+
+    # Find all duplicate securities (same ticker + exchange_operating_mic)
+    duplicates = Security
+      .where.not(ticker: nil)
+      .where.not(exchange_operating_mic: nil)
+      .group(:ticker, :exchange_operating_mic)
+      .having("COUNT(*) > 1")
+      .pluck(:ticker, :exchange_operating_mic)
+
+    puts "Found #{duplicates.length} sets of duplicate securities"
+    total_holdings = 0
+    total_trades = 0
+    total_prices = 0
+
+    duplicates.each do |ticker, exchange_operating_mic|
+      securities = Security.where(ticker: ticker, exchange_operating_mic: exchange_operating_mic)
+        .order(created_at: :asc)
+
+      canonical = securities.first
+      duplicates = securities[1..]
+
+      puts "\nProcessing #{ticker} (#{exchange_operating_mic}):"
+      puts "  Canonical: #{canonical.id} (created: #{canonical.created_at})"
+      puts "  Duplicates: #{duplicates.map(&:id).join(', ')}"
+
+      # Count affected records
+      holdings_count = Account::Holding.where(security_id: duplicates).count
+      trades_count = Account::Trade.where(security_id: duplicates).count
+      prices_count = Security::Price.where(security_id: duplicates).count
+
+      total_holdings += holdings_count
+      total_trades += trades_count
+      total_prices += prices_count
+
+      puts "  Would update:"
+      puts "    - #{holdings_count} holdings"
+      puts "    - #{trades_count} trades"
+      puts "    - #{prices_count} prices"
+
+      unless dry_run
+        begin
+          ActiveRecord::Base.transaction do
+            # Update all references to point to the canonical security
+            Account::Holding.where(security_id: duplicates).update_all(security_id: canonical.id)
+            Account::Trade.where(security_id: duplicates).update_all(security_id: canonical.id)
+            Security::Price.where(security_id: duplicates).update_all(security_id: canonical.id)
+
+            # Delete the duplicates
+            duplicates.each(&:destroy!)
+          end
+          puts "  ✓ Successfully merged and removed duplicates"
+        rescue => e
+          puts "  ✗ Error processing #{ticker}: #{e.message}"
+        end
+      end
+    end
+
+    puts "\nSummary:"
+    puts "  Total duplicate sets: #{duplicates.length}"
+    puts "  Total affected records:"
+    puts "    - #{total_holdings} holdings"
+    puts "    - #{total_trades} trades"
+    puts "    - #{total_prices} prices"
+    puts "  Mode: #{dry_run ? 'Dry run (no changes made)' : 'Live run (changes applied)'}"
+    puts "\nDe-duplication complete!"
+  end
+end

test/fixtures/securities.yml

@@ -1,12 +1,12 @@
 aapl:
   ticker: AAPL
   name: Apple
-  exchange_mic: XNAS
+  exchange_operating_mic: XNAS
   country_code: US
 
 msft:
   ticker: MSFT
   name: Microsoft
-  exchange_mic: XNAS
+  exchange_operating_mic: XNAS
   country_code: US