Initial pass at Synth-based ticker selection (#1392)

* Initial pass at Synth-based ticker selection

* Update _tickers.turbo_stream.erb

* Functional combobox display

* A few cleanup steps

* Linter

* Prevent long strings

* Another step towards functional combobox

* Deprecated files

* Custom Combobox implementation

* Lint

* Test suite fixes

* Lint

* Make direct use of mic codes

* Update splits

* Update trades_test.rb

.env.example

@@ -11,14 +11,10 @@
 # For users who have other applications listening at 3000, this allows them to set a value puma will listen to.
 PORT=3000
 
-# Exchange Rate & US Stock Pricing API
+# Exchange Rate & Stock Pricing API
-# This is used to convert between different currencies in the app. In addition, it fetches US stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.
+# This is used to convert between different currencies in the app. In addition, it fetches global stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com.
 SYNTH_API_KEY=
 
-# Non-US Stock Pricing API
-# This is used to fetch non-US stock prices. We use Marketstack.com for this and while they offer a free tier, it is quite limited. You'll almost certainly need their Basic plan, which is $9.99 per month.
-MARKETSTACK_API_KEY=
-
 # SMTP Configuration
 # This is only needed if you intend on sending emails from your Maybe instance (such as for password resets or email financial reports).
 # Resend.com is a good option that offers a free tier for sending emails.

.env.local.example

@@ -3,6 +3,3 @@ SELF_HOSTED=false
 
 # Enable Synth market data (careful, this will use your API credits)
 SYNTH_API_KEY=yourapikeyhere
-
-# Enable Marketstack market data (careful, this will use your API credits)
-MARKETSTACK_API_KEY=yourapikeyhere

app/controllers/account/trades_controller.rb

@@ -34,7 +34,10 @@ class Account::TradesController < ApplicationController
   end
 
   def securities
-    @pagy, @securities = pagy(Security.order(:name).search(params[:q]), limit: 20)
+    query = params[:q]
+    return render json: [] if query.blank? || query.length < 2 || query.length > 100
+
+    @securities = Security::SynthComboboxOption.find_in_synth(query)
   end
 
   private

app/jobs/securities_import_job.rb

@@ -1,13 +0,0 @@
-class SecuritiesImportJob < ApplicationJob
-  queue_as :default
-
-  def perform(country_code = nil)
-    exchanges = StockExchange.in_country(country_code)
-    market_stack_client = Provider::Marketstack.new(ENV["MARKETSTACK_API_KEY"])
-
-    exchanges.each do |exchange|
-      importer = Security::Importer.new(market_stack_client, exchange.mic)
-      importer.import
-    end
-  end
-end

app/models/account/trade_builder.rb

@@ -31,8 +31,9 @@ class Account::TradeBuilder < Account::EntryBuilder
     end
 
     def security
-      return Security.find(ticker) if ticker.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i)
+      ticker_symbol, exchange_mic = ticker.split("|")
-      Security.find_or_create_by(ticker: ticker)
+
+      Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic)
     end
 
     def amount

app/models/provider/marketstack.rb

@@ -1,120 +0,0 @@
-class Provider::Marketstack
-  include Retryable
-
-  def initialize(api_key)
-    @api_key = api_key
-  end
-
-  def fetch_security_prices(ticker:, start_date:, end_date:)
-    prices = paginate("#{base_url}/eod", {
-      symbols: ticker,
-      date_from: start_date.to_s,
-      date_to: end_date.to_s
-    }) do |body|
-      body.dig("data").map do |price|
-        {
-          date: price["date"],
-          price: price["close"]&.to_f,
-          currency: "USD"
-        }
-      end
-    end
-
-    SecurityPriceResponse.new(
-      prices: prices,
-      success?: true,
-      raw_response: prices.to_json
-    )
-  rescue StandardError => error
-    SecurityPriceResponse.new(
-      success?: false,
-      error: error,
-      raw_response: error
-    )
-  end
-
-  def fetch_tickers(exchange_mic: nil)
-    url = exchange_mic ? "#{base_url}/tickers?exchange=#{exchange_mic}" : "#{base_url}/tickers"
-    tickers = paginate(url) do |body|
-      body.dig("data").map do |ticker|
-        {
-          name: ticker["name"],
-          symbol: ticker["symbol"],
-          exchange_mic: exchange_mic || ticker.dig("stock_exchange", "mic"),
-          exchange_acronym: ticker.dig("stock_exchange", "acronym"),
-          country_code: ticker.dig("stock_exchange", "country_code")
-        }
-      end
-    end
-
-    TickerResponse.new(
-      tickers: tickers,
-      success?: true,
-      raw_response: tickers.to_json
-    )
-  rescue StandardError => error
-    TickerResponse.new(
-      success?: false,
-      error: error,
-      raw_response: error
-    )
-  end
-
-  private
-
-    attr_reader :api_key
-
-    SecurityPriceResponse = Struct.new(:prices, :success?, :error, :raw_response, keyword_init: true)
-    TickerResponse = Struct.new(:tickers, :success?, :error, :raw_response, keyword_init: true)
-
-    def base_url
-      "https://api.marketstack.com/v1"
-    end
-
-    def client
-      @client ||= Faraday.new(url: base_url) do |faraday|
-        faraday.params["access_key"] = api_key
-      end
-    end
-
-    def build_error(response)
-      Provider::Base::ProviderError.new(<<~ERROR)
-        Failed to fetch data from #{self.class}
-          Status: #{response.status}
-          Body: #{response.body.inspect}
-      ERROR
-    end
-
-    def fetch_page(url, page, params = {})
-      client.get(url) do |req|
-        params.each { |k, v| req.params[k.to_s] = v.to_s }
-        req.params["offset"] = (page - 1) * 100 # Marketstack uses offset-based pagination
-        req.params["limit"] = 10000 # Maximum allowed by Marketstack
-      end
-    end
-
-    def paginate(url, params = {})
-      results = []
-      page = 1
-      total_results = Float::INFINITY
-
-      while results.length < total_results
-        response = fetch_page(url, page, params)
-
-        if response.success?
-          body = JSON.parse(response.body)
-          page_results = yield(body)
-          results.concat(page_results)
-
-          total_results = body.dig("pagination", "total")
-          page += 1
-        else
-          raise build_error(response)
-        end
-
-        break if results.length >= total_results
-      end
-
-      results
-    end
-end

app/models/provider/synth.rb

@@ -122,6 +122,31 @@ class Provider::Synth
       raw_response: error
   end
 
+  def search_securities(query:, dataset: "limited", country_code:)
+    response = client.get("#{base_url}/tickers/search") do |req|
+      req.params["name"] = query
+      req.params["dataset"] = dataset
+      req.params["country_code"] = country_code
+    end
+
+    parsed = JSON.parse(response.body)
+
+    securities = parsed.dig("data").map do |security|
+      {
+        symbol: security.dig("symbol"),
+        name: security.dig("name"),
+        logo_url: security.dig("logo_url"),
+        exchange_acronym: security.dig("exchange", "acronym"),
+        exchange_mic: security.dig("exchange", "mic_code")
+      }
+    end
+
+    SearchSecuritiesResponse.new \
+      securities: securities,
+      success?: true,
+      raw_response: response
+  end
+
   private
 
     attr_reader :api_key
@@ -130,6 +155,7 @@ class Provider::Synth
     SecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: true
     ExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: true
     UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
+    SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
 
     def base_url
       "https://api.synthfinance.com"

app/models/security.rb

@@ -1,4 +1,5 @@
 class Security < ApplicationRecord
+  include Providable
   before_save :upcase_ticker
 
   has_many :trades, dependent: :nullify, class_name: "Account::Trade"
@@ -7,25 +8,6 @@ class Security < ApplicationRecord
   validates :ticker, presence: true
   validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false }
 
-  scope :search, ->(query) {
-    return none if query.blank? || query.length < 2
-
-    # Clean and normalize the search terms
-    sanitized_query = query.split.map do |term|
-      cleaned_term = term.gsub(/[^a-zA-Z0-9]/, " ").strip
-      next if cleaned_term.blank?
-      cleaned_term
-    end.compact.join(" | ")
-
-    return none if sanitized_query.blank?
-
-    sanitized_query = ActiveRecord::Base.connection.quote(sanitized_query)
-
-    where("search_vector @@ to_tsquery('simple', #{sanitized_query}) AND exchange_mic IS NOT NULL")
-      .select("securities.*, ts_rank_cd(search_vector, to_tsquery('simple', #{sanitized_query})) AS rank")
-      .reorder("rank DESC")
-  }
-
   def current_price
     @current_price ||= Security::Price.find_price(security: self, date: Date.current)
     return nil if @current_price.nil?
@@ -33,7 +15,7 @@ class Security < ApplicationRecord
   end
 
   def to_combobox_display
-    "#{ticker} - #{name} (#{exchange_acronym})"
+    "#{ticker} (#{exchange_acronym})"
   end
 
 

app/models/security/importer.rb

@@ -1,45 +0,0 @@
-class Security::Importer
-  def initialize(provider, stock_exchange = nil)
-    @provider = provider
-    @stock_exchange = stock_exchange
-  end
-
-  def import
-    securities = @provider.fetch_tickers(exchange_mic: @stock_exchange)&.tickers
-
-    # Deduplicate securities based on ticker and exchange_mic
-    securities_to_create = securities
-      .map do |security|
-        {
-          name: security[:name],
-          ticker: security[:symbol],
-          country_code: security[:country_code],
-          exchange_mic: security[:exchange_mic],
-          exchange_acronym: security[:exchange_acronym]
-        }
-      end
-      .compact
-      .uniq { |security| [ security[:ticker], security[:exchange_mic] ] }
-
-    # First update any existing securities that only have a ticker
-    Security.where(exchange_mic: nil)
-      .where(ticker: securities_to_create.map { |s| s[:ticker] })
-      .update_all(
-        securities_to_create.map do |security|
-          {
-            name: security[:name],
-            country_code: security[:country_code],
-            exchange_mic: security[:exchange_mic],
-            exchange_acronym: security[:exchange_acronym]
-          }
-        end.first
-      )
-
-    # Then create/update any remaining securities
-    Security.upsert_all(
-      securities_to_create,
-      unique_by: [ :ticker, :exchange_mic ],
-      update_only: [ :name, :country_code, :exchange_acronym ]
-    ) unless securities_to_create.empty?
-  end
-end

app/models/security/synth_combobox_option.rb

@@ -0,0 +1,20 @@
+class Security::SynthComboboxOption
+  include ActiveModel::Model
+  include Providable
+
+  attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic
+
+  class << self
+    def find_in_synth(query)
+      security_prices_provider.search_securities(query:, dataset: "limited", country_code: Current.family.country).securities.map { |attrs| new(**attrs) }
+    end
+  end
+
+  def id
+    "#{symbol}|#{exchange_mic}" # submitted by combobox as value
+  end
+
+  def to_combobox_display
+    "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected
+  end
+end

app/views/account/trades/_form.html.erb

@@ -8,7 +8,7 @@
       <%= form.select :type, options_for_select([%w[Buy buy], %w[Sell sell], %w[Deposit transfer_in], %w[Withdrawal transfer_out], %w[Interest interest]], "buy"), { label: t(".type") }, { data: { "trade-form-target": "typeInput" } } %>
       <div data-trade-form-target="tickerInput">
         <div class="form-field combobox">
-          <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder"), autocomplete: :list, free_text: true %>
+          <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder") %>
         </div>
       </div>
 

app/views/account/trades/_security.turbo_stream.erb

@@ -0,0 +1,11 @@
+<div class="flex items-center">
+  <%= image_tag(security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %>
+  <div class="flex flex-col">
+    <span class="text-sm font-medium">
+      <%= security.name.presence || security.symbol %>
+    </span>
+    <span class="text-xs text-gray-500">
+      <%= "#{security.symbol} (#{security.exchange_acronym})" %>
+    </span>
+  </div>
+</div>

app/views/account/trades/_tickers.turbo_stream.erb

@@ -1,7 +0,0 @@
-<div class="flex items-center">
-  <%= image_tag("https://logo.synthfinance.com/ticker/#{tickers&.ticker}", class: "rounded-full h-8 w-8 inline-block mr-2") %>
-  <div class="flex flex-col">
-    <span class="text-sm font-medium"><%= tickers&.name.presence || tickers&.ticker %></span>
-    <span class="text-xs text-gray-500"><%= "#{tickers&.ticker} (#{tickers&.exchange_acronym})" %></span>
-  </div>
-</div>

app/views/account/trades/securities.turbo_stream.erb

@@ -1,3 +1,2 @@
 <%= async_combobox_options @securities,
-      render_in: { partial: "account/trades/tickers" },
+    render_in: { partial: "account/trades/security" } %>
-      next_page: @pagy.next %>

db/migrate/20241029234028_remove_search_vector.rb

@@ -0,0 +1,5 @@
+class RemoveSearchVector < ActiveRecord::Migration[7.2]
+  def change
+    remove_column :securities, :search_vector
+  end
+end

db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb

@@ -0,0 +1,10 @@
+class FixNotNullStockExchangeData < ActiveRecord::Migration[7.2]
+  def change
+    change_column_null :stock_exchanges, :currency_code, true
+    change_column_null :stock_exchanges, :currency_symbol, true
+    change_column_null :stock_exchanges, :currency_name, true
+    change_column_null :stock_exchanges, :city, true
+    change_column_null :stock_exchanges, :timezone_name, true
+    change_column_null :stock_exchanges, :timezone_abbr, true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
+ActiveRecord::Schema[7.2].define(version: 2024_10_30_121302) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -119,7 +119,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
     t.boolean "is_active", default: true, null: false
     t.date "last_sync_date"
     t.uuid "institution_id"
-    t.virtual "classification", type: :string, as: "\nCASE\n    WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n    ELSE 'asset'::text\nEND", stored: true
+    t.virtual "classification", type: :string, as: "\nCASE\n    WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n    ELSE 'asset'::text\nEND", stored: true
     t.uuid "import_id"
     t.string "mode"
     t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type"
@@ -481,9 +481,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
     t.string "country_code"
     t.string "exchange_mic"
     t.string "exchange_acronym"
-    t.virtual "search_vector", type: :tsvector, as: "(setweight(to_tsvector('simple'::regconfig, (COALESCE(ticker, ''::character varying))::text), 'B'::\"char\") || to_tsvector('simple'::regconfig, (COALESCE(name, ''::character varying))::text))", stored: true
     t.index ["country_code"], name: "index_securities_on_country_code"
-    t.index ["search_vector"], name: "index_securities_on_search_vector", using: :gin
     t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true
   end
 
@@ -524,14 +522,14 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do
     t.string "mic", null: false
     t.string "country", null: false
     t.string "country_code", null: false
-    t.string "city", null: false
+    t.string "city"
     t.string "website"
-    t.string "timezone_name", null: false
+    t.string "timezone_name"
-    t.string "timezone_abbr", null: false
+    t.string "timezone_abbr"
     t.string "timezone_abbr_dst"
-    t.string "currency_code", null: false
+    t.string "currency_code"
-    t.string "currency_symbol", null: false
+    t.string "currency_symbol"
-    t.string "currency_name", null: false
+    t.string "currency_name"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["country"], name: "index_stock_exchanges_on_country"

test/controllers/account/trades_controller_test.rb

@@ -97,7 +97,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest
         account_entry: {
           type: "buy",
           date: Date.current,
-          ticker: "NVDA",
+          ticker: "NVDA (NASDAQ)",
           qty: 10,
           price: 10
         }
@@ -118,7 +118,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest
       post account_trades_url(@entry.account), params: {
         account_entry: {
           type: "sell",
-          ticker: "AAPL",
+          ticker: "AAPL (NYSE)",
           date: Date.current,
           currency: "USD",
           qty: 10,

test/fixtures/stock_exchanges.yml

@@ -0,0 +1,13 @@
+nasdaq:
+  name: NASDAQ
+  mic: XNAS
+  acronym: NASDAQ
+  country: USA
+  country_code: US
+
+nyse:
+  name: New York Stock Exchange
+  mic: XNYS
+  acronym: NYSE
+  country: USA
+  country_code: US

test/system/trades_test.rb

@@ -9,6 +9,16 @@ class TradesTest < ApplicationSystemTestCase
     @account = accounts(:investment)
 
     visit_account_trades
+
+    Security::SynthComboboxOption.stubs(:find_in_synth).returns([
+      Security::SynthComboboxOption.new(
+        symbol: "AAPL",
+        name: "Apple Inc.",
+        logo_url: "https://logo.synthfinance.com/ticker/AAPL",
+        exchange_acronym: "NASDAQ",
+        exchange_mic: "XNAS"
+      )
+    ])
   end
 
   test "can create buy transaction" do