diff --git a/.env.example b/.env.example index 02865a5a..1169e684 100644 --- a/.env.example +++ b/.env.example @@ -11,14 +11,10 @@ # For users who have other applications listening at 3000, this allows them to set a value puma will listen to. PORT=3000 -# Exchange Rate & US Stock Pricing API -# This is used to convert between different currencies in the app. In addition, it fetches US stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com. +# Exchange Rate & Stock Pricing API +# This is used to convert between different currencies in the app. In addition, it fetches global stock prices. We use Synth, which is a Maybe product. You can sign up for a free account at synthfinance.com. SYNTH_API_KEY= -# Non-US Stock Pricing API -# This is used to fetch non-US stock prices. We use Marketstack.com for this and while they offer a free tier, it is quite limited. You'll almost certainly need their Basic plan, which is $9.99 per month. -MARKETSTACK_API_KEY= - # SMTP Configuration # This is only needed if you intend on sending emails from your Maybe instance (such as for password resets or email financial reports). # Resend.com is a good option that offers a free tier for sending emails. diff --git a/.env.local.example b/.env.local.example index cd38f220..d393f623 100644 --- a/.env.local.example +++ b/.env.local.example @@ -3,6 +3,3 @@ SELF_HOSTED=false # Enable Synth market data (careful, this will use your API credits) SYNTH_API_KEY=yourapikeyhere - -# Enable Marketstack market data (careful, this will use your API credits) -MARKETSTACK_API_KEY=yourapikeyhere diff --git a/app/controllers/account/trades_controller.rb b/app/controllers/account/trades_controller.rb index 8b5a878c..0b8bc99d 100644 --- a/app/controllers/account/trades_controller.rb +++ b/app/controllers/account/trades_controller.rb @@ -34,7 +34,10 @@ class Account::TradesController < ApplicationController end def securities - @pagy, @securities = pagy(Security.order(:name).search(params[:q]), limit: 20) + query = params[:q] + return render json: [] if query.blank? || query.length < 2 || query.length > 100 + + @securities = Security::SynthComboboxOption.find_in_synth(query) end private diff --git a/app/jobs/securities_import_job.rb b/app/jobs/securities_import_job.rb deleted file mode 100644 index 85fef904..00000000 --- a/app/jobs/securities_import_job.rb +++ /dev/null @@ -1,13 +0,0 @@ -class SecuritiesImportJob < ApplicationJob - queue_as :default - - def perform(country_code = nil) - exchanges = StockExchange.in_country(country_code) - market_stack_client = Provider::Marketstack.new(ENV["MARKETSTACK_API_KEY"]) - - exchanges.each do |exchange| - importer = Security::Importer.new(market_stack_client, exchange.mic) - importer.import - end - end -end diff --git a/app/models/account/trade_builder.rb b/app/models/account/trade_builder.rb index 4610407f..3219fb8a 100644 --- a/app/models/account/trade_builder.rb +++ b/app/models/account/trade_builder.rb @@ -31,8 +31,9 @@ class Account::TradeBuilder < Account::EntryBuilder end def security - return Security.find(ticker) if ticker.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i) - Security.find_or_create_by(ticker: ticker) + ticker_symbol, exchange_mic = ticker.split("|") + + Security.find_or_create_by(ticker: ticker_symbol, exchange_mic: exchange_mic) end def amount diff --git a/app/models/provider/marketstack.rb b/app/models/provider/marketstack.rb deleted file mode 100644 index 10e2aa5d..00000000 --- a/app/models/provider/marketstack.rb +++ /dev/null @@ -1,120 +0,0 @@ -class Provider::Marketstack - include Retryable - - def initialize(api_key) - @api_key = api_key - end - - def fetch_security_prices(ticker:, start_date:, end_date:) - prices = paginate("#{base_url}/eod", { - symbols: ticker, - date_from: start_date.to_s, - date_to: end_date.to_s - }) do |body| - body.dig("data").map do |price| - { - date: price["date"], - price: price["close"]&.to_f, - currency: "USD" - } - end - end - - SecurityPriceResponse.new( - prices: prices, - success?: true, - raw_response: prices.to_json - ) - rescue StandardError => error - SecurityPriceResponse.new( - success?: false, - error: error, - raw_response: error - ) - end - - def fetch_tickers(exchange_mic: nil) - url = exchange_mic ? "#{base_url}/tickers?exchange=#{exchange_mic}" : "#{base_url}/tickers" - tickers = paginate(url) do |body| - body.dig("data").map do |ticker| - { - name: ticker["name"], - symbol: ticker["symbol"], - exchange_mic: exchange_mic || ticker.dig("stock_exchange", "mic"), - exchange_acronym: ticker.dig("stock_exchange", "acronym"), - country_code: ticker.dig("stock_exchange", "country_code") - } - end - end - - TickerResponse.new( - tickers: tickers, - success?: true, - raw_response: tickers.to_json - ) - rescue StandardError => error - TickerResponse.new( - success?: false, - error: error, - raw_response: error - ) - end - - private - - attr_reader :api_key - - SecurityPriceResponse = Struct.new(:prices, :success?, :error, :raw_response, keyword_init: true) - TickerResponse = Struct.new(:tickers, :success?, :error, :raw_response, keyword_init: true) - - def base_url - "https://api.marketstack.com/v1" - end - - def client - @client ||= Faraday.new(url: base_url) do |faraday| - faraday.params["access_key"] = api_key - end - end - - def build_error(response) - Provider::Base::ProviderError.new(<<~ERROR) - Failed to fetch data from #{self.class} - Status: #{response.status} - Body: #{response.body.inspect} - ERROR - end - - def fetch_page(url, page, params = {}) - client.get(url) do |req| - params.each { |k, v| req.params[k.to_s] = v.to_s } - req.params["offset"] = (page - 1) * 100 # Marketstack uses offset-based pagination - req.params["limit"] = 10000 # Maximum allowed by Marketstack - end - end - - def paginate(url, params = {}) - results = [] - page = 1 - total_results = Float::INFINITY - - while results.length < total_results - response = fetch_page(url, page, params) - - if response.success? - body = JSON.parse(response.body) - page_results = yield(body) - results.concat(page_results) - - total_results = body.dig("pagination", "total") - page += 1 - else - raise build_error(response) - end - - break if results.length >= total_results - end - - results - end -end diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb index d5f28e50..6dabd941 100644 --- a/app/models/provider/synth.rb +++ b/app/models/provider/synth.rb @@ -122,6 +122,31 @@ class Provider::Synth raw_response: error end + def search_securities(query:, dataset: "limited", country_code:) + response = client.get("#{base_url}/tickers/search") do |req| + req.params["name"] = query + req.params["dataset"] = dataset + req.params["country_code"] = country_code + end + + parsed = JSON.parse(response.body) + + securities = parsed.dig("data").map do |security| + { + symbol: security.dig("symbol"), + name: security.dig("name"), + logo_url: security.dig("logo_url"), + exchange_acronym: security.dig("exchange", "acronym"), + exchange_mic: security.dig("exchange", "mic_code") + } + end + + SearchSecuritiesResponse.new \ + securities: securities, + success?: true, + raw_response: response + end + private attr_reader :api_key @@ -130,6 +155,7 @@ class Provider::Synth SecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: true ExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: true UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true + SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true def base_url "https://api.synthfinance.com" diff --git a/app/models/security.rb b/app/models/security.rb index d1a160b8..732599ce 100644 --- a/app/models/security.rb +++ b/app/models/security.rb @@ -1,4 +1,5 @@ class Security < ApplicationRecord + include Providable before_save :upcase_ticker has_many :trades, dependent: :nullify, class_name: "Account::Trade" @@ -7,25 +8,6 @@ class Security < ApplicationRecord validates :ticker, presence: true validates :ticker, uniqueness: { scope: :exchange_mic, case_sensitive: false } - scope :search, ->(query) { - return none if query.blank? || query.length < 2 - - # Clean and normalize the search terms - sanitized_query = query.split.map do |term| - cleaned_term = term.gsub(/[^a-zA-Z0-9]/, " ").strip - next if cleaned_term.blank? - cleaned_term - end.compact.join(" | ") - - return none if sanitized_query.blank? - - sanitized_query = ActiveRecord::Base.connection.quote(sanitized_query) - - where("search_vector @@ to_tsquery('simple', #{sanitized_query}) AND exchange_mic IS NOT NULL") - .select("securities.*, ts_rank_cd(search_vector, to_tsquery('simple', #{sanitized_query})) AS rank") - .reorder("rank DESC") - } - def current_price @current_price ||= Security::Price.find_price(security: self, date: Date.current) return nil if @current_price.nil? @@ -33,7 +15,7 @@ class Security < ApplicationRecord end def to_combobox_display - "#{ticker} - #{name} (#{exchange_acronym})" + "#{ticker} (#{exchange_acronym})" end diff --git a/app/models/security/importer.rb b/app/models/security/importer.rb deleted file mode 100644 index 4146fea2..00000000 --- a/app/models/security/importer.rb +++ /dev/null @@ -1,45 +0,0 @@ -class Security::Importer - def initialize(provider, stock_exchange = nil) - @provider = provider - @stock_exchange = stock_exchange - end - - def import - securities = @provider.fetch_tickers(exchange_mic: @stock_exchange)&.tickers - - # Deduplicate securities based on ticker and exchange_mic - securities_to_create = securities - .map do |security| - { - name: security[:name], - ticker: security[:symbol], - country_code: security[:country_code], - exchange_mic: security[:exchange_mic], - exchange_acronym: security[:exchange_acronym] - } - end - .compact - .uniq { |security| [ security[:ticker], security[:exchange_mic] ] } - - # First update any existing securities that only have a ticker - Security.where(exchange_mic: nil) - .where(ticker: securities_to_create.map { |s| s[:ticker] }) - .update_all( - securities_to_create.map do |security| - { - name: security[:name], - country_code: security[:country_code], - exchange_mic: security[:exchange_mic], - exchange_acronym: security[:exchange_acronym] - } - end.first - ) - - # Then create/update any remaining securities - Security.upsert_all( - securities_to_create, - unique_by: [ :ticker, :exchange_mic ], - update_only: [ :name, :country_code, :exchange_acronym ] - ) unless securities_to_create.empty? - end -end diff --git a/app/models/security/synth_combobox_option.rb b/app/models/security/synth_combobox_option.rb new file mode 100644 index 00000000..a4746ee7 --- /dev/null +++ b/app/models/security/synth_combobox_option.rb @@ -0,0 +1,20 @@ +class Security::SynthComboboxOption + include ActiveModel::Model + include Providable + + attr_accessor :symbol, :name, :logo_url, :exchange_acronym, :exchange_mic + + class << self + def find_in_synth(query) + security_prices_provider.search_securities(query:, dataset: "limited", country_code: Current.family.country).securities.map { |attrs| new(**attrs) } + end + end + + def id + "#{symbol}|#{exchange_mic}" # submitted by combobox as value + end + + def to_combobox_display + "#{symbol} - #{name} (#{exchange_acronym})" # shown in combobox input when selected + end +end diff --git a/app/views/account/trades/_form.html.erb b/app/views/account/trades/_form.html.erb index 283a7725..f8c4033a 100644 --- a/app/views/account/trades/_form.html.erb +++ b/app/views/account/trades/_form.html.erb @@ -8,7 +8,7 @@ <%= form.select :type, options_for_select([%w[Buy buy], %w[Sell sell], %w[Deposit transfer_in], %w[Withdrawal transfer_out], %w[Interest interest]], "buy"), { label: t(".type") }, { data: { "trade-form-target": "typeInput" } } %>
- <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder"), autocomplete: :list, free_text: true %> + <%= form.combobox :ticker, securities_account_trades_path(entry.account), label: t(".holding"), placeholder: t(".ticker_placeholder") %>
diff --git a/app/views/account/trades/_security.turbo_stream.erb b/app/views/account/trades/_security.turbo_stream.erb new file mode 100644 index 00000000..f270b028 --- /dev/null +++ b/app/views/account/trades/_security.turbo_stream.erb @@ -0,0 +1,11 @@ +
+ <%= image_tag(security.logo_url, class: "rounded-full h-8 w-8 inline-block mr-2" ) %> +
+ + <%= security.name.presence || security.symbol %> + + + <%= "#{security.symbol} (#{security.exchange_acronym})" %> + +
+
\ No newline at end of file diff --git a/app/views/account/trades/_tickers.turbo_stream.erb b/app/views/account/trades/_tickers.turbo_stream.erb deleted file mode 100644 index 56436ab8..00000000 --- a/app/views/account/trades/_tickers.turbo_stream.erb +++ /dev/null @@ -1,7 +0,0 @@ -
- <%= image_tag("https://logo.synthfinance.com/ticker/#{tickers&.ticker}", class: "rounded-full h-8 w-8 inline-block mr-2") %> -
- <%= tickers&.name.presence || tickers&.ticker %> - <%= "#{tickers&.ticker} (#{tickers&.exchange_acronym})" %> -
-
diff --git a/app/views/account/trades/securities.turbo_stream.erb b/app/views/account/trades/securities.turbo_stream.erb index 7ed8e90e..a3128c77 100644 --- a/app/views/account/trades/securities.turbo_stream.erb +++ b/app/views/account/trades/securities.turbo_stream.erb @@ -1,3 +1,2 @@ <%= async_combobox_options @securities, - render_in: { partial: "account/trades/tickers" }, - next_page: @pagy.next %> \ No newline at end of file + render_in: { partial: "account/trades/security" } %> \ No newline at end of file diff --git a/db/migrate/20241029234028_remove_search_vector.rb b/db/migrate/20241029234028_remove_search_vector.rb new file mode 100644 index 00000000..99d94602 --- /dev/null +++ b/db/migrate/20241029234028_remove_search_vector.rb @@ -0,0 +1,5 @@ +class RemoveSearchVector < ActiveRecord::Migration[7.2] + def change + remove_column :securities, :search_vector + end +end diff --git a/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb new file mode 100644 index 00000000..9a1f1012 --- /dev/null +++ b/db/migrate/20241030121302_fix_not_null_stock_exchange_data.rb @@ -0,0 +1,10 @@ +class FixNotNullStockExchangeData < ActiveRecord::Migration[7.2] + def change + change_column_null :stock_exchanges, :currency_code, true + change_column_null :stock_exchanges, :currency_symbol, true + change_column_null :stock_exchanges, :currency_name, true + change_column_null :stock_exchanges, :city, true + change_column_null :stock_exchanges, :timezone_name, true + change_column_null :stock_exchanges, :timezone_abbr, true + end +end diff --git a/db/schema.rb b/db/schema.rb index 507fd0f9..d9d597d7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do +ActiveRecord::Schema[7.2].define(version: 2024_10_30_121302) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -119,7 +119,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do t.boolean "is_active", default: true, null: false t.date "last_sync_date" t.uuid "institution_id" - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.string "mode" t.index ["accountable_id", "accountable_type"], name: "index_accounts_on_accountable_id_and_accountable_type" @@ -481,9 +481,7 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do t.string "country_code" t.string "exchange_mic" t.string "exchange_acronym" - t.virtual "search_vector", type: :tsvector, as: "(setweight(to_tsvector('simple'::regconfig, (COALESCE(ticker, ''::character varying))::text), 'B'::\"char\") || to_tsvector('simple'::regconfig, (COALESCE(name, ''::character varying))::text))", stored: true t.index ["country_code"], name: "index_securities_on_country_code" - t.index ["search_vector"], name: "index_securities_on_search_vector", using: :gin t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true end @@ -524,14 +522,14 @@ ActiveRecord::Schema[7.2].define(version: 2024_10_29_184115) do t.string "mic", null: false t.string "country", null: false t.string "country_code", null: false - t.string "city", null: false + t.string "city" t.string "website" - t.string "timezone_name", null: false - t.string "timezone_abbr", null: false + t.string "timezone_name" + t.string "timezone_abbr" t.string "timezone_abbr_dst" - t.string "currency_code", null: false - t.string "currency_symbol", null: false - t.string "currency_name", null: false + t.string "currency_code" + t.string "currency_symbol" + t.string "currency_name" t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["country"], name: "index_stock_exchanges_on_country" diff --git a/test/controllers/account/trades_controller_test.rb b/test/controllers/account/trades_controller_test.rb index 46f53046..b9c72852 100644 --- a/test/controllers/account/trades_controller_test.rb +++ b/test/controllers/account/trades_controller_test.rb @@ -97,7 +97,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest account_entry: { type: "buy", date: Date.current, - ticker: "NVDA", + ticker: "NVDA (NASDAQ)", qty: 10, price: 10 } @@ -118,7 +118,7 @@ class Account::TradesControllerTest < ActionDispatch::IntegrationTest post account_trades_url(@entry.account), params: { account_entry: { type: "sell", - ticker: "AAPL", + ticker: "AAPL (NYSE)", date: Date.current, currency: "USD", qty: 10, diff --git a/test/fixtures/stock_exchanges.yml b/test/fixtures/stock_exchanges.yml new file mode 100644 index 00000000..872e5b27 --- /dev/null +++ b/test/fixtures/stock_exchanges.yml @@ -0,0 +1,13 @@ +nasdaq: + name: NASDAQ + mic: XNAS + acronym: NASDAQ + country: USA + country_code: US + +nyse: + name: New York Stock Exchange + mic: XNYS + acronym: NYSE + country: USA + country_code: US diff --git a/test/system/trades_test.rb b/test/system/trades_test.rb index c79c3507..c9539276 100644 --- a/test/system/trades_test.rb +++ b/test/system/trades_test.rb @@ -9,6 +9,16 @@ class TradesTest < ApplicationSystemTestCase @account = accounts(:investment) visit_account_trades + + Security::SynthComboboxOption.stubs(:find_in_synth).returns([ + Security::SynthComboboxOption.new( + symbol: "AAPL", + name: "Apple Inc.", + logo_url: "https://logo.synthfinance.com/ticker/AAPL", + exchange_acronym: "NASDAQ", + exchange_mic: "XNAS" + ) + ]) end test "can create buy transaction" do