Docker/Maybe
Docker/Maybe
1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-05 13:35:21 +02:00
Code Issues Releases Activity Actions

Data provider simplification, tests, and documentation (#1997)

Browse source 
* Ignore env.test from source control

* Simplification of providers interface

* Synth tests

* Update money to use new find rates method

* Remove unused issues code

* Additional issue feature removals

* Update price data fetching and tests

* Update documentation for providers

* Security test fixes

* Fix self host test

* Update synth usage data access

* Remove AI pr schema changes
This commit is contained in:
Zach Gollwitzer 2025-03-17 11:54:53 -04:00 • committed by GitHub
parent dd75cadebc
commit f65b93a352
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
95 changed files with 2014 additions and 1638 deletions
Download patch file Download diff file Expand all files Collapse all files

20
app/controllers/issue/exchange_rate_provider_missings_controller.rb
View file

@ -1,20 +0,0 @@
class Issue::ExchangeRateProviderMissingsController < ApplicationController
before_action :set_issue, only: :update
def update
Setting.synth_api_key = exchange_rate_params[:synth_api_key]
account = @issue.issuable
account.sync_later
redirect_back_or_to account
end
private
def set_issue
@issue = Current.family.issues.find(params[:id])
end
def exchange_rate_params
params.require(:issue_exchange_rate_provider_missing).permit(:synth_api_key)
end
end

13
app/controllers/issues_controller.rb
View file

@ -1,13 +0,0 @@
class IssuesController < ApplicationController
before_action :set_issue, only: :show
def show
render template: "#{@issue.class.name.underscore.pluralize}/show", layout: "issues"
end
private
def set_issue
@issue = Current.family.issues.find(params[:id])
end
end

8
app/controllers/securities_controller.rb
View file

@ -1,8 +1,8 @@
class SecuritiesController < ApplicationController
def index
@securities = Security.search_provider({
search: params[:q],
country: params[:country_code] == "US" ? "US" : nil
})
@securities = Security.search_provider(
params[:q],
country_code: params[:country_code] == "US" ? "US" : nil
)
end
end

2
app/controllers/settings/hostings_controller.rb
View file

@ -5,7 +5,7 @@ class Settings::HostingsController < ApplicationController
before_action :ensure_admin, only: :clear_cache
def show
@synth_usage = Current.family.synth_usage
@synth_usage = Providers.synth&.usage
end
def update

3
app/jobs/enrich_transaction_batch_job.rb
View file

@ -2,7 +2,6 @@ class EnrichTransactionBatchJob < ApplicationJob
queue_as :latency_high
def perform(account, batch_size = 100, offset = 0)
enricher = Account::DataEnricher.new(account)
enricher.enrich_transaction_batch(batch_size, offset)
account.enrich_transaction_batch(batch_size, offset)
end
end

4
app/models/account.rb
View file

@ -1,5 +1,5 @@
class Account < ApplicationRecord
include Syncable, Monetizable, Issuable, Chartable, Enrichable, Linkable, Convertible
include Syncable, Monetizable, Chartable, Enrichable, Linkable, Convertible
validates :name, :balance, :currency, presence: true
@ -13,7 +13,6 @@ class Account < ApplicationRecord
has_many :trades, through: :entries, source: :entryable, source_type: "Account::Trade"
has_many :holdings, dependent: :destroy, class_name: "Account::Holding"
has_many :balances, dependent: :destroy
has_many :issues, as: :issuable, dependent: :destroy
monetize :balance, :cash_balance
@ -88,7 +87,6 @@ class Account < ApplicationRecord
def post_sync
broadcast_remove_to(family, target: "syncing-notice")
resolve_stale_issues
accountable.post_sync
end

5
app/models/account/convertible.rb
View file

@ -7,14 +7,13 @@ module Account::Convertible
return
end
rates = ExchangeRate.find_rates(
affected_row_count = ExchangeRate.sync_provider_rates(
from: currency,
to: target_currency,
start_date: start_date,
cache: true # caches from provider to DB
)
Rails.logger.info("Synced #{rates.count} exchange rates for account #{id}")
Rails.logger.info("Synced #{affected_row_count} exchange rates for account #{id}")
end
private

66
app/models/account/data_enricher.rb
View file

@ -1,66 +0,0 @@
class Account::DataEnricher
attr_reader :account
def initialize(account)
@account = account
end
def run
total_unenriched = account.entries.account_transactions
.joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
.where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
.count
if total_unenriched > 0
batch_size = 50
batches = (total_unenriched.to_f / batch_size).ceil
batches.times do |batch|
EnrichTransactionBatchJob.perform_later(account, batch_size, batch * batch_size)
end
end
end
def enrich_transaction_batch(batch_size = 50, offset = 0)
candidates = account.entries.account_transactions
.includes(entryable: [ :merchant, :category ])
.joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
.where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
.offset(offset)
.limit(batch_size)
Rails.logger.info("Enriching batch of #{candidates.count} transactions for account #{account.id} (offset: #{offset})")
merchants = {}
candidates.each do |entry|
begin
info = entry.fetch_enrichment_info
next unless info.present?
if info.name.present?
merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
if info.icon_url.present?
merchant.icon_url = info.icon_url
end
end
entryable_attributes = { id: entry.entryable_id }
entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
Account.transaction do
merchant.save! if merchant.present?
entry.update!(
enriched_at: Time.current,
enriched_name: info.name,
entryable_attributes: entryable_attributes
)
end
rescue => e
Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
end
end
end
end

61
app/models/account/enrichable.rb
View file

@ -2,11 +2,70 @@ module Account::Enrichable
extend ActiveSupport::Concern
def enrich_data
DataEnricher.new(self).run
total_unenriched = entries.account_transactions
.joins("JOIN account_transactions at ON at.id = account_entries.entryable_id AND account_entries.entryable_type = 'Account::Transaction'")
.where("account_entries.enriched_at IS NULL OR at.merchant_id IS NULL OR at.category_id IS NULL")
.count
if total_unenriched > 0
batch_size = 50
batches = (total_unenriched.to_f / batch_size).ceil
batches.times do |batch|
EnrichTransactionBatchJob.perform_now(self, batch_size, batch * batch_size)
# EnrichTransactionBatchJob.perform_later(self, batch_size, batch * batch_size)
end
end
end
def enrich_transaction_batch(batch_size = 50, offset = 0)
transactions_batch = enrichable_transactions.offset(offset).limit(batch_size)
Rails.logger.info("Enriching batch of #{transactions_batch.count} transactions for account #{id} (offset: #{offset})")
merchants = {}
transactions_batch.each do |transaction|
begin
info = transaction.fetch_enrichment_info
next unless info.present?
if info.name.present?
merchant = merchants[info.name] ||= family.merchants.find_or_create_by(name: info.name)
if info.icon_url.present?
merchant.icon_url = info.icon_url
end
end
Account.transaction do
merchant.save! if merchant.present?
transaction.update!(merchant: merchant) if merchant.present? && transaction.merchant_id.nil?
transaction.entry.update!(
enriched_at: Time.current,
enriched_name: info.name,
)
end
rescue => e
Rails.logger.warn("Error enriching transaction #{transaction.id}: #{e.message}")
end
end
end
private
def enrichable?
family.data_enrichment_enabled? || (linked? && Rails.application.config.app_mode.hosted?)
end
def enrichable_transactions
transactions.active
.includes(:merchant, :category)
.where(
"account_entries.enriched_at IS NULL",
"OR merchant_id IS NULL",
"OR category_id IS NULL"
)
end
end

2
app/models/account/entry.rb
View file

@ -1,5 +1,5 @@
class Account::Entry < ApplicationRecord
include Monetizable, Provided
include Monetizable
monetize :amount

11
app/models/account/entry/provided.rb
View file

@ -1,11 +0,0 @@
module Account::Entry::Provided
extend ActiveSupport::Concern
include Synthable
def fetch_enrichment_info
return nil unless synth_client.present?
synth_client.enrich_transaction(name).info
end
end

13
app/models/account/holding/portfolio_cache.rb
View file

@ -79,12 +79,11 @@ class Account::Holding::PortfolioCache
securities.each do |security|
Rails.logger.info "Loading security: ID=#{security.id} Ticker=#{security.ticker}"
# Highest priority prices
db_or_provider_prices = Security::Price.find_prices(
security: security,
start_date: account.start_date,
end_date: Date.current
).map do |price|
# Load prices from provider to DB
security.sync_provider_prices(start_date: account.start_date)
# High priority prices from DB (synced from provider)
db_prices = security.prices.where(date: account.start_date..Date.current).map do |price|
PriceWithPriority.new(
price: price,
priority: 1
@ -125,7 +124,7 @@ class Account::Holding::PortfolioCache
@security_cache[security.id] = {
security: security,
prices: db_or_provider_prices + trade_prices + holding_prices
prices: db_prices + trade_prices + holding_prices
}
end
end

2
app/models/account/transaction.rb
View file

@ -1,5 +1,5 @@
class Account::Transaction < ApplicationRecord
include Account::Entryable, Transferable
include Account::Entryable, Transferable, Provided
belongs_to :category, optional: true
belongs_to :merchant, optional: true

15
app/models/account/transaction/provided.rb Normal file
View file

@ -0,0 +1,15 @@
module Account::Transaction::Provided
extend ActiveSupport::Concern
def fetch_enrichment_info
return nil unless Providers.synth # Only Synth can provide this data
response = Providers.synth.enrich_transaction(
entry.name,
amount: entry.amount,
date: entry.date
)
response.data
end
end

52
app/models/concerns/issuable.rb
View file

@ -1,52 +0,0 @@
module Issuable
extend ActiveSupport::Concern
included do
has_many :issues, dependent: :destroy, as: :issuable
end
def has_issues?
issues.active.any?
end
def resolve_stale_issues
issues.active.each do |issue|
issue.resolve! if issue.stale?
end
end
def observe_unknown_issue(error)
observe_issue(
Issue::Unknown.new(data: { error: error.message })
)
end
def observe_missing_exchange_rates(from:, to:, dates:)
observe_issue(
Issue::ExchangeRatesMissing.new(data: { from_currency: from, to_currency: to, dates: dates })
)
end
def observe_missing_exchange_rate_provider
observe_issue(
Issue::ExchangeRateProviderMissing.new
)
end
def highest_priority_issue
issues.active.ordered.first
end
private
def observe_issue(new_issue)
existing_issue = issues.find_by(type: new_issue.type, resolved_at: nil)
if existing_issue
existing_issue.update!(last_observed_at: Time.current, data: new_issue.data)
else
new_issue.issuable = self
new_issue.save!
end
end
end

37
app/models/concerns/synthable.rb
View file

@ -1,37 +0,0 @@
module Synthable
extend ActiveSupport::Concern
class_methods do
def synth_usage
synth_client&.usage
end
def synth_overage?
synth_usage&.usage&.utilization.to_i >= 100
end
def synth_healthy?
synth_client&.healthy?
end
def synth_client
api_key = ENV.fetch("SYNTH_API_KEY", Setting.synth_api_key)
return nil unless api_key.present?
Provider::Synth.new(api_key)
end
end
def synth_client
self.class.synth_client
end
def synth_usage
self.class.synth_usage
end
def synth_overage?
self.class.synth_overage?
end
end

24
app/models/exchange_rate.rb
View file

@ -2,27 +2,5 @@ class ExchangeRate < ApplicationRecord
include Provided
validates :from_currency, :to_currency, :date, :rate, presence: true
class << self
def find_rate(from:, to:, date:, cache: true)
result = find_by \
from_currency: from,
to_currency: to,
date: date
result || fetch_rate_from_provider(from:, to:, date:, cache:)
end
def find_rates(from:, to:, start_date:, end_date: Date.current, cache: true)
rates = self.where(from_currency: from, to_currency: to, date: start_date..end_date).to_a
all_dates = (start_date..end_date).to_a
existing_dates = rates.map(&:date)
missing_dates = all_dates - existing_dates
if missing_dates.any?
rates += fetch_rates_from_provider(from:, to:, start_date: missing_dates.first, end_date: missing_dates.last, cache:)
end
rates
end
end
validates :date, uniqueness: { scope: %i[from_currency to_currency] }
end

15
app/models/exchange_rate/provideable.rb Normal file
View file

@ -0,0 +1,15 @@
# Defines the interface an exchange rate provider must implement
module ExchangeRate::Provideable
extend ActiveSupport::Concern
FetchRateData = Data.define(:rate)
FetchRatesData = Data.define(:rates)
def fetch_exchange_rate(from:, to:, date:)
raise NotImplementedError, "Subclasses must implement #fetch_exchange_rate"
end
def fetch_exchange_rates(from:, to:, start_date:, end_date:)
raise NotImplementedError, "Subclasses must implement #fetch_exchange_rates"
end
end

75
app/models/exchange_rate/provided.rb
View file

@ -1,63 +1,44 @@
module ExchangeRate::Provided
extend ActiveSupport::Concern
include Synthable
class_methods do
def provider
synth_client
Providers.synth
end
private
def fetch_rates_from_provider(from:, to:, start_date:, end_date: Date.current, cache: false)
return [] unless provider.present?
def find_or_fetch_rate(from:, to:, date: Date.current, cache: true)
rate = find_by(from_currency: from, to_currency: to, date: date)
return rate if rate.present?
response = provider.fetch_exchange_rates \
from: from,
to: to,
start_date: start_date,
end_date: end_date
return nil unless provider.present? # No provider configured (some self-hosted apps)
if response.success?
response.rates.map do |exchange_rate|
rate = ExchangeRate.new \
from_currency: from,
to_currency: to,
date: exchange_rate.dig(:date).to_date,
rate: exchange_rate.dig(:rate)
response = provider.fetch_exchange_rate(from: from, to: to, date: date)
rate.save! if cache
rate
rescue ActiveRecord::RecordNotUnique
next
end
else
[]
end
return nil unless response.success? # Provider error
rate = response.data.rate
rate.save! if cache
rate
end
def sync_provider_rates(from:, to:, start_date:, end_date: Date.current)
unless provider.present?
Rails.logger.warn("No provider configured for ExchangeRate.sync_provider_rates")
return 0
end
def fetch_rate_from_provider(from:, to:, date:, cache: false)
return nil unless provider.present?
fetched_rates = provider.fetch_exchange_rates(from: from, to: to, start_date: start_date, end_date: end_date)
response = provider.fetch_exchange_rate \
from: from,
to: to,
date: date
if response.success?
rate = ExchangeRate.new \
from_currency: from,
to_currency: to,
rate: response.rate,
date: date
if cache
rate.save! rescue ActiveRecord::RecordNotUnique
end
rate
else
nil
end
unless fetched_rates.success?
Rails.logger.error("Provider error for ExchangeRate.sync_provider_rates: #{fetched_rates.error}")
return 0
end
rates_data = fetched_rates.data.rates.map do |rate|
rate.attributes.slice("from_currency", "to_currency", "date", "rate")
end
ExchangeRate.upsert_all(rates_data, unique_by: %i[from_currency to_currency date])
end
end
end

7
app/models/family.rb
View file

@ -1,5 +1,5 @@
class Family < ApplicationRecord
include Synthable, Plaidable, Syncable, AutoTransferMatchable
include Syncable, AutoTransferMatchable
DATE_FORMATS = [
[ "MM-DD-YYYY", "%m-%d-%Y" ],
@ -19,7 +19,6 @@ class Family < ApplicationRecord
has_many :invitations, dependent: :destroy
has_many :imports, dependent: :destroy
has_many :issues, through: :accounts
has_many :entries, through: :accounts
has_many :transactions, through: :accounts
@ -75,9 +74,9 @@ class Family < ApplicationRecord
def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil)
provider = if region.to_sym == :eu
self.class.plaid_eu_provider
Providers.plaid_eu
else
self.class.plaid_us_provider
Providers.plaid_us
end
# early return when no provider

11
app/models/financial_assistant.rb Normal file
View file

@ -0,0 +1,11 @@
class FinancialAssistant
include Provided
def initialize(chat)
@chat = chat
end
def query(prompt, model_key: "gpt-4o")
llm_provider = self.class.llm_provider_for(model_key)
end
end

13
app/models/financial_assistant/provided.rb Normal file
View file

@ -0,0 +1,13 @@
module FinancialAssistant::Provided
extend ActiveSupport::Concern
# Placeholder for AI chat PR
def llm_provider_for(model_key)
case model_key
when "gpt-4o"
Providers.openai
else
raise "Unknown LLM model key: #{model_key}"
end
end
end

35
app/models/issue.rb
View file

@ -1,35 +0,0 @@
class Issue < ApplicationRecord
belongs_to :issuable, polymorphic: true
after_initialize :set_default_severity
enum :severity, { critical: 1, error: 2, warning: 3, info: 4 }
validates :severity, presence: true
scope :active, -> { where(resolved_at: nil) }
scope :ordered, -> { order(:severity) }
def title
model_name.human
end
# The conditions that must be met for an issue to be fixed
def stale?
raise NotImplementedError, "#{self.class} must implement #{__method__}"
end
def resolve!
update!(resolved_at: Time.current)
end
def default_severity
:warning
end
private
def set_default_severity
self.severity ||= default_severity
end
end

9
app/models/issue/exchange_rate_provider_missing.rb
View file

@ -1,9 +0,0 @@
class Issue::ExchangeRateProviderMissing < Issue
def default_severity
:error
end
def stale?
ExchangeRate.provider_healthy?
end
end

15
app/models/issue/exchange_rates_missing.rb
View file

@ -1,15 +0,0 @@
class Issue::ExchangeRatesMissing < Issue
store_accessor :data, :from_currency, :to_currency, :dates
validates :from_currency, :to_currency, :dates, presence: true
def stale?
if dates.length == 1
ExchangeRate.find_rate(from: from_currency, to: to_currency, date: dates.first).present?
else
sorted_dates = dates.sort
rates = ExchangeRate.find_rates(from: from_currency, to: to_currency, start_date: sorted_dates.first, end_date: sorted_dates.last)
rates.length == dates.length
end
end
end

11
app/models/issue/unknown.rb
View file

@ -1,11 +0,0 @@
class Issue::Unknown < Issue
def default_severity
:warning
end
# Unknown issues are always stale because we only want to show them
# to the user once. If the same error occurs again, we'll create a new instance.
def stale?
true
end
end

2
app/models/plaid_account.rb
View file

@ -1,6 +1,4 @@
class PlaidAccount < ApplicationRecord
include Plaidable
TYPE_MAPPING = {
"depository" => Depository,
"credit" => CreditCard,

2
app/models/plaid_item.rb
View file

@ -1,5 +1,5 @@
class PlaidItem < ApplicationRecord
include Plaidable, Syncable
include Provided, Syncable
enum :plaid_region, { us: "us", eu: "eu" }
enum :status, { good: "good", requires_update: "requires_update" }, default: :good

6
app/models/concerns/plaidable.rb → app/models/plaid_item/provided.rb
View file

@ -1,13 +1,13 @@
module Plaidable
module PlaidItem::Provided
extend ActiveSupport::Concern
class_methods do
def plaid_us_provider
Provider::Plaid.new(Rails.application.config.plaid, region: :us) if Rails.application.config.plaid
Providers.plaid_us
end
def plaid_eu_provider
Provider::Plaid.new(Rails.application.config.plaid_eu, region: :eu) if Rails.application.config.plaid_eu
Providers.plaid_eu
end
def plaid_provider_for_region(region)

35
app/models/provider.rb Normal file
View file

@ -0,0 +1,35 @@
class Provider
include Retryable
ProviderError = Class.new(StandardError)
ProviderResponse = Data.define(:success?, :data, :error)
private
PaginatedData = Data.define(:paginated, :first_page, :total_pages)
UsageData = Data.define(:used, :limit, :utilization, :plan)
# Subclasses can specify errors that can be retried
def retryable_errors
[]
end
def provider_response(retries: nil, &block)
data = if retries
retrying(retryable_errors, max_retries: retries) { yield }
else
yield
end
ProviderResponse.new(
success?: true,
data: data,
error: nil,
)
rescue StandardError => error
ProviderResponse.new(
success?: false,
data: nil,
error: error,
)
end
end

18
app/models/provider/base.rb
View file

@ -1,18 +0,0 @@
class Provider::Base
ProviderError = Class.new(StandardError)
TRANSIENT_NETWORK_ERRORS = [
Faraday::TimeoutError,
Faraday::ConnectionFailed,
Faraday::SSLError,
Faraday::ClientError,
Faraday::ServerError
]
class << self
def known_transient_errors
TRANSIENT_NETWORK_ERRORS + [ ProviderError ]
end
end
end

376
app/models/provider/synth.rb
View file

@ -1,217 +1,226 @@
class Provider::Synth
include Retryable
class Provider::Synth < Provider
include ExchangeRate::Provideable
include Security::Provideable
def initialize(api_key)
@api_key = api_key
end
def healthy?
response = client.get("#{base_url}/user")
JSON.parse(response.body).dig("id").present?
provider_response do
response = client.get("#{base_url}/user")
JSON.parse(response.body).dig("id").present?
end
end
def usage
response = client.get("#{base_url}/user")
provider_response do
response = client.get("#{base_url}/user")
if response.status == 401
return UsageResponse.new(
success?: false,
error: "Unauthorized: Invalid API key",
raw_response: response
parsed = JSON.parse(response.body)
remaining = parsed.dig("api_calls_remaining")
limit = parsed.dig("api_limit")
used = limit - remaining
UsageData.new(
used: used,
limit: limit,
utilization: used.to_f / limit * 100,
plan: parsed.dig("plan"),
)
end
parsed = JSON.parse(response.body)
remaining = parsed.dig("api_calls_remaining")
limit = parsed.dig("api_limit")
used = limit - remaining
UsageResponse.new(
used: used,
limit: limit,
utilization: used.to_f / limit * 100,
plan: parsed.dig("plan"),
success?: true,
raw_response: response
)
rescue StandardError => error
UsageResponse.new(
success?: false,
error: error,
raw_response: error
)
end
def fetch_security_prices(ticker:, start_date:, end_date:, mic_code: nil)
params = {
start_date: start_date,
end_date: end_date
}
params[:mic_code] = mic_code if mic_code.present?
prices = paginate(
"#{base_url}/tickers/#{ticker}/open-close",
params
) do |body|
body.dig("prices").map do |price|
{
date: price.dig("date"),
price: price.dig("close")&.to_f || price.dig("open")&.to_f,
currency: body.dig("currency") || "USD"
}
end
end
SecurityPriceResponse.new \
prices: prices,
success?: true,
raw_response: prices.to_json
rescue StandardError => error
SecurityPriceResponse.new \
success?: false,
error: error,
raw_response: error
end
# ================================
# Exchange Rates
# ================================
def fetch_exchange_rate(from:, to:, date:)
retrying Provider::Base.known_transient_errors do |on_last_attempt|
provider_response retries: 2 do
response = client.get("#{base_url}/rates/historical") do |req|
req.params["date"] = date.to_s
req.params["from"] = from
req.params["to"] = to
end
if response.success?
ExchangeRateResponse.new \
rate: JSON.parse(response.body).dig("data", "rates", to),
success?: true,
raw_response: response
else
if on_last_attempt
ExchangeRateResponse.new \
success?: false,
error: build_error(response),
raw_response: response
else
raise build_error(response)
end
end
rates = JSON.parse(response.body).dig("data", "rates")
ExchangeRate::Provideable::FetchRateData.new(
rate: ExchangeRate.new(
from_currency: from,
to_currency: to,
date: date,
rate: rates.dig(to)
)
)
end
end
def fetch_exchange_rates(from:, to:, start_date:, end_date:)
exchange_rates = paginate(
"#{base_url}/rates/historical-range",
from: from,
to: to,
date_start: start_date.to_s,
date_end: end_date.to_s
) do |body|
body.dig("data").map do |exchange_rate|
{
date: exchange_rate.dig("date"),
rate: exchange_rate.dig("rates", to)
}
provider_response retries: 1 do
data = paginate(
"#{base_url}/rates/historical-range",
from: from,
to: to,
date_start: start_date.to_s,
date_end: end_date.to_s
) do |body|
body.dig("data")
end
end
ExchangeRatesResponse.new \
rates: exchange_rates,
success?: true,
raw_response: exchange_rates.to_json
rescue StandardError => error
ExchangeRatesResponse.new \
success?: false,
error: error,
raw_response: error
ExchangeRate::Provideable::FetchRatesData.new(
rates: data.paginated.map do |exchange_rate|
ExchangeRate.new(
from_currency: from,
to_currency: to,
date: exchange_rate.dig("date"),
rate: exchange_rate.dig("rates", to)
)
end
)
end
end
def search_securities(query:, dataset: "limited", country_code: nil, exchange_operating_mic: nil)
response = client.get("#{base_url}/tickers/search") do |req|
req.params["name"] = query
req.params["dataset"] = dataset
req.params["country_code"] = country_code if country_code.present?
req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present?
req.params["limit"] = 25
# ================================
# Securities
# ================================
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
provider_response do
response = client.get("#{base_url}/tickers/search") do |req|
req.params["name"] = symbol
req.params["dataset"] = "limited"
req.params["country_code"] = country_code if country_code.present?
req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present?
req.params["limit"] = 25
end
parsed = JSON.parse(response.body)
Security::Provideable::Search.new(
securities: parsed.dig("data").map do |security|
Security.new(
ticker: security.dig("symbol"),
name: security.dig("name"),
logo_url: security.dig("logo_url"),
exchange_acronym: security.dig("exchange", "acronym"),
exchange_mic: security.dig("exchange", "mic_code"),
exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
country_code: security.dig("exchange", "country_code")
)
end
)
end
end
parsed = JSON.parse(response.body)
def fetch_security_info(security)
provider_response do
response = client.get("#{base_url}/tickers/#{security.ticker}") do |req|
req.params["mic_code"] = security.exchange_mic if security.exchange_mic.present?
req.params["operating_mic"] = security.exchange_operating_mic if security.exchange_operating_mic.present?
end
securities = parsed.dig("data").map do |security|
{
ticker: security.dig("symbol"),
name: security.dig("name"),
logo_url: security.dig("logo_url"),
exchange_acronym: security.dig("exchange", "acronym"),
exchange_mic: security.dig("exchange", "mic_code"),
exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
country_code: security.dig("exchange", "country_code")
data = JSON.parse(response.body).dig("data")
Security::Provideable::SecurityInfo.new(
ticker: security.ticker,
name: data.dig("name"),
links: data.dig("links"),
logo_url: data.dig("logo_url"),
description: data.dig("description"),
kind: data.dig("kind")
)
end
end
def fetch_security_price(security, date:)
provider_response do
historical_data = fetch_security_prices(security, start_date: date, end_date: date)
raise ProviderError, "No prices found for security #{security.ticker} on date #{date}" if historical_data.data.prices.empty?
Security::Provideable::PriceData.new(
price: historical_data.data.prices.first
)
end
end
def fetch_security_prices(security, start_date:, end_date:)
provider_response retries: 1 do
params = {
start_date: start_date,
end_date: end_date
}
end
SearchSecuritiesResponse.new \
securities: securities,
success?: true,
raw_response: response
params[:operating_mic_code] = security.exchange_operating_mic if security.exchange_operating_mic.present?
data = paginate(
"#{base_url}/tickers/#{security.ticker}/open-close",
params
) do |body|
body.dig("prices")
end
currency = data.first_page.dig("currency")
country_code = data.first_page.dig("exchange", "country_code")
exchange_mic = data.first_page.dig("exchange", "mic_code")
exchange_operating_mic = data.first_page.dig("exchange", "operating_mic_code")
Security::Provideable::PricesData.new(
prices: data.paginated.map do |price|
Security::Price.new(
security: security,
date: price.dig("date"),
price: price.dig("close") || price.dig("open"),
currency: currency
)
end
)
end
end
def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil)
response = client.get("#{base_url}/tickers/#{ticker}") do |req|
req.params["mic_code"] = mic_code if mic_code.present?
req.params["operating_mic"] = operating_mic if operating_mic.present?
end
parsed = JSON.parse(response.body)
SecurityInfoResponse.new \
info: parsed.dig("data"),
success?: true,
raw_response: response
end
# ================================
# Transactions
# ================================
def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
params = {
description: description,
amount: amount,
date: date,
city: city,
state: state,
country: country
}.compact
provider_response do
params = {
description: description,
amount: amount,
date: date,
city: city,
state: state,
country: country
}.compact
response = client.get("#{base_url}/enrich", params)
response = client.get("#{base_url}/enrich", params)
parsed = JSON.parse(response.body)
parsed = JSON.parse(response.body)
EnrichTransactionResponse.new \
info: EnrichTransactionInfo.new(
TransactionEnrichmentData.new(
name: parsed.dig("merchant"),
icon_url: parsed.dig("icon"),
category: parsed.dig("category")
),
success?: true,
raw_response: response
rescue StandardError => error
EnrichTransactionResponse.new \
success?: false,
error: error,
raw_response: error
)
end
end
private
attr_reader :api_key
ExchangeRateResponse = Struct.new :rate, :success?, :error, :raw_response, keyword_init: true
SecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: true
ExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: true
UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
TransactionEnrichmentData = Data.define(:name, :icon_url, :category)
def retryable_errors
[
Faraday::TimeoutError,
Faraday::ConnectionFailed,
Faraday::SSLError,
Faraday::ClientError,
Faraday::ServerError
]
end
def base_url
ENV["SYNTH_URL"] || "https://api.synthfinance.com"
@ -227,26 +236,15 @@ class Provider::Synth
def client
@client ||= Faraday.new(url: base_url) do |faraday|
faraday.response :raise_error
faraday.headers["Authorization"] = "Bearer #{api_key}"
faraday.headers["X-Source"] = app_name
faraday.headers["X-Source-Type"] = app_type
end
end
def build_error(response)
Provider::Base::ProviderError.new(<<~ERROR)
Failed to fetch data from #{self.class}
Status: #{response.status}
Body: #{response.body.inspect}
ERROR
end
def fetch_page(url, page, params = {})
client.get(url) do |req|
req.headers["Authorization"] = "Bearer #{api_key}"
params.each { |k, v| req.params[k.to_s] = v.to_s }
req.params["page"] = page
end
client.get(url, params.merge(page: page))
end
def paginate(url, params = {})
@ -254,24 +252,26 @@ class Provider::Synth
page = 1
current_page = 0
total_pages = 1
first_page = nil
while current_page < total_pages
response = fetch_page(url, page, params)
if response.success?
body = JSON.parse(response.body)
page_results = yield(body)
results.concat(page_results)
body = JSON.parse(response.body)
first_page = body unless first_page
page_results = yield(body)
results.concat(page_results)
current_page = body.dig("paging", "current_page")
total_pages = body.dig("paging", "total_pages")
current_page = body.dig("paging", "current_page")
total_pages = body.dig("paging", "total_pages")
page += 1
else
raise build_error(response)
end
page += 1
end
results
PaginatedData.new(
paginated: results,
first_page: first_page,
total_pages: total_pages
)
end
end

35
app/models/providers.rb Normal file
View file

@ -0,0 +1,35 @@
module Providers
module_function
def synth
api_key = ENV.fetch("SYNTH_API_KEY", Setting.synth_api_key)
return nil unless api_key.present?
Provider::Synth.new(api_key)
end
def plaid_us
config = Rails.application.config.plaid
return nil unless config.present?
Provider::Plaid.new(config, region: :us)
end
def plaid_eu
config = Rails.application.config.plaid_eu
return nil unless config.present?
Provider::Plaid.new(config, region: :eu)
end
def github
Provider::Github.new
end
def openai
# TODO: Placeholder for AI chat PR
end
end

2
app/models/security.rb
View file

@ -10,7 +10,7 @@ class Security < ApplicationRecord
validates :ticker, uniqueness: { scope: :exchange_operating_mic, case_sensitive: false }
def current_price
@current_price ||= Security::Price.find_price(security: self, date: Date.current)
@current_price ||= find_or_fetch_price
return nil if @current_price.nil?
Money.new(@current_price.price, @current_price.currency)
end

31
app/models/security/price.rb
View file

@ -1,33 +1,6 @@
class Security::Price < ApplicationRecord
include Provided
belongs_to :security
validates :price, :currency, presence: true
class << self
def find_price(security:, date:, cache: true)
result = find_by(security:, date:)
result || fetch_price_from_provider(security:, date:, cache:)
end
def find_prices(security:, start_date:, end_date: Date.current, cache: true)
prices = where(security_id: security.id, date: start_date..end_date).to_a
all_dates = (start_date..end_date).to_a.to_set
existing_dates = prices.map(&:date).to_set
missing_dates = (all_dates - existing_dates).sort
if missing_dates.any?
prices += fetch_prices_from_provider(
security: security,
start_date: missing_dates.first,
end_date: missing_dates.last,
cache: cache
)
end
prices
end
end
validates :date, :price, :currency, presence: true
validates :date, uniqueness: { scope: %i[security_id currency] }
end

65
app/models/security/price/provided.rb
View file

@ -1,65 +0,0 @@
module Security::Price::Provided
extend ActiveSupport::Concern
include Synthable
class_methods do
def provider
synth_client
end
private
def fetch_price_from_provider(security:, date:, cache: false)
return nil unless provider.present?
return nil unless security.has_prices?
response = provider.fetch_security_prices \
ticker: security.ticker,
mic_code: security.exchange_operating_mic,
start_date: date,
end_date: date
if response.success? && response.prices.size > 0
price = Security::Price.new \
security: security,
date: response.prices.first[:date],
price: response.prices.first[:price],
currency: response.prices.first[:currency]
price.save! if cache
price
else
nil
end
end
def fetch_prices_from_provider(security:, start_date:, end_date:, cache: false)
return [] unless provider.present?
return [] unless security
return [] unless security.has_prices?
response = provider.fetch_security_prices \
ticker: security.ticker,
mic_code: security.exchange_operating_mic,
start_date: start_date,
end_date: end_date
if response.success?
response.prices.map do |price|
new_price = Security::Price.find_or_initialize_by(
security: security,
date: price[:date]
) do |p|
p.price = price[:price]
p.currency = price[:currency]
end
new_price.save! if cache && new_price.new_record?
new_price
end
else
[]
end
end
end
end

31
app/models/security/provideable.rb Normal file
View file

@ -0,0 +1,31 @@
module Security::Provideable
extend ActiveSupport::Concern
Search = Data.define(:securities)
PriceData = Data.define(:price)
PricesData = Data.define(:prices)
SecurityInfo = Data.define(
:ticker,
:name,
:links,
:logo_url,
:description,
:kind,
)
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
raise NotImplementedError, "Subclasses must implement #search_securities"
end
def fetch_security_info(security)
raise NotImplementedError, "Subclasses must implement #fetch_security_info"
end
def fetch_security_price(security, date:)
raise NotImplementedError, "Subclasses must implement #fetch_security_price"
end
def fetch_security_prices(security, start_date:, end_date:)
raise NotImplementedError, "Subclasses must implement #fetch_security_prices"
end
end

61
app/models/security/provided.rb
View file

@ -1,28 +1,65 @@
module Security::Provided
extend ActiveSupport::Concern
include Synthable
class_methods do
def provider
synth_client
Providers.synth
end
def search_provider(query)
return [] if query[:search].blank? || query[:search].length < 2
def search_provider(symbol, country_code: nil, exchange_operating_mic: nil)
return [] if symbol.blank? || symbol.length < 2
response = provider.search_securities(
query: query[:search],
dataset: "limited",
country_code: query[:country],
exchange_operating_mic: query[:exchange_operating_mic]
)
response = provider.search_securities(symbol, country_code: country_code, exchange_operating_mic: exchange_operating_mic)
if response.success?
response.securities.map { |attrs| new(**attrs) }
response.data.securities
else
[]
end
end
end
def sync_provider_prices(start_date:, end_date: Date.current)
unless has_prices?
Rails.logger.warn("Security id=#{id} ticker=#{ticker} is not known by provider, skipping price sync")
return 0
end
unless provider.present?
Rails.logger.warn("No security provider configured, cannot sync prices for id=#{id} ticker=#{ticker}")
return 0
end
response = provider.fetch_security_prices(self, start_date: start_date, end_date: end_date)
unless response.success?
Rails.logger.error("Provider error for sync_provider_prices with id=#{id} ticker=#{ticker}: #{response.error}")
return 0
end
fetched_prices = response.data.prices.map do |price|
price.attributes.slice("security_id", "date", "price", "currency")
end
Security::Price.upsert_all(fetched_prices, unique_by: %i[security_id date currency])
end
def find_or_fetch_price(date: Date.current, cache: true)
price = prices.find_by(date: date)
return price if price.present?
response = provider.fetch_security_price(self, date: date)
return nil unless response.success? # Provider error
price = response.data.price
price.save! if cache
price
end
private
def provider
self.class.provider
end
end

2
app/models/trade_import.rb
View file

@ -97,7 +97,7 @@ class TradeImport < Import
provider_security = @provider_securities_cache[cache_key] ||= begin
Security.search_provider(
query: ticker,
ticker,
exchange_operating_mic: exchange_operating_mic
).first
end

6
app/models/upgrader/provided.rb
View file

@ -4,11 +4,7 @@ module Upgrader::Provided
class_methods do
private
def fetch_latest_upgrade_candidates_from_provider
git_repository_provider.fetch_latest_upgrade_candidates
end
def git_repository_provider
Provider::Github.new
Providers.github.fetch_latest_upgrade_candidates
end
end
end

2
app/views/account/trades/_form.html.erb
View file

@ -37,7 +37,7 @@
required: true %>
</div>
<% else %>
<%= form.text_field :manual_ticker, label: "Ticker", placeholder: "AAPL", required: true %>
<%= form.text_field :manual_ticker, label: "Ticker symbol", placeholder: "AAPL", required: true %>
<% end %>
<% end %>

7
app/views/accounts/_account.html.erb
View file

@ -17,13 +17,6 @@
</p>
<% else %>
<%= link_to account.name, account, class: [(account.is_active ? "text-primary" : "text-subdued"), "text-sm font-medium hover:underline"], data: { turbo_frame: "_top" } %>
<% if account.has_issues? %>
<div class="text-sm flex items-center gap-1 text-error">
<%= lucide_icon "alert-octagon", class: "shrink-0 w-4 h-4" %>
<%= tag.span t(".has_issues") %>
<%= link_to t(".troubleshoot"), issue_path(account.issues.first), class: "underline", data: { turbo_frame: :drawer } %>
</div>
<% end %>
<% end %>
</div>

2
app/views/accounts/_account_sidebar_tabs.html.erb
View file

@ -1,6 +1,6 @@
<%# locals: (family:) %>
<% if family.requires_data_provider? && family.synth_client.nil? %>
<% if family.requires_data_provider? && Providers.synth.nil? %>
<details class="group bg-yellow-tint-10 rounded-lg p-2 text-yellow-600 mb-3 text-xs">
<summary class="flex items-center justify-between gap-2">
<div class="flex items-center gap-2">

4
app/views/accounts/show/_header.html.erb
View file

@ -35,8 +35,4 @@
<%= render "accounts/show/menu", account: account %>
</div>
</div>
<% if account.highest_priority_issue %>
<%= render partial: "issues/issue", locals: { issue: account.highest_priority_issue } %>
<% end %>
</header>

12
app/views/issue/_request_synth_data_action.html.erb
View file

@ -1,12 +0,0 @@
<p>The Synth data provider could not find the requested data.</p>
<p>We are actively developing Synth to be a low cost and easy to use data provider. You can help us improve Synth by
requesting the data you need.</p>
<p>Please post in our <%= link_to "Discord server", "https://link.maybe.co/discord", target: "_blank" %> with the
following information:</p>
<ul>
<li>What type of data is missing?</li>
<li>Any other information you think might be helpful</li>
</ul>

28
app/views/issue/exchange_rate_provider_missings/show.html.erb
View file

@ -1,28 +0,0 @@
<%= content_for :title, @issue.title %>
<%= content_for :description do %>
<p>You have set your family currency preference to <%= Current.family.currency %>. <%= @issue.issuable.name %> has
entries in another currency, which means we have to fetch exchange rates from a data provider to accurately show
historical results.</p>
<p>We have detected that your exchange rates provider is not configured yet.</p>
<% end %>
<%= content_for :action do %>
<% if self_hosted? %>
<p>To fix this issue, you need to provide an API key for your exchange rate provider.</p>
<p>Currently, we support <%= link_to "Synth Finance", "https://synthfinance.com", target: "_blank" %>, so you need
to
get a free API key from the link provided.</p>
<p>Once you have your API key, paste it below to configure it.</p>
<%= styled_form_with model: @issue, url: issue_exchange_rate_provider_missing_path(@issue), method: :patch, class: "space-y-3" do |form| %>
<%= form.text_field :synth_api_key, label: "Synth API Key", placeholder: "Synth API Key", type: "password", class: "w-full", value: Setting.synth_api_key %>
<%= form.submit "Save and Re-Sync Account", class: "btn-primary" %>
<% end %>
<% else %>
<p>Please contact the Maybe team.</p>
<% end %>
<% end %>

11
app/views/issue/exchange_rates_missings/show.html.erb
View file

@ -1,11 +0,0 @@
<%= content_for :title, @issue.title %>
<%= content_for :description do %>
<p>Some exchange rates are missing for this account.</p>
<pre><code><%= JSON.pretty_generate(@issue.data) %></code></pre>
<% end %>
<%= content_for :action do %>
<%= render "issue/request_synth_data_action" %>
<% end %>

11
app/views/issue/prices_missings/show.html.erb
View file

@ -1,11 +0,0 @@
<%= content_for :title, @issue.title %>
<%= content_for :description do %>
<p>Some stock prices are missing for this account.</p>
<pre><code><%= JSON.pretty_generate(@issue.data) %></code></pre>
<% end %>
<%= content_for :action do %>
<%= render "issue/request_synth_data_action" %>
<% end %>

23
app/views/issue/unknowns/show.html.erb
View file

@ -1,23 +0,0 @@
<%= content_for :title, @issue.title %>
<%= content_for :description do %>
<p>An unknown issue has occurred.</p>
<pre><code><%= JSON.pretty_generate(@issue.data || "No data provided for this issue") %></code></pre>
<% end %>
<%= content_for :action do %>
<p>There is no fix for this issue yet.</p>
<p>Maybe is in active development and we value your feedback. There are a couple ways you can report this issue to
help us make Maybe better:</p>
<ul>
<li>Post in our <%= link_to "Discord server", "https://link.maybe.co/discord", target: "_blank" %></li>
<li>Open an issue on
our <%= link_to "Github repository", "https://github.com/maybe-finance/maybe/issues", target: "_blank" %></li>
</ul>
<p>If there is data shown in the code block above that you think might be helpful, please include it in your
report.</p>
<% end %>

15
app/views/issues/_issue.html.erb
View file

@ -1,15 +0,0 @@
<%# locals: (issue:) %>
<% priority_class = issue.critical? || issue.error? ? "bg-error/5" : "bg-warning/5" %>
<% text_class = issue.critical? || issue.error? ? "text-error" : "text-warning" %>
<%= tag.div class: "flex gap-6 items-center rounded-xl px-4 py-3 #{priority_class}" do %>
<div class="flex gap-3 items-center grow <%= text_class %>">
<%= lucide_icon("alert-octagon", class: "w-5 h-5 shrink-0") %>
<p class="text-sm break-words"><%= issue.title %></p>
</div>
<div class="flex items-center gap-4 ml-auto">
<%= link_to "Troubleshoot", issue_path(issue), class: "#{text_class} font-medium hover:underline", data: { turbo_frame: :drawer } %>
</div>
<% end %>

15
app/views/layouts/issues.html.erb
View file

@ -1,15 +0,0 @@
<%= render "layouts/shared/htmldoc" do %>
<%= drawer do %>
<article class="prose">
<%= tag.h2 do %>
<%= yield :title %>
<% end %>
<%= tag.h3 "Issue Description" %>
<%= yield :description %>
<%= tag.h3 "How to fix this issue" %>
<%= yield :action %>
</article>
<% end %>
<% end %>

10
app/views/settings/hostings/_synth_settings.html.erb
View file

@ -30,18 +30,18 @@
<div class="space-y-2">
<p class="text-sm text-secondary">
<%= t(".api_calls_used",
used: number_with_delimiter(@synth_usage.used),
limit: number_with_delimiter(@synth_usage.limit),
percentage: number_to_percentage(@synth_usage.utilization, precision: 1)) %>
used: number_with_delimiter(@synth_usage.data.used),
limit: number_with_delimiter(@synth_usage.data.limit),
percentage: number_to_percentage(@synth_usage.data.utilization, precision: 1)) %>
</p>
<div class="w-52 h-1.5 bg-gray-100 rounded-2xl">
<div class="h-full bg-green-500 rounded-2xl"
style="width: <%= [@synth_usage.utilization, 2].max %>%;"></div>
style="width: <%= [@synth_usage.data.utilization, 2].max %>%;"></div>
</div>
</div>
<div class="bg-gray-100 rounded-md px-1.5 py-0.5 w-fit">
<p class="text-xs font-medium text-secondary uppercase">
<%= t(".plan", plan: @synth_usage.plan) %>
<%= t(".plan", plan: @synth_usage.data.plan) %>
</p>
</div>
</div>