mirror of
https://github.com/maybe-finance/maybe.git
synced 2025-08-05 05:25:24 +02:00
Data provider simplification, tests, and documentation (#1997)
* Ignore env.test from source control * Simplification of providers interface * Synth tests * Update money to use new find rates method * Remove unused issues code * Additional issue feature removals * Update price data fetching and tests * Update documentation for providers * Security test fixes * Fix self host test * Update synth usage data access * Remove AI pr schema changes
This commit is contained in:
Zach Gollwitzer
GitHub
parent
dd75cadebc
commit
f65b93a352
95 changed files with 2014 additions and 1638 deletions
|
|
@ -1,18 +0,0 @@
|
|||
|
||||
class Provider::Base
|
||||
ProviderError = Class.new(StandardError)
|
||||
|
||||
TRANSIENT_NETWORK_ERRORS = [
|
||||
Faraday::TimeoutError,
|
||||
Faraday::ConnectionFailed,
|
||||
Faraday::SSLError,
|
||||
Faraday::ClientError,
|
||||
Faraday::ServerError
|
||||
]
|
||||
|
||||
class << self
|
||||
def known_transient_errors
|
||||
TRANSIENT_NETWORK_ERRORS + [ ProviderError ]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,217 +1,226 @@
|
|||
class Provider::Synth
|
||||
include Retryable
|
||||
class Provider::Synth < Provider
|
||||
include ExchangeRate::Provideable
|
||||
include Security::Provideable
|
||||
|
||||
def initialize(api_key)
|
||||
@api_key = api_key
|
||||
end
|
||||
|
||||
def healthy?
|
||||
response = client.get("#{base_url}/user")
|
||||
JSON.parse(response.body).dig("id").present?
|
||||
provider_response do
|
||||
response = client.get("#{base_url}/user")
|
||||
JSON.parse(response.body).dig("id").present?
|
||||
end
|
||||
end
|
||||
|
||||
def usage
|
||||
response = client.get("#{base_url}/user")
|
||||
provider_response do
|
||||
response = client.get("#{base_url}/user")
|
||||
|
||||
if response.status == 401
|
||||
return UsageResponse.new(
|
||||
success?: false,
|
||||
error: "Unauthorized: Invalid API key",
|
||||
raw_response: response
|
||||
parsed = JSON.parse(response.body)
|
||||
|
||||
remaining = parsed.dig("api_calls_remaining")
|
||||
limit = parsed.dig("api_limit")
|
||||
used = limit - remaining
|
||||
|
||||
UsageData.new(
|
||||
used: used,
|
||||
limit: limit,
|
||||
utilization: used.to_f / limit * 100,
|
||||
plan: parsed.dig("plan"),
|
||||
)
|
||||
end
|
||||
|
||||
parsed = JSON.parse(response.body)
|
||||
|
||||
remaining = parsed.dig("api_calls_remaining")
|
||||
limit = parsed.dig("api_limit")
|
||||
used = limit - remaining
|
||||
|
||||
UsageResponse.new(
|
||||
used: used,
|
||||
limit: limit,
|
||||
utilization: used.to_f / limit * 100,
|
||||
plan: parsed.dig("plan"),
|
||||
success?: true,
|
||||
raw_response: response
|
||||
)
|
||||
rescue StandardError => error
|
||||
UsageResponse.new(
|
||||
success?: false,
|
||||
error: error,
|
||||
raw_response: error
|
||||
)
|
||||
end
|
||||
|
||||
def fetch_security_prices(ticker:, start_date:, end_date:, mic_code: nil)
|
||||
params = {
|
||||
start_date: start_date,
|
||||
end_date: end_date
|
||||
}
|
||||
|
||||
params[:mic_code] = mic_code if mic_code.present?
|
||||
|
||||
prices = paginate(
|
||||
"#{base_url}/tickers/#{ticker}/open-close",
|
||||
params
|
||||
) do |body|
|
||||
body.dig("prices").map do |price|
|
||||
{
|
||||
date: price.dig("date"),
|
||||
price: price.dig("close")&.to_f || price.dig("open")&.to_f,
|
||||
currency: body.dig("currency") || "USD"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
SecurityPriceResponse.new \
|
||||
prices: prices,
|
||||
success?: true,
|
||||
raw_response: prices.to_json
|
||||
rescue StandardError => error
|
||||
SecurityPriceResponse.new \
|
||||
success?: false,
|
||||
error: error,
|
||||
raw_response: error
|
||||
end
|
||||
# ================================
|
||||
# Exchange Rates
|
||||
# ================================
|
||||
|
||||
def fetch_exchange_rate(from:, to:, date:)
|
||||
retrying Provider::Base.known_transient_errors do |on_last_attempt|
|
||||
provider_response retries: 2 do
|
||||
response = client.get("#{base_url}/rates/historical") do |req|
|
||||
req.params["date"] = date.to_s
|
||||
req.params["from"] = from
|
||||
req.params["to"] = to
|
||||
end
|
||||
|
||||
if response.success?
|
||||
ExchangeRateResponse.new \
|
||||
rate: JSON.parse(response.body).dig("data", "rates", to),
|
||||
success?: true,
|
||||
raw_response: response
|
||||
else
|
||||
if on_last_attempt
|
||||
ExchangeRateResponse.new \
|
||||
success?: false,
|
||||
error: build_error(response),
|
||||
raw_response: response
|
||||
else
|
||||
raise build_error(response)
|
||||
end
|
||||
end
|
||||
rates = JSON.parse(response.body).dig("data", "rates")
|
||||
|
||||
ExchangeRate::Provideable::FetchRateData.new(
|
||||
rate: ExchangeRate.new(
|
||||
from_currency: from,
|
||||
to_currency: to,
|
||||
date: date,
|
||||
rate: rates.dig(to)
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_exchange_rates(from:, to:, start_date:, end_date:)
|
||||
exchange_rates = paginate(
|
||||
"#{base_url}/rates/historical-range",
|
||||
from: from,
|
||||
to: to,
|
||||
date_start: start_date.to_s,
|
||||
date_end: end_date.to_s
|
||||
) do |body|
|
||||
body.dig("data").map do |exchange_rate|
|
||||
{
|
||||
date: exchange_rate.dig("date"),
|
||||
rate: exchange_rate.dig("rates", to)
|
||||
}
|
||||
provider_response retries: 1 do
|
||||
data = paginate(
|
||||
"#{base_url}/rates/historical-range",
|
||||
from: from,
|
||||
to: to,
|
||||
date_start: start_date.to_s,
|
||||
date_end: end_date.to_s
|
||||
) do |body|
|
||||
body.dig("data")
|
||||
end
|
||||
end
|
||||
|
||||
ExchangeRatesResponse.new \
|
||||
rates: exchange_rates,
|
||||
success?: true,
|
||||
raw_response: exchange_rates.to_json
|
||||
rescue StandardError => error
|
||||
ExchangeRatesResponse.new \
|
||||
success?: false,
|
||||
error: error,
|
||||
raw_response: error
|
||||
ExchangeRate::Provideable::FetchRatesData.new(
|
||||
rates: data.paginated.map do |exchange_rate|
|
||||
ExchangeRate.new(
|
||||
from_currency: from,
|
||||
to_currency: to,
|
||||
date: exchange_rate.dig("date"),
|
||||
rate: exchange_rate.dig("rates", to)
|
||||
)
|
||||
end
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def search_securities(query:, dataset: "limited", country_code: nil, exchange_operating_mic: nil)
|
||||
response = client.get("#{base_url}/tickers/search") do |req|
|
||||
req.params["name"] = query
|
||||
req.params["dataset"] = dataset
|
||||
req.params["country_code"] = country_code if country_code.present?
|
||||
req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present?
|
||||
req.params["limit"] = 25
|
||||
# ================================
|
||||
# Securities
|
||||
# ================================
|
||||
|
||||
def search_securities(symbol, country_code: nil, exchange_operating_mic: nil)
|
||||
provider_response do
|
||||
response = client.get("#{base_url}/tickers/search") do |req|
|
||||
req.params["name"] = symbol
|
||||
req.params["dataset"] = "limited"
|
||||
req.params["country_code"] = country_code if country_code.present?
|
||||
req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present?
|
||||
req.params["limit"] = 25
|
||||
end
|
||||
|
||||
parsed = JSON.parse(response.body)
|
||||
|
||||
Security::Provideable::Search.new(
|
||||
securities: parsed.dig("data").map do |security|
|
||||
Security.new(
|
||||
ticker: security.dig("symbol"),
|
||||
name: security.dig("name"),
|
||||
logo_url: security.dig("logo_url"),
|
||||
exchange_acronym: security.dig("exchange", "acronym"),
|
||||
exchange_mic: security.dig("exchange", "mic_code"),
|
||||
exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
|
||||
country_code: security.dig("exchange", "country_code")
|
||||
)
|
||||
end
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
parsed = JSON.parse(response.body)
|
||||
def fetch_security_info(security)
|
||||
provider_response do
|
||||
response = client.get("#{base_url}/tickers/#{security.ticker}") do |req|
|
||||
req.params["mic_code"] = security.exchange_mic if security.exchange_mic.present?
|
||||
req.params["operating_mic"] = security.exchange_operating_mic if security.exchange_operating_mic.present?
|
||||
end
|
||||
|
||||
securities = parsed.dig("data").map do |security|
|
||||
{
|
||||
ticker: security.dig("symbol"),
|
||||
name: security.dig("name"),
|
||||
logo_url: security.dig("logo_url"),
|
||||
exchange_acronym: security.dig("exchange", "acronym"),
|
||||
exchange_mic: security.dig("exchange", "mic_code"),
|
||||
exchange_operating_mic: security.dig("exchange", "operating_mic_code"),
|
||||
country_code: security.dig("exchange", "country_code")
|
||||
data = JSON.parse(response.body).dig("data")
|
||||
|
||||
Security::Provideable::SecurityInfo.new(
|
||||
ticker: security.ticker,
|
||||
name: data.dig("name"),
|
||||
links: data.dig("links"),
|
||||
logo_url: data.dig("logo_url"),
|
||||
description: data.dig("description"),
|
||||
kind: data.dig("kind")
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_security_price(security, date:)
|
||||
provider_response do
|
||||
historical_data = fetch_security_prices(security, start_date: date, end_date: date)
|
||||
|
||||
raise ProviderError, "No prices found for security #{security.ticker} on date #{date}" if historical_data.data.prices.empty?
|
||||
|
||||
Security::Provideable::PriceData.new(
|
||||
price: historical_data.data.prices.first
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_security_prices(security, start_date:, end_date:)
|
||||
provider_response retries: 1 do
|
||||
params = {
|
||||
start_date: start_date,
|
||||
end_date: end_date
|
||||
}
|
||||
end
|
||||
|
||||
SearchSecuritiesResponse.new \
|
||||
securities: securities,
|
||||
success?: true,
|
||||
raw_response: response
|
||||
params[:operating_mic_code] = security.exchange_operating_mic if security.exchange_operating_mic.present?
|
||||
|
||||
data = paginate(
|
||||
"#{base_url}/tickers/#{security.ticker}/open-close",
|
||||
params
|
||||
) do |body|
|
||||
body.dig("prices")
|
||||
end
|
||||
|
||||
currency = data.first_page.dig("currency")
|
||||
country_code = data.first_page.dig("exchange", "country_code")
|
||||
exchange_mic = data.first_page.dig("exchange", "mic_code")
|
||||
exchange_operating_mic = data.first_page.dig("exchange", "operating_mic_code")
|
||||
|
||||
Security::Provideable::PricesData.new(
|
||||
prices: data.paginated.map do |price|
|
||||
Security::Price.new(
|
||||
security: security,
|
||||
date: price.dig("date"),
|
||||
price: price.dig("close") || price.dig("open"),
|
||||
currency: currency
|
||||
)
|
||||
end
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
def fetch_security_info(ticker:, mic_code: nil, operating_mic: nil)
|
||||
response = client.get("#{base_url}/tickers/#{ticker}") do |req|
|
||||
req.params["mic_code"] = mic_code if mic_code.present?
|
||||
req.params["operating_mic"] = operating_mic if operating_mic.present?
|
||||
end
|
||||
|
||||
parsed = JSON.parse(response.body)
|
||||
|
||||
SecurityInfoResponse.new \
|
||||
info: parsed.dig("data"),
|
||||
success?: true,
|
||||
raw_response: response
|
||||
end
|
||||
# ================================
|
||||
# Transactions
|
||||
# ================================
|
||||
|
||||
def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
|
||||
params = {
|
||||
description: description,
|
||||
amount: amount,
|
||||
date: date,
|
||||
city: city,
|
||||
state: state,
|
||||
country: country
|
||||
}.compact
|
||||
provider_response do
|
||||
params = {
|
||||
description: description,
|
||||
amount: amount,
|
||||
date: date,
|
||||
city: city,
|
||||
state: state,
|
||||
country: country
|
||||
}.compact
|
||||
|
||||
response = client.get("#{base_url}/enrich", params)
|
||||
response = client.get("#{base_url}/enrich", params)
|
||||
|
||||
parsed = JSON.parse(response.body)
|
||||
parsed = JSON.parse(response.body)
|
||||
|
||||
EnrichTransactionResponse.new \
|
||||
info: EnrichTransactionInfo.new(
|
||||
TransactionEnrichmentData.new(
|
||||
name: parsed.dig("merchant"),
|
||||
icon_url: parsed.dig("icon"),
|
||||
category: parsed.dig("category")
|
||||
),
|
||||
success?: true,
|
||||
raw_response: response
|
||||
rescue StandardError => error
|
||||
EnrichTransactionResponse.new \
|
||||
success?: false,
|
||||
error: error,
|
||||
raw_response: error
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
attr_reader :api_key
|
||||
|
||||
ExchangeRateResponse = Struct.new :rate, :success?, :error, :raw_response, keyword_init: true
|
||||
SecurityPriceResponse = Struct.new :prices, :success?, :error, :raw_response, keyword_init: true
|
||||
ExchangeRatesResponse = Struct.new :rates, :success?, :error, :raw_response, keyword_init: true
|
||||
UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
|
||||
SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
|
||||
SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
|
||||
EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
|
||||
EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
|
||||
TransactionEnrichmentData = Data.define(:name, :icon_url, :category)
|
||||
|
||||
def retryable_errors
|
||||
[
|
||||
Faraday::TimeoutError,
|
||||
Faraday::ConnectionFailed,
|
||||
Faraday::SSLError,
|
||||
Faraday::ClientError,
|
||||
Faraday::ServerError
|
||||
]
|
||||
end
|
||||
|
||||
def base_url
|
||||
ENV["SYNTH_URL"] || "https://api.synthfinance.com"
|
||||
|
|
@ -227,26 +236,15 @@ class Provider::Synth
|
|||
|
||||
def client
|
||||
@client ||= Faraday.new(url: base_url) do |faraday|
|
||||
faraday.response :raise_error
|
||||
faraday.headers["Authorization"] = "Bearer #{api_key}"
|
||||
faraday.headers["X-Source"] = app_name
|
||||
faraday.headers["X-Source-Type"] = app_type
|
||||
end
|
||||
end
|
||||
|
||||
def build_error(response)
|
||||
Provider::Base::ProviderError.new(<<~ERROR)
|
||||
Failed to fetch data from #{self.class}
|
||||
Status: #{response.status}
|
||||
Body: #{response.body.inspect}
|
||||
ERROR
|
||||
end
|
||||
|
||||
def fetch_page(url, page, params = {})
|
||||
client.get(url) do |req|
|
||||
req.headers["Authorization"] = "Bearer #{api_key}"
|
||||
params.each { |k, v| req.params[k.to_s] = v.to_s }
|
||||
req.params["page"] = page
|
||||
end
|
||||
client.get(url, params.merge(page: page))
|
||||
end
|
||||
|
||||
def paginate(url, params = {})
|
||||
|
|
@ -254,24 +252,26 @@ class Provider::Synth
|
|||
page = 1
|
||||
current_page = 0
|
||||
total_pages = 1
|
||||
first_page = nil
|
||||
|
||||
while current_page < total_pages
|
||||
response = fetch_page(url, page, params)
|
||||
|
||||
if response.success?
|
||||
body = JSON.parse(response.body)
|
||||
page_results = yield(body)
|
||||
results.concat(page_results)
|
||||
body = JSON.parse(response.body)
|
||||
first_page = body unless first_page
|
||||
page_results = yield(body)
|
||||
results.concat(page_results)
|
||||
|
||||
current_page = body.dig("paging", "current_page")
|
||||
total_pages = body.dig("paging", "total_pages")
|
||||
current_page = body.dig("paging", "current_page")
|
||||
total_pages = body.dig("paging", "total_pages")
|
||||
|
||||
page += 1
|
||||
else
|
||||
raise build_error(response)
|
||||
end
|
||||
page += 1
|
||||
end
|
||||
|
||||
results
|
||||
PaginatedData.new(
|
||||
paginated: results,
|
||||
first_page: first_page,
|
||||
total_pages: total_pages
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue