From d2a36b10c6c1980fa5840456e990743dab521122 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 16:56:34 -0600 Subject: [PATCH 01/15] progress on Teller ETL service --- .env.example | 1 + apps/server/src/app/lib/endpoint.ts | 6 +- apps/server/src/env.ts | 1 + .../features/src/providers/teller/index.ts | 1 + .../src/providers/teller/teller.etl.ts | 267 ++++++++++++++++++ .../src/providers/teller/teller.service.ts | 106 ++++++- .../shared/src/services/queue.service.ts | 1 + libs/server/shared/src/utils/error-utils.ts | 9 + libs/server/shared/src/utils/index.ts | 1 + libs/server/shared/src/utils/teller-utils.ts | 30 ++ libs/shared/src/types/general-types.ts | 6 + libs/teller-api/src/teller-api.ts | 83 ++++-- libs/teller-api/src/types/account-balance.ts | 4 + libs/teller-api/src/types/account-details.ts | 5 + libs/teller-api/src/types/accounts.ts | 12 + libs/teller-api/src/types/authentication.ts | 4 + libs/teller-api/src/types/error.ts | 6 + libs/teller-api/src/types/index.ts | 1 + libs/teller-api/src/types/transactions.ts | 9 + .../migration.sql | 37 +++ prisma/schema.prisma | 20 ++ 21 files changed, 583 insertions(+), 27 deletions(-) create mode 100644 libs/server/features/src/providers/teller/teller.etl.ts create mode 100644 libs/server/shared/src/utils/teller-utils.ts create mode 100644 libs/teller-api/src/types/error.ts create mode 100644 prisma/migrations/20240115222631_add_fields_for_teller/migration.sql diff --git a/.env.example b/.env.example index fb329150..8c6641d2 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,7 @@ NX_FINICITY_PARTNER_SECRET= # Teller API keys (https://teller.io) NX_TELLER_SIGNING_SECRET= NX_TELLER_APP_ID= +NX_TELLER_ENV=sandbox # Email credentials NX_POSTMARK_FROM_ADDRESS=account@example.com diff --git a/apps/server/src/app/lib/endpoint.ts b/apps/server/src/app/lib/endpoint.ts index 01856bc3..c0341afa 100644 --- a/apps/server/src/app/lib/endpoint.ts +++ b/apps/server/src/app/lib/endpoint.ts @@ -43,6 +43,7 @@ import { FinicityWebhookHandler, PlaidWebhookHandler, TellerService, + TellerETL, TellerWebhookHandler, InsightService, SecurityPricingService, @@ -149,8 +150,10 @@ const tellerService = new TellerService( logger.child({ service: 'TellerService' }), prisma, teller, + new TellerETL(logger.child({ service: 'TellerETL' }), prisma, teller), + cryptoService, getTellerWebhookUrl(), - true + env.NX_TELLER_ENV === 'sandbox' ) // account-connection @@ -158,6 +161,7 @@ const tellerService = new TellerService( const accountConnectionProviderFactory = new AccountConnectionProviderFactory({ plaid: plaidService, finicity: finicityService, + teller: tellerService, }) const transactionStrategy = new TransactionBalanceSyncStrategy( diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts index e69f5f4d..b7d6adf4 100644 --- a/apps/server/src/env.ts +++ b/apps/server/src/env.ts @@ -43,6 +43,7 @@ const envSchema = z.object({ NX_TELLER_SIGNING_SECRET: z.string().default('REPLACE_THIS'), NX_TELLER_APP_ID: z.string().default('REPLACE_THIS'), + NX_TELLER_ENV: z.string().default('sandbox'), NX_SENTRY_DSN: z.string().optional(), NX_SENTRY_ENV: z.string().optional(), diff --git a/libs/server/features/src/providers/teller/index.ts b/libs/server/features/src/providers/teller/index.ts index 3f35c0cf..00a28837 100644 --- a/libs/server/features/src/providers/teller/index.ts +++ b/libs/server/features/src/providers/teller/index.ts @@ -1,2 +1,3 @@ export * from './teller.webhook' export * from './teller.service' +export * from './teller.etl' diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts new file mode 100644 index 00000000..b1491a35 --- /dev/null +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -0,0 +1,267 @@ +import type { AccountConnection, PrismaClient } from '@prisma/client' +import type { Logger } from 'winston' +import { SharedUtil, AccountUtil, type SharedType } from '@maybe-finance/shared' +import type { FinicityApi, FinicityTypes } from '@maybe-finance/finicity-api' +import type { TellerApi, TellerTypes } from '@maybe-finance/teller-api' +import { DbUtil, TellerUtil, type IETL } from '@maybe-finance/server/shared' +import { Prisma } from '@prisma/client' +import _ from 'lodash' +import { DateTime } from 'luxon' + +export type TellerRawData = { + accounts: TellerTypes.Account[] + transactions: TellerTypes.Transaction[] + transactionsDateRange: SharedType.DateRange +} + +export type TellerData = { + accounts: TellerTypes.Account[] + transactions: TellerTypes.Transaction[] + transactionsDateRange: SharedType.DateRange +} + +type Connection = Pick + +export class TellerETL implements IETL { + public constructor( + private readonly logger: Logger, + private readonly prisma: PrismaClient, + private readonly teller: Pick + ) {} + + async extract(connection: Connection): Promise { + if (!connection.tellerInstitutionId) { + throw new Error(`connection ${connection.id} is missing tellerInstitutionId`) + } + + const user = await this.prisma.user.findUniqueOrThrow({ + where: { id: connection.userId }, + select: { + id: true, + tellerUserId: true, + }, + }) + + if (!user.tellerUserId) { + throw new Error(`user ${user.id} is missing tellerUserId`) + } + + // TODO: Check if Teller supports date ranges for transactions + const transactionsDateRange = { + start: DateTime.now().minus(TellerUtil.TELLER_WINDOW_MAX), + end: DateTime.now(), + } + + const accounts = await this._extractAccounts(user.tellerUserId) + + const transactions = await this._extractTransactions( + user.tellerUserId, + accounts.map((a) => a.id), + transactionsDateRange + ) + + this.logger.info( + `Extracted Teller data for customer ${user.tellerUserId} accounts=${accounts.length} transactions=${transactions.length}`, + { connection: connection.id, transactionsDateRange } + ) + + return { + accounts, + transactions, + transactionsDateRange, + } + } + + async transform(_connection: Connection, data: TellerData): Promise { + return { + ...data, + } + } + + async load(connection: Connection, data: TellerData): Promise { + await this.prisma.$transaction([ + ...this._loadAccounts(connection, data), + ...this._loadTransactions(connection, data), + ]) + + this.logger.info(`Loaded Teller data for connection ${connection.id}`, { + connection: connection.id, + }) + } + + private async _extractAccounts(tellerUserId: string) { + const { accounts } = await this.teller.getAccounts({ accessToken: undefined }) + + return accounts.filter( + (a) => a.institutionLoginId.toString() === institutionLoginId && a.currency === 'USD' + ) + } + + private _loadAccounts(connection: Connection, { accounts }: Pick) { + return [ + // upsert accounts + ...accounts.map((tellerAccount) => { + return this.prisma.account.upsert({ + where: { + accountConnectionId_tellerAccountId: { + accountConnectionId: connection.id, + tellerAccountId: tellerAccount.id, + }, + }, + create: { + type: TellerUtil.getType(tellerAccount.type), + provider: 'teller', + categoryProvider: PlaidUtil.plaidTypesToCategory(plaidAccount.type), + subcategoryProvider: plaidAccount.subtype ?? 'other', + accountConnectionId: connection.id, + plaidAccountId: plaidAccount.account_id, + name: tellerAccount.name, + plaidType: tellerAccount.type, + plaidSubtype: tellerAccount.subtype, + mask: plaidAccount.mask, + ...PlaidUtil.getAccountBalanceData( + plaidAccount.balances, + plaidAccount.type + ), + }, + update: { + type: TellerUtil.getType(tellerAccount.type), + categoryProvider: PlaidUtil.plaidTypesToCategory(tellerAccount.type), + subcategoryProvider: tellerAccount.subtype ?? 'other', + plaidType: tellerAccount.type, + plaidSubtype: tellerAccount.subtype, + ..._.omit( + PlaidUtil.getAccountBalanceData( + plaidAccount.balances, + plaidAccount.type + ), + ['currentBalanceStrategy', 'availableBalanceStrategy'] + ), + }, + }) + }), + // any accounts that are no longer in Plaid should be marked inactive + this.prisma.account.updateMany({ + where: { + accountConnectionId: connection.id, + AND: [ + { tellerAccountId: { not: null } }, + { tellerAccountId: { notIn: accounts.map((a) => a.id) } }, + ], + }, + data: { + isActive: false, + }, + }), + ] + } + + private async _extractTransactions( + customerId: string, + accountIds: string[], + dateRange: SharedType.DateRange + ) { + const accountTransactions = await Promise.all( + accountIds.map((accountId) => + SharedUtil.paginate({ + pageSize: 1000, // https://api-reference.finicity.com/#/rest/api-endpoints/transactions/get-customer-account-transactions + fetchData: async (offset, count) => { + const transactions = await SharedUtil.withRetry( + () => + this.teller.getTransactions({ + accountId, + accessToken: undefined, + fromDate: dateRange.start.toUnixInteger(), + toDate: dateRange.end.toUnixInteger(), + start: offset + 1, + limit: count, + }), + { + maxRetries: 3, + } + ) + + return transactions + }, + }) + ) + ) + + return accountTransactions.flat() + } + + private _loadTransactions( + connection: Connection, + { + transactions, + transactionsDateRange, + }: Pick + ) { + if (!transactions.length) return [] + + const txnUpsertQueries = _.chunk(transactions, 1_000).map((chunk) => { + return this.prisma.$executeRaw` + INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category) + VALUES + ${Prisma.join( + chunk.map((tellerTransaction) => { + const { + id, + account_id, + description, + amount, + status, + type, + details, + date, + } = tellerTransaction + + return Prisma.sql`( + (SELECT id FROM account WHERE account_connection_id = ${ + connection.id + } AND teller_account_id = ${account_id.toString()}), + ${id}, + ${date}::date, + ${[description].filter(Boolean).join(' ')}, + ${DbUtil.toDecimal(-amount)}, + ${status === 'pending'}, + ${'USD'}, + ${details.counterparty.name ?? ''}, + ${type}, + ${details.category ?? ''}, + )` + }) + )} + ON CONFLICT (teller_transaction_id) DO UPDATE + SET + name = EXCLUDED.name, + amount = EXCLUDED.amount, + pending = EXCLUDED.pending, + merchant_name = EXCLUDED.merchant_name, + teller_type = EXCLUDED.teller_type, + teller_category = EXCLUDED.teller_category; + ` + }) + + return [ + // upsert transactions + ...txnUpsertQueries, + // delete teller-specific transactions that are no longer in teller + this.prisma.transaction.deleteMany({ + where: { + account: { + accountConnectionId: connection.id, + }, + AND: [ + { tellerTransactionId: { not: null } }, + { tellerTransactionId: { notIn: transactions.map((t) => `${t.id}`) } }, + ], + date: { + gte: transactionsDateRange.start.startOf('day').toJSDate(), + lte: transactionsDateRange.end.endOf('day').toJSDate(), + }, + }, + }), + ] + } +} diff --git a/libs/server/features/src/providers/teller/teller.service.ts b/libs/server/features/src/providers/teller/teller.service.ts index de393b22..bc6a4237 100644 --- a/libs/server/features/src/providers/teller/teller.service.ts +++ b/libs/server/features/src/providers/teller/teller.service.ts @@ -1,5 +1,14 @@ import type { Logger } from 'winston' import type { AccountConnection, PrismaClient, User } from '@prisma/client' +import type { IInstitutionProvider } from '../../institution' +import type { + AccountConnectionSyncEvent, + IAccountConnectionProvider, +} from '../../account-connection' +import { SharedUtil } from '@maybe-finance/shared' +import type { SyncConnectionOptions, CryptoService, IETL } from '@maybe-finance/server/shared' +import _ from 'lodash' +import { ErrorUtil, etl } from '@maybe-finance/server/shared' import type { TellerApi } from '@maybe-finance/teller-api' export interface ITellerConnect { @@ -11,12 +20,107 @@ export interface ITellerConnect { ): Promise<{ link: string }> } -export class TellerService { +export class TellerService implements IAccountConnectionProvider, IInstitutionProvider { constructor( private readonly logger: Logger, private readonly prisma: PrismaClient, private readonly teller: TellerApi, + private readonly etl: IETL, + private readonly crypto: CryptoService, private readonly webhookUrl: string | Promise, private readonly testMode: boolean ) {} + + async sync(connection: AccountConnection, options?: SyncConnectionOptions) { + if (options && options.type !== 'teller') throw new Error('invalid sync options') + + await etl(this.etl, connection) + } + + async onSyncEvent(connection: AccountConnection, event: AccountConnectionSyncEvent) { + switch (event.type) { + case 'success': { + await this.prisma.accountConnection.update({ + where: { id: connection.id }, + data: { + status: 'OK', + }, + }) + break + } + case 'error': { + const { error } = event + + await this.prisma.accountConnection.update({ + where: { id: connection.id }, + data: { + status: 'ERROR', + tellerError: ErrorUtil.isTellerError(error) + ? (error.response.data as any) + : undefined, + }, + }) + break + } + } + } + + async delete(connection: AccountConnection) { + // purge teller data + if (connection.tellerAccessToken && connection.tellerAccountId) { + await this.teller.deleteAccount({ + accessToken: this.crypto.decrypt(connection.tellerAccessToken), + accountId: connection.tellerAccountId, + }) + + this.logger.info(`Item ${connection.tellerAccountId} removed`) + } + } + + async getInstitutions() { + const tellerInstitutions = await SharedUtil.paginate({ + pageSize: 500, + delay: + process.env.NODE_ENV !== 'production' + ? { + onDelay: (message: string) => this.logger.debug(message), + milliseconds: 7_000, // Sandbox rate limited at 10 calls / minute + } + : undefined, + fetchData: (offset, count) => + SharedUtil.withRetry( + () => + this.teller.getInstitutions().then((data) => { + this.logger.debug( + `paginated teller fetch inst=${data.institutions.length} (total=${data.institutions.length} offset=${offset} count=${count})` + ) + return data.institutions + }), + { + maxRetries: 3, + onError: (error, attempt) => { + this.logger.error( + `Plaid fetch institutions request failed attempt=${attempt} offset=${offset} count=${count}`, + { error: ErrorUtil.parseError(error) } + ) + + return !ErrorUtil.isPlaidError(error) || error.response.status >= 500 + }, + } + ), + }) + + return _.uniqBy(tellerInstitutions, (i) => i.id).map((tellerInstitution) => { + const { id, name } = tellerInstitution + return { + providerId: id, + name, + url: undefined, + logo: `https://teller.io/images/banks/${id}.jpg}`, + primaryColor: undefined, + oauth: undefined, + data: tellerInstitution, + } + }) + } } diff --git a/libs/server/shared/src/services/queue.service.ts b/libs/server/shared/src/services/queue.service.ts index 2e6c2160..319fef92 100644 --- a/libs/server/shared/src/services/queue.service.ts +++ b/libs/server/shared/src/services/queue.service.ts @@ -41,6 +41,7 @@ export type SyncConnectionOptions = products?: Array<'transactions' | 'investment-transactions' | 'holdings' | 'liabilities'> } | { type: 'finicity'; initialSync?: boolean } + | { type: 'teller'; initialSync?: boolean } export type SyncConnectionQueueJobData = { accountConnectionId: AccountConnection['id'] diff --git a/libs/server/shared/src/utils/error-utils.ts b/libs/server/shared/src/utils/error-utils.ts index 3366b3fb..65cefc6a 100644 --- a/libs/server/shared/src/utils/error-utils.ts +++ b/libs/server/shared/src/utils/error-utils.ts @@ -32,6 +32,15 @@ export function isPlaidError(err: unknown): err is SharedType.AxiosPlaidError { return 'error_type' in data && 'error_code' in data && 'error_message' in data } +export function isTellerError(err: unknown): err is SharedType.AxiosTellerError { + if (!err) return false + if (!axios.isAxiosError(err)) return false + if (typeof err.response?.data !== 'object') return false + + const { data } = err.response + return 'code' in data.error && 'message' in data.error +} + export function parseError(error: unknown): SharedType.ParsedError { if (isPlaidError(error)) { return parsePlaidError(error) diff --git a/libs/server/shared/src/utils/index.ts b/libs/server/shared/src/utils/index.ts index bad4dc18..4ba6606d 100644 --- a/libs/server/shared/src/utils/index.ts +++ b/libs/server/shared/src/utils/index.ts @@ -2,6 +2,7 @@ export * as AuthUtil from './auth-utils' export * as DbUtil from './db-utils' export * as FinicityUtil from './finicity-utils' export * as PlaidUtil from './plaid-utils' +export * as TellerUtil from './teller-utils' export * as ErrorUtil from './error-utils' // All "generic" server utils grouped here diff --git a/libs/server/shared/src/utils/teller-utils.ts b/libs/server/shared/src/utils/teller-utils.ts new file mode 100644 index 00000000..58ce8b88 --- /dev/null +++ b/libs/server/shared/src/utils/teller-utils.ts @@ -0,0 +1,30 @@ +import { AccountCategory, AccountType } from '@prisma/client' +import type { TellerTypes } from '@maybe-finance/teller-api' +import { Duration } from 'luxon' + +/** + * Update this with the max window that Teller supports + */ +export const TELLER_WINDOW_MAX = Duration.fromObject({ years: 1 }) + +export function getType(type: TellerTypes.AccountTypes): AccountType { + switch (type) { + case 'depository': + return AccountType.DEPOSITORY + case 'credit': + return AccountType.CREDIT + default: + return AccountType.OTHER_ASSET + } +} + +export function tellerTypesToCategory(tellerType: TellerTypes.AccountTypes): AccountCategory { + switch (tellerType) { + case 'depository': + return AccountCategory.cash + case 'credit': + return AccountCategory.credit + default: + return AccountCategory.other + } +} diff --git a/libs/shared/src/types/general-types.ts b/libs/shared/src/types/general-types.ts index 97117f01..30a28824 100644 --- a/libs/shared/src/types/general-types.ts +++ b/libs/shared/src/types/general-types.ts @@ -1,6 +1,7 @@ import type { Prisma } from '@prisma/client' import type { PlaidError } from 'plaid' import type { AxiosError } from 'axios' +import type { TellerTypes } from '@maybe-finance/teller-api' import type { Contexts, Primitive } from '@sentry/types' import type DecimalJS from 'decimal.js' import type { O } from 'ts-toolbelt' @@ -79,6 +80,11 @@ export type ParsedError = { export type AxiosPlaidError = O.Required, 'response' | 'config'> +export type AxiosTellerError = O.Required< + AxiosError, + 'response' | 'config' +> + export type StatusPageResponse = { page?: { id?: string diff --git a/libs/teller-api/src/teller-api.ts b/libs/teller-api/src/teller-api.ts index 46670971..ca3a180a 100644 --- a/libs/teller-api/src/teller-api.ts +++ b/libs/teller-api/src/teller-api.ts @@ -9,6 +9,13 @@ import type { DeleteAccountResponse, GetAccountDetailsResponse, GetInstitutionsResponse, + AuthenticatedRequest, + GetAccountRequest, + DeleteAccountRequest, + GetAccountDetailsRequest, + GetAccountBalancesRequest, + GetTransactionsRequest, + GetTransactionRequest, } from './types' import axios from 'axios' import * as fs from 'fs' @@ -26,8 +33,8 @@ export class TellerApi { * https://teller.io/docs/api/accounts */ - async getAccounts(): Promise { - return this.get(`/accounts`) + async getAccounts({ accessToken }: AuthenticatedRequest): Promise { + return this.get(`/accounts`, accessToken) } /** @@ -36,8 +43,8 @@ export class TellerApi { * https://teller.io/docs/api/accounts */ - async getAccount(accountId: string): Promise { - return this.get(`/accounts/${accountId}`) + async getAccount({ accountId, accessToken }: GetAccountRequest): Promise { + return this.get(`/accounts/${accountId}`, accessToken) } /** @@ -46,8 +53,11 @@ export class TellerApi { * https://teller.io/docs/api/accounts */ - async deleteAccount(accountId: string): Promise { - return this.delete(`/accounts/${accountId}`) + async deleteAccount({ + accountId, + accessToken, + }: DeleteAccountRequest): Promise { + return this.delete(`/accounts/${accountId}`, accessToken) } /** @@ -56,8 +66,11 @@ export class TellerApi { * https://teller.io/docs/api/account/details */ - async getAccountDetails(accountId: string): Promise { - return this.get(`/accounts/${accountId}/details`) + async getAccountDetails({ + accountId, + accessToken, + }: GetAccountDetailsRequest): Promise { + return this.get(`/accounts/${accountId}/details`, accessToken) } /** @@ -66,8 +79,11 @@ export class TellerApi { * https://teller.io/docs/api/account/balances */ - async getAccountBalances(accountId: string): Promise { - return this.get(`/accounts/${accountId}/balances`) + async getAccountBalances({ + accountId, + accessToken, + }: GetAccountBalancesRequest): Promise { + return this.get(`/accounts/${accountId}/balances`, accessToken) } /** @@ -76,8 +92,11 @@ export class TellerApi { * https://teller.io/docs/api/transactions */ - async getTransactions(accountId: string): Promise { - return this.get(`/accounts/${accountId}/transactions`) + async getTransactions({ + accountId, + accessToken, + }: GetTransactionsRequest): Promise { + return this.get(`/accounts/${accountId}/transactions`, accessToken) } /** @@ -86,12 +105,14 @@ export class TellerApi { * https://teller.io/docs/api/transactions */ - async getTransaction( - accountId: string, - transactionId: string - ): Promise { + async getTransaction({ + accountId, + transactionId, + accessToken, + }: GetTransactionRequest): Promise { return this.get( - `/accounts/${accountId}/transactions/${transactionId}` + `/accounts/${accountId}/transactions/${transactionId}`, + accessToken ) } @@ -101,21 +122,21 @@ export class TellerApi { * https://teller.io/docs/api/identity */ - async getIdentity(): Promise { - return this.get(`/identity`) + async getIdentity({ accessToken }: AuthenticatedRequest): Promise { + return this.get(`/identity`, accessToken) } /** - * Get list of supported institutions + * Get list of supported institutions, access token not needed * * https://teller.io/docs/api/identity */ async getInstitutions(): Promise { - return this.get(`/institutions`) + return this.get(`/institutions`, '') } - private async getApi(): Promise { + private async getApi(accessToken: string): Promise { const cert = fs.readFileSync('../../../certs/teller-certificate.pem', 'utf8') const key = fs.readFileSync('../../../certs/teller-private-key.pem', 'utf8') @@ -133,6 +154,15 @@ export class TellerApi { Accept: 'application/json', }, }) + + this.api.interceptors.request.use((config) => { + // Add the access_token to the auth object + config.auth = { + username: 'ACCESS_TOKEN', + password: accessToken, + } + return config + }) } return this.api @@ -141,30 +171,33 @@ export class TellerApi { /** Generic API GET request method */ private async get( path: string, + accessToken: string, params?: any, config?: AxiosRequestConfig ): Promise { - const api = await this.getApi() + const api = await this.getApi(accessToken) return api.get(path, { params, ...config }).then(({ data }) => data) } /** Generic API POST request method */ private async post( path: string, + accessToken: string, body?: any, config?: AxiosRequestConfig ): Promise { - const api = await this.getApi() + const api = await this.getApi(accessToken) return api.post(path, body, config).then(({ data }) => data) } /** Generic API DELETE request method */ private async delete( path: string, + accessToken: string, params?: any, config?: AxiosRequestConfig ): Promise { - const api = await this.getApi() + const api = await this.getApi(accessToken) return api.delete(path, { params, ...config }).then(({ data }) => data) } } diff --git a/libs/teller-api/src/types/account-balance.ts b/libs/teller-api/src/types/account-balance.ts index 01f0a944..d2e21193 100644 --- a/libs/teller-api/src/types/account-balance.ts +++ b/libs/teller-api/src/types/account-balance.ts @@ -1,4 +1,5 @@ // https://teller.io/docs/api/account/balances +import type { AuthenticatedRequest } from './authentication' export type AccountBalance = { account_id: string @@ -11,3 +12,6 @@ export type AccountBalance = { } export type GetAccountBalancesResponse = AccountBalance +export interface GetAccountBalancesRequest extends AuthenticatedRequest { + accountId: string +} diff --git a/libs/teller-api/src/types/account-details.ts b/libs/teller-api/src/types/account-details.ts index 3dc47ed0..fb6e8405 100644 --- a/libs/teller-api/src/types/account-details.ts +++ b/libs/teller-api/src/types/account-details.ts @@ -1,5 +1,7 @@ // https://teller.io/docs/api/account/details +import type { AuthenticatedRequest } from './authentication' + export type AccountDetails = { account_id: string account_number: string @@ -15,3 +17,6 @@ export type AccountDetails = { } export type GetAccountDetailsResponse = AccountDetails +export interface GetAccountDetailsRequest extends AuthenticatedRequest { + accountId: string +} diff --git a/libs/teller-api/src/types/accounts.ts b/libs/teller-api/src/types/accounts.ts index 8faad5c7..e3ae9075 100644 --- a/libs/teller-api/src/types/accounts.ts +++ b/libs/teller-api/src/types/accounts.ts @@ -1,7 +1,13 @@ // https://teller.io/docs/api/accounts +import type { AuthenticatedRequest } from './authentication' export type AccountTypes = 'depository' | 'credit' +export enum AccountType { + 'depository', + 'credit', +} + export type DepositorySubtypes = | 'checking' | 'savings' @@ -45,3 +51,9 @@ export type Account = DepositoryAccount | CreditAccount export type GetAccountsResponse = { accounts: Account[] } export type GetAccountResponse = Account export type DeleteAccountResponse = void + +export interface GetAccountRequest extends AuthenticatedRequest { + accountId: string +} + +export type DeleteAccountRequest = GetAccountRequest diff --git a/libs/teller-api/src/types/authentication.ts b/libs/teller-api/src/types/authentication.ts index 1f45b91a..b2826a81 100644 --- a/libs/teller-api/src/types/authentication.ts +++ b/libs/teller-api/src/types/authentication.ts @@ -3,3 +3,7 @@ export type AuthenticationResponse = { token: string } + +export type AuthenticatedRequest = { + accessToken: string +} diff --git a/libs/teller-api/src/types/error.ts b/libs/teller-api/src/types/error.ts new file mode 100644 index 00000000..33b702c2 --- /dev/null +++ b/libs/teller-api/src/types/error.ts @@ -0,0 +1,6 @@ +export type TellerError = { + error: { + code: string + message: string + } +} diff --git a/libs/teller-api/src/types/index.ts b/libs/teller-api/src/types/index.ts index f3b60309..ca90d347 100644 --- a/libs/teller-api/src/types/index.ts +++ b/libs/teller-api/src/types/index.ts @@ -2,6 +2,7 @@ export * from './accounts' export * from './account-balance' export * from './account-details' export * from './authentication' +export * from './error' export * from './identity' export * from './institutions' export * from './transactions' diff --git a/libs/teller-api/src/types/transactions.ts b/libs/teller-api/src/types/transactions.ts index 9c7dc07b..1d482aa8 100644 --- a/libs/teller-api/src/types/transactions.ts +++ b/libs/teller-api/src/types/transactions.ts @@ -1,5 +1,7 @@ // https://teller.io/docs/api/account/transactions +import type { AuthenticatedRequest } from './authentication' + type DetailCategory = | 'accommodation' | 'advertising' @@ -57,3 +59,10 @@ export type Transaction = { export type GetTransactionsResponse = Transaction[] export type GetTransactionResponse = Transaction +export interface GetTransactionsRequest extends AuthenticatedRequest { + accountId: string +} +export interface GetTransactionRequest extends AuthenticatedRequest { + accountId: string + transactionId: string +} diff --git a/prisma/migrations/20240115222631_add_fields_for_teller/migration.sql b/prisma/migrations/20240115222631_add_fields_for_teller/migration.sql new file mode 100644 index 00000000..e05cf085 --- /dev/null +++ b/prisma/migrations/20240115222631_add_fields_for_teller/migration.sql @@ -0,0 +1,37 @@ +/* + Warnings: + + - A unique constraint covering the columns `[account_connection_id,teller_account_id]` on the table `account` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[teller_transaction_id]` on the table `transaction` will be added. If there are existing duplicate values, this will fail. + - A unique constraint covering the columns `[teller_user_id]` on the table `user` will be added. If there are existing duplicate values, this will fail. + +*/ +-- AlterEnum +ALTER TYPE "AccountConnectionType" ADD VALUE 'teller'; + +-- AlterTable +ALTER TABLE "account" ADD COLUMN "teller_account_id" TEXT, +ADD COLUMN "teller_type" TEXT; + +-- AlterTable +ALTER TABLE "account_connection" ADD COLUMN "teller_access_token" TEXT, +ADD COLUMN "teller_account_id" TEXT, +ADD COLUMN "teller_error" JSONB, +ADD COLUMN "teller_institution_id" TEXT; + +-- AlterTable +ALTER TABLE "transaction" ADD COLUMN "teller_category" TEXT, +ADD COLUMN "teller_transaction_id" TEXT, +ADD COLUMN "teller_type" TEXT; + +-- AlterTable +ALTER TABLE "user" ADD COLUMN "teller_user_id" TEXT; + +-- CreateIndex +CREATE UNIQUE INDEX "account_account_connection_id_teller_account_id_key" ON "account"("account_connection_id", "teller_account_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "transaction_teller_transaction_id_key" ON "transaction"("teller_transaction_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "user_teller_user_id_key" ON "user"("teller_user_id"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 587a274c..160ea264 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -43,6 +43,7 @@ enum AccountSyncStatus { enum AccountConnectionType { plaid finicity + teller } model AccountConnection { @@ -69,6 +70,12 @@ model AccountConnection { finicityInstitutionId String? @map("finicity_institution_id") finicityError Json? @map("finicity_error") + // teller data + tellerAccountId String? @map("teller_account_id") + tellerAccessToken String? @map("teller_access_token") + tellerInstitutionId String? @map("teller_institution_id") + tellerError Json? @map("teller_error") + accounts Account[] @@index([userId]) @@ -152,6 +159,10 @@ model Account { finicityType String? @map("finicity_type") finicityDetail Json? @map("finicity_detail") @db.JsonB + // teller data + tellerAccountId String? @map("teller_account_id") + tellerType String? @map("teller_type") + // manual account data vehicleMeta Json? @map("vehicle_meta") @db.JsonB propertyMeta Json? @map("property_meta") @db.JsonB @@ -172,6 +183,7 @@ model Account { @@unique([accountConnectionId, plaidAccountId]) @@unique([accountConnectionId, finicityAccountId]) + @@unique([accountConnectionId, tellerAccountId]) @@index([accountConnectionId]) @@index([userId]) @@map("account") @@ -346,6 +358,11 @@ model Transaction { finicityType String? @map("finicity_type") finicityCategorization Json? @map("finicity_categorization") @db.JsonB + // teller data + tellerTransactionId String? @unique @map("teller_transaction_id") + tellerType String? @map("teller_type") + tellerCategory String? @map("teller_category") + @@index([accountId, date]) @@index([amount]) @@map("transaction") @@ -430,6 +447,9 @@ model User { // plaid data plaidLinkToken String? @map("plaid_link_token") // temporary token stored to maintain state across browsers + // teller data + tellerUserId String? @unique @map("teller_user_id") + // Onboarding / usage goals household Household? state String? From f3ba272745609f5c5fee04eda41d039d3845a36e Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 17:23:05 -0600 Subject: [PATCH 02/15] noop postmark sends if API key is not defined --- apps/server/src/app/lib/postmark.ts | 2 +- apps/server/src/env.ts | 2 +- apps/workers/src/app/lib/postmark.ts | 2 +- apps/workers/src/env.ts | 2 +- .../features/src/email/email.service.ts | 30 +++++++++++++++---- package.json | 2 +- 6 files changed, 30 insertions(+), 10 deletions(-) diff --git a/apps/server/src/app/lib/postmark.ts b/apps/server/src/app/lib/postmark.ts index 48eddf1b..ff0e87c6 100644 --- a/apps/server/src/app/lib/postmark.ts +++ b/apps/server/src/app/lib/postmark.ts @@ -1,6 +1,6 @@ import { ServerClient } from 'postmark' import env from '../../env' -const postmark = new ServerClient(env.NX_POSTMARK_API_TOKEN) +const postmark = env.NX_POSTMARK_API_TOKEN ? new ServerClient(env.NX_POSTMARK_API_TOKEN) : undefined export default postmark diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts index e69f5f4d..fabdc7d2 100644 --- a/apps/server/src/env.ts +++ b/apps/server/src/env.ts @@ -74,7 +74,7 @@ const envSchema = z.object({ NX_POSTMARK_FROM_ADDRESS: z.string().default('account@maybe.co'), NX_POSTMARK_REPLY_TO_ADDRESS: z.string().default('support@maybe.co'), - NX_POSTMARK_API_TOKEN: z.string().default('REPLACE_THIS'), + NX_POSTMARK_API_TOKEN: z.string().optional(), }) const env = envSchema.parse(process.env) diff --git a/apps/workers/src/app/lib/postmark.ts b/apps/workers/src/app/lib/postmark.ts index 48eddf1b..ff0e87c6 100644 --- a/apps/workers/src/app/lib/postmark.ts +++ b/apps/workers/src/app/lib/postmark.ts @@ -1,6 +1,6 @@ import { ServerClient } from 'postmark' import env from '../../env' -const postmark = new ServerClient(env.NX_POSTMARK_API_TOKEN) +const postmark = env.NX_POSTMARK_API_TOKEN ? new ServerClient(env.NX_POSTMARK_API_TOKEN) : undefined export default postmark diff --git a/apps/workers/src/env.ts b/apps/workers/src/env.ts index c41d9692..df560b4e 100644 --- a/apps/workers/src/env.ts +++ b/apps/workers/src/env.ts @@ -29,7 +29,7 @@ const envSchema = z.object({ NX_POSTMARK_FROM_ADDRESS: z.string().default('account@maybe.co'), NX_POSTMARK_REPLY_TO_ADDRESS: z.string().default('support@maybe.co'), - NX_POSTMARK_API_TOKEN: z.string().default('REPLACE_THIS'), + NX_POSTMARK_API_TOKEN: z.string().optional(), NX_STRIPE_SECRET_KEY: z.string().default('sk_test_REPLACE_THIS'), NX_CDN_PRIVATE_BUCKET: z.string().default('REPLACE_THIS'), diff --git a/libs/server/features/src/email/email.service.ts b/libs/server/features/src/email/email.service.ts index 6c908985..b8b1d3b7 100644 --- a/libs/server/features/src/email/email.service.ts +++ b/libs/server/features/src/email/email.service.ts @@ -22,7 +22,7 @@ export interface IEmailService { export class EmailService implements IEmailService { constructor( private readonly logger: Logger, - private readonly postmark: PostmarkServerClient, + private readonly postmark: PostmarkServerClient | undefined, private readonly defaultAddresses: { from: string; replyTo?: string } ) {} @@ -85,6 +85,11 @@ export class EmailService implements IEmailService { message.TemplateModel ) + if (!this.postmark) { + this.logger.info('Postmark API key not provided, skipping email send') + return undefined as unknown as MessageSendingResponse + } + return await this.postmark.sendEmailWithTemplate(message) } @@ -94,6 +99,11 @@ export class EmailService implements IEmailService { { text: message.TextBody, html: message.HtmlBody } ) + if (!this.postmark) { + this.logger.info('Postmark API key not provided, skipping email send') + return undefined as unknown as MessageSendingResponse + } + return await this.postmark.sendEmail(message) } @@ -108,9 +118,13 @@ export class EmailService implements IEmailService { return ( await Promise.all( - chunk(messages, 500).map((chunk) => - this.postmark.sendEmailBatchWithTemplates(chunk) - ) + chunk(messages, 500).map((chunk) => { + if (!this.postmark) { + this.logger.info('Postmark API key not provided, skipping email send') + return [] as MessageSendingResponse[] + } + return this.postmark.sendEmailBatchWithTemplates(chunk) + }) ) ).flat() } @@ -124,7 +138,13 @@ export class EmailService implements IEmailService { return ( await Promise.all( - chunk(messages, 500).map((chunk) => this.postmark.sendEmailBatch(chunk)) + chunk(messages, 500).map((chunk) => { + if (!this.postmark) { + this.logger.info('Postmark API key not provided, skipping email send') + return [] as MessageSendingResponse[] + } + return this.postmark.sendEmailBatch(chunk) + }) ) ).flat() } diff --git a/package.json b/package.json index 4a99fc2b..64d4aaeb 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "nx run-many --target serve --projects=client,server,workers --parallel --host 0.0.0.0 --nx-bail=true --maxParallel=100", "dev:services": "COMPOSE_PROFILES=services docker-compose up -d", - "dev:services:all": "COMPOSE_PROFILES=services,ngrok,stripe docker-compose up", + "dev:services:all": "COMPOSE_PROFILES=services,ngrok docker-compose up", "dev:workers:test": "nx test workers --skip-nx-cache --runInBand", "dev:server:test": "nx test server --skip-nx-cache --runInBand", "dev:test:unit": "yarn dev:ci:test --testPathPattern='^(?!.*integration).*$' --verbose --skip-nx-cache", From 9ea4333260a354adc6052cff9f3d70d24c33208e Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 17:24:35 -0600 Subject: [PATCH 03/15] update README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 396cf5b1..a3c8eca8 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,14 @@ Then run the following yarn commands: ``` yarn install -yarn run dev:services +yarn run dev:services:all yarn prisma:migrate:dev yarn prisma:seed yarn dev ``` ## Contributing + To contribute, please see our [contribution guide](https://github.com/maybe-finance/maybe/blob/main/CONTRIBUTING.md). ## High-priority issues @@ -91,9 +92,9 @@ To pull market data in (for investments), you'll need a Polygon.io API key. You - [Handling money](https://github.com/maybe-finance/maybe/wiki/Handling-Money) - [REST API](https://github.com/maybe-finance/maybe/wiki/REST-API) - ## Repo Activity -![Repo Activity](https://repobeats.axiom.co/api/embed/7866c9790deba0baf63ca1688b209130b306ea4e.svg "Repobeats analytics image") + +![Repo Activity](https://repobeats.axiom.co/api/embed/7866c9790deba0baf63ca1688b209130b306ea4e.svg 'Repobeats analytics image') ## Credits From cdfed8233ff2d1070d20c2de922b3dc52525e28b Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 17:32:56 -0600 Subject: [PATCH 04/15] run docker in background --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 64d4aaeb..96c52336 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "nx run-many --target serve --projects=client,server,workers --parallel --host 0.0.0.0 --nx-bail=true --maxParallel=100", "dev:services": "COMPOSE_PROFILES=services docker-compose up -d", - "dev:services:all": "COMPOSE_PROFILES=services,ngrok docker-compose up", + "dev:services:all": "COMPOSE_PROFILES=services,ngrok docker-compose up -d", "dev:workers:test": "nx test workers --skip-nx-cache --runInBand", "dev:server:test": "nx test server --skip-nx-cache --runInBand", "dev:test:unit": "yarn dev:ci:test --testPathPattern='^(?!.*integration).*$' --verbose --skip-nx-cache", From db1c75bff8af338205fbb53623cf19e3b66c1ff9 Mon Sep 17 00:00:00 2001 From: Josh Pigford Date: Mon, 15 Jan 2024 18:51:47 -0600 Subject: [PATCH 05/15] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 1 + ...eature_request.md => feature-request-or-improvement.md} | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) rename .github/ISSUE_TEMPLATE/{feature_request.md => feature-request-or-improvement.md} (76%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 67e8d549..c133fbea 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -4,6 +4,7 @@ about: Create a report to help us improve title: '' labels: bug assignees: '' + --- **Describe the bug** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature-request-or-improvement.md similarity index 76% rename from .github/ISSUE_TEMPLATE/feature_request.md rename to .github/ISSUE_TEMPLATE/feature-request-or-improvement.md index 2f28cead..3fb9fcaf 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature-request-or-improvement.md @@ -1,12 +1,13 @@ --- -name: Feature request -about: Suggest an idea for this project +name: Feature request or improvement +about: Suggest a new feature or improvement title: '' labels: '' assignees: '' + --- -**Is your feature request related to a problem? Please describe.** +**Is your request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** From d8fab13d8548333e879508083db09b2a0b0c740c Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 19:00:47 -0600 Subject: [PATCH 06/15] more work --- .../src/providers/teller/teller.etl.ts | 75 ++++++++----------- libs/teller-api/src/types/accounts.ts | 2 +- prisma/schema.prisma | 2 + 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index b1491a35..52443779 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -1,9 +1,8 @@ import type { AccountConnection, PrismaClient } from '@prisma/client' import type { Logger } from 'winston' import { SharedUtil, AccountUtil, type SharedType } from '@maybe-finance/shared' -import type { FinicityApi, FinicityTypes } from '@maybe-finance/finicity-api' import type { TellerApi, TellerTypes } from '@maybe-finance/teller-api' -import { DbUtil, TellerUtil, type IETL } from '@maybe-finance/server/shared' +import { DbUtil, TellerUtil, type IETL, type ICryptoService } from '@maybe-finance/server/shared' import { Prisma } from '@prisma/client' import _ from 'lodash' import { DateTime } from 'luxon' @@ -20,19 +19,28 @@ export type TellerData = { transactionsDateRange: SharedType.DateRange } -type Connection = Pick +type Connection = Pick< + AccountConnection, + 'id' | 'userId' | 'tellerInstitutionId' | 'tellerAccessToken' +> export class TellerETL implements IETL { public constructor( private readonly logger: Logger, private readonly prisma: PrismaClient, - private readonly teller: Pick + private readonly teller: Pick, + private readonly crypto: ICryptoService ) {} async extract(connection: Connection): Promise { if (!connection.tellerInstitutionId) { throw new Error(`connection ${connection.id} is missing tellerInstitutionId`) } + if (!connection.tellerAccessToken) { + throw new Error(`connection ${connection.id} is missing tellerAccessToken`) + } + + const accessToken = this.crypto.decrypt(connection.tellerAccessToken) const user = await this.prisma.user.findUniqueOrThrow({ where: { id: connection.userId }, @@ -52,12 +60,11 @@ export class TellerETL implements IETL { end: DateTime.now(), } - const accounts = await this._extractAccounts(user.tellerUserId) + const accounts = await this._extractAccounts(accessToken) const transactions = await this._extractTransactions( - user.tellerUserId, - accounts.map((a) => a.id), - transactionsDateRange + accessToken, + accounts.map((a) => a.id) ) this.logger.info( @@ -89,12 +96,8 @@ export class TellerETL implements IETL { }) } - private async _extractAccounts(tellerUserId: string) { - const { accounts } = await this.teller.getAccounts({ accessToken: undefined }) - - return accounts.filter( - (a) => a.institutionLoginId.toString() === institutionLoginId && a.currency === 'USD' - ) + private async _extractAccounts(accessToken: string) { + return await this.teller.getAccounts({ accessToken }) } private _loadAccounts(connection: Connection, { accounts }: Pick) { @@ -111,36 +114,30 @@ export class TellerETL implements IETL { create: { type: TellerUtil.getType(tellerAccount.type), provider: 'teller', - categoryProvider: PlaidUtil.plaidTypesToCategory(plaidAccount.type), - subcategoryProvider: plaidAccount.subtype ?? 'other', + categoryProvider: TellerUtil.tellerTypesToCategory(tellerAccount.type), + subcategoryProvider: tellerAccount.subtype ?? 'other', accountConnectionId: connection.id, - plaidAccountId: plaidAccount.account_id, + tellerAccountId: tellerAccount.id, name: tellerAccount.name, - plaidType: tellerAccount.type, - plaidSubtype: tellerAccount.subtype, - mask: plaidAccount.mask, - ...PlaidUtil.getAccountBalanceData( - plaidAccount.balances, - plaidAccount.type - ), + tellerType: tellerAccount.type, + tellerSubtype: tellerAccount.subtype, + mask: tellerAccount.last_four, + ...TellerUtil.getAccountBalanceData(tellerAccount, tellerAccount.type), }, update: { type: TellerUtil.getType(tellerAccount.type), - categoryProvider: PlaidUtil.plaidTypesToCategory(tellerAccount.type), + categoryProvider: TellerUtil.tellerTypesToCategory(tellerAccount.type), subcategoryProvider: tellerAccount.subtype ?? 'other', - plaidType: tellerAccount.type, - plaidSubtype: tellerAccount.subtype, + tellerType: tellerAccount.type, + tellerSubtype: tellerAccount.subtype, ..._.omit( - PlaidUtil.getAccountBalanceData( - plaidAccount.balances, - plaidAccount.type - ), + TellerUtil.getAccountBalanceData(tellerAccount, tellerAccount.type), ['currentBalanceStrategy', 'availableBalanceStrategy'] ), }, }) }), - // any accounts that are no longer in Plaid should be marked inactive + // any accounts that are no longer in Teller should be marked inactive this.prisma.account.updateMany({ where: { accountConnectionId: connection.id, @@ -156,25 +153,17 @@ export class TellerETL implements IETL { ] } - private async _extractTransactions( - customerId: string, - accountIds: string[], - dateRange: SharedType.DateRange - ) { + private async _extractTransactions(accessToken: string, accountIds: string[]) { const accountTransactions = await Promise.all( accountIds.map((accountId) => SharedUtil.paginate({ - pageSize: 1000, // https://api-reference.finicity.com/#/rest/api-endpoints/transactions/get-customer-account-transactions + pageSize: 1000, // TODO: Check with Teller on max page size fetchData: async (offset, count) => { const transactions = await SharedUtil.withRetry( () => this.teller.getTransactions({ accountId, - accessToken: undefined, - fromDate: dateRange.start.toUnixInteger(), - toDate: dateRange.end.toUnixInteger(), - start: offset + 1, - limit: count, + accessToken: accessToken, }), { maxRetries: 3, diff --git a/libs/teller-api/src/types/accounts.ts b/libs/teller-api/src/types/accounts.ts index e3ae9075..594fb6a5 100644 --- a/libs/teller-api/src/types/accounts.ts +++ b/libs/teller-api/src/types/accounts.ts @@ -48,7 +48,7 @@ interface CreditAccount extends BaseAccount { export type Account = DepositoryAccount | CreditAccount -export type GetAccountsResponse = { accounts: Account[] } +export type GetAccountsResponse = Account[] export type GetAccountResponse = Account export type DeleteAccountResponse = void diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 160ea264..007cad61 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -109,6 +109,7 @@ enum AccountProvider { user plaid finicity + teller } enum AccountBalanceStrategy { @@ -162,6 +163,7 @@ model Account { // teller data tellerAccountId String? @map("teller_account_id") tellerType String? @map("teller_type") + tellerSubtype String? @map("teller_subtype") // manual account data vehicleMeta Json? @map("vehicle_meta") @db.JsonB From 2f3f0e7e53db0e4b42bb73c958dbf9dda54baccf Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 19:05:33 -0600 Subject: [PATCH 07/15] override default MIN_SUPPORTED_DATE on date picker --- libs/client/features/src/onboarding/steps/Profile.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/client/features/src/onboarding/steps/Profile.tsx b/libs/client/features/src/onboarding/steps/Profile.tsx index 7201ead9..1bb5d704 100644 --- a/libs/client/features/src/onboarding/steps/Profile.tsx +++ b/libs/client/features/src/onboarding/steps/Profile.tsx @@ -144,6 +144,7 @@ function ProfileForm({ title, onSubmit, defaultValues }: ProfileViewProps) { From ad30b02ebcf2a7c2eea95efc46431b7e7fdaf582 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 20:28:35 -0600 Subject: [PATCH 08/15] finish initial pass on teller etl service --- .../src/providers/teller/teller.etl.ts | 33 ++++++++++++----- libs/server/shared/src/utils/teller-utils.ts | 35 ++++++++++++++++++- libs/teller-api/src/types/accounts.ts | 5 +++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index 52443779..241934bb 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -14,7 +14,7 @@ export type TellerRawData = { } export type TellerData = { - accounts: TellerTypes.Account[] + accounts: TellerTypes.AccountWithBalances[] transactions: TellerTypes.Transaction[] transactionsDateRange: SharedType.DateRange } @@ -28,7 +28,10 @@ export class TellerETL implements IETL { public constructor( private readonly logger: Logger, private readonly prisma: PrismaClient, - private readonly teller: Pick, + private readonly teller: Pick< + TellerApi, + 'getAccounts' | 'getTransactions' | 'getAccountBalances' + >, private readonly crypto: ICryptoService ) {} @@ -97,13 +100,25 @@ export class TellerETL implements IETL { } private async _extractAccounts(accessToken: string) { - return await this.teller.getAccounts({ accessToken }) + const accounts = await this.teller.getAccounts({ accessToken }) + const accountsWithBalances = await Promise.all( + accounts.map(async (a) => { + const balance = await this.teller.getAccountBalances({ + accountId: a.id, + accessToken, + }) + return { ...a, balance } + }) + ) + return accountsWithBalances } private _loadAccounts(connection: Connection, { accounts }: Pick) { return [ // upsert accounts ...accounts.map((tellerAccount) => { + const type = TellerUtil.getType(tellerAccount.type) + const classification = AccountUtil.getClassification(type) return this.prisma.account.upsert({ where: { accountConnectionId_tellerAccountId: { @@ -122,7 +137,7 @@ export class TellerETL implements IETL { tellerType: tellerAccount.type, tellerSubtype: tellerAccount.subtype, mask: tellerAccount.last_four, - ...TellerUtil.getAccountBalanceData(tellerAccount, tellerAccount.type), + ...TellerUtil.getAccountBalanceData(tellerAccount, classification), }, update: { type: TellerUtil.getType(tellerAccount.type), @@ -130,10 +145,10 @@ export class TellerETL implements IETL { subcategoryProvider: tellerAccount.subtype ?? 'other', tellerType: tellerAccount.type, tellerSubtype: tellerAccount.subtype, - ..._.omit( - TellerUtil.getAccountBalanceData(tellerAccount, tellerAccount.type), - ['currentBalanceStrategy', 'availableBalanceStrategy'] - ), + ..._.omit(TellerUtil.getAccountBalanceData(tellerAccount, classification), [ + 'currentBalanceStrategy', + 'availableBalanceStrategy', + ]), }, }) }), @@ -158,7 +173,7 @@ export class TellerETL implements IETL { accountIds.map((accountId) => SharedUtil.paginate({ pageSize: 1000, // TODO: Check with Teller on max page size - fetchData: async (offset, count) => { + fetchData: async () => { const transactions = await SharedUtil.withRetry( () => this.teller.getTransactions({ diff --git a/libs/server/shared/src/utils/teller-utils.ts b/libs/server/shared/src/utils/teller-utils.ts index 58ce8b88..a8e8e3ee 100644 --- a/libs/server/shared/src/utils/teller-utils.ts +++ b/libs/server/shared/src/utils/teller-utils.ts @@ -1,4 +1,10 @@ -import { AccountCategory, AccountType } from '@prisma/client' +import { + Prisma, + AccountCategory, + AccountType, + type AccountClassification, + type Account, +} from '@prisma/client' import type { TellerTypes } from '@maybe-finance/teller-api' import { Duration } from 'luxon' @@ -7,6 +13,33 @@ import { Duration } from 'luxon' */ export const TELLER_WINDOW_MAX = Duration.fromObject({ years: 1 }) +export function getAccountBalanceData( + { balances, currency }: Pick, + classification: AccountClassification +): Pick< + Account, + | 'currentBalanceProvider' + | 'currentBalanceStrategy' + | 'availableBalanceProvider' + | 'availableBalanceStrategy' + | 'currencyCode' +> { + // Flip balance values to positive for liabilities + const sign = classification === 'liability' ? -1 : 1 + + return { + currentBalanceProvider: new Prisma.Decimal( + balances.ledger ? sign * Number(balances.ledger) : 0 + ), + currentBalanceStrategy: 'current', + availableBalanceProvider: new Prisma.Decimal( + balances.available ? sign * Number(balances.available) : 0 + ), + availableBalanceStrategy: 'available', + currencyCode: currency, + } +} + export function getType(type: TellerTypes.AccountTypes): AccountType { switch (type) { case 'depository': diff --git a/libs/teller-api/src/types/accounts.ts b/libs/teller-api/src/types/accounts.ts index 594fb6a5..5df29953 100644 --- a/libs/teller-api/src/types/accounts.ts +++ b/libs/teller-api/src/types/accounts.ts @@ -1,4 +1,5 @@ // https://teller.io/docs/api/accounts +import type { AccountBalance } from './account-balance' import type { AuthenticatedRequest } from './authentication' export type AccountTypes = 'depository' | 'credit' @@ -48,6 +49,10 @@ interface CreditAccount extends BaseAccount { export type Account = DepositoryAccount | CreditAccount +export type AccountWithBalances = Account & { + balances: AccountBalance +} + export type GetAccountsResponse = Account[] export type GetAccountResponse = Account export type DeleteAccountResponse = void From 87556ede442cfa44ca4b6c2a44596e2a9629eff8 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 20:31:12 -0600 Subject: [PATCH 09/15] add migration --- .../migration.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 prisma/migrations/20240116023100_add_additional_teller_fields/migration.sql diff --git a/prisma/migrations/20240116023100_add_additional_teller_fields/migration.sql b/prisma/migrations/20240116023100_add_additional_teller_fields/migration.sql new file mode 100644 index 00000000..1627c0f0 --- /dev/null +++ b/prisma/migrations/20240116023100_add_additional_teller_fields/migration.sql @@ -0,0 +1,5 @@ +-- AlterEnum +ALTER TYPE "AccountProvider" ADD VALUE 'teller'; + +-- AlterTable +ALTER TABLE "account" ADD COLUMN "teller_subtype" TEXT; From e139df7a7d7bb7480ccaacc59b3b9202365dce00 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 20:37:19 -0600 Subject: [PATCH 10/15] add missing crypto service requirement --- apps/server/src/app/lib/endpoint.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/app/lib/endpoint.ts b/apps/server/src/app/lib/endpoint.ts index c0341afa..8e01b00d 100644 --- a/apps/server/src/app/lib/endpoint.ts +++ b/apps/server/src/app/lib/endpoint.ts @@ -54,7 +54,6 @@ import { ProjectionCalculator, StripeWebhookHandler, } from '@maybe-finance/server/features' -import { SharedType } from '@maybe-finance/shared' import prisma from './prisma' import plaid, { getPlaidWebhookUrl } from './plaid' import finicity, { getFinicityTxPushUrl, getFinicityWebhookUrl } from './finicity' @@ -150,7 +149,7 @@ const tellerService = new TellerService( logger.child({ service: 'TellerService' }), prisma, teller, - new TellerETL(logger.child({ service: 'TellerETL' }), prisma, teller), + new TellerETL(logger.child({ service: 'TellerETL' }), prisma, teller, cryptoService), cryptoService, getTellerWebhookUrl(), env.NX_TELLER_ENV === 'sandbox' From f612971efe17c395d1dace3990c026e77b1e3f5a Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 20:45:19 -0600 Subject: [PATCH 11/15] add worker service --- apps/workers/src/app/lib/di.ts | 14 ++++++++++++++ apps/workers/src/app/lib/teller.ts | 5 +++++ apps/workers/src/env.ts | 1 + 3 files changed, 20 insertions(+) create mode 100644 apps/workers/src/app/lib/teller.ts diff --git a/apps/workers/src/app/lib/di.ts b/apps/workers/src/app/lib/di.ts index 55f3341a..987ebedd 100644 --- a/apps/workers/src/app/lib/di.ts +++ b/apps/workers/src/app/lib/di.ts @@ -28,6 +28,8 @@ import { LoanBalanceSyncStrategy, PlaidETL, PlaidService, + TellerETL, + TellerService, SecurityPricingProcessor, SecurityPricingService, TransactionBalanceSyncStrategy, @@ -55,6 +57,7 @@ import logger from './logger' import prisma from './prisma' import plaid from './plaid' import finicity from './finicity' +import teller from './teller' import postmark from './postmark' import stripe from './stripe' import env from '../../env' @@ -124,11 +127,22 @@ const finicityService = new FinicityService( env.NX_FINICITY_ENV === 'sandbox' ) +const tellerService = new TellerService( + logger.child({ service: 'TellerService' }), + prisma, + teller, + new TellerETL(logger.child({ service: 'TellerETL' }), prisma, teller, cryptoService), + cryptoService, + '', + env.NX_TELLER_ENV === 'sandbox' +) + // account-connection const accountConnectionProviderFactory = new AccountConnectionProviderFactory({ plaid: plaidService, finicity: finicityService, + teller: tellerService, }) const transactionStrategy = new TransactionBalanceSyncStrategy( diff --git a/apps/workers/src/app/lib/teller.ts b/apps/workers/src/app/lib/teller.ts new file mode 100644 index 00000000..00a7beb1 --- /dev/null +++ b/apps/workers/src/app/lib/teller.ts @@ -0,0 +1,5 @@ +import { TellerApi } from '@maybe-finance/teller-api' + +const teller = new TellerApi() + +export default teller diff --git a/apps/workers/src/env.ts b/apps/workers/src/env.ts index df560b4e..f58963bf 100644 --- a/apps/workers/src/env.ts +++ b/apps/workers/src/env.ts @@ -17,6 +17,7 @@ const envSchema = z.object({ NX_TELLER_SIGNING_SECRET: z.string().default('REPLACE_THIS'), NX_TELLER_APP_ID: z.string().default('REPLACE_THIS'), + NX_TELLER_ENV: z.string().default('sandbox'), NX_SENTRY_DSN: z.string().optional(), NX_SENTRY_ENV: z.string().optional(), From bb35133b185aa8d74d6a7f7d5bd548f49249f663 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Mon, 15 Jan 2024 22:06:32 -0600 Subject: [PATCH 12/15] clean up --- libs/server/features/src/providers/teller/teller.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.service.ts b/libs/server/features/src/providers/teller/teller.service.ts index bc6a4237..e884639f 100644 --- a/libs/server/features/src/providers/teller/teller.service.ts +++ b/libs/server/features/src/providers/teller/teller.service.ts @@ -100,11 +100,11 @@ export class TellerService implements IAccountConnectionProvider, IInstitutionPr maxRetries: 3, onError: (error, attempt) => { this.logger.error( - `Plaid fetch institutions request failed attempt=${attempt} offset=${offset} count=${count}`, + `Teller fetch institutions request failed attempt=${attempt} offset=${offset} count=${count}`, { error: ErrorUtil.parseError(error) } ) - return !ErrorUtil.isPlaidError(error) || error.response.status >= 500 + return !ErrorUtil.isTellerError(error) || error.response.status >= 500 }, } ), From c9842544787ad138731ab6b42a41b920fbd1bb54 Mon Sep 17 00:00:00 2001 From: Josh Pigford Date: Tue, 16 Jan 2024 09:02:40 -0600 Subject: [PATCH 13/15] .env organization --- .env.example | 51 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 8c6641d2..99acc901 100644 --- a/.env.example +++ b/.env.example @@ -1,28 +1,49 @@ -# Used by `prisma` commands -NX_DATABASE_URL=postgresql://maybe:maybe@localhost:5433/maybe_local -NX_DATABASE_SECRET= - -# Market data API keys (https://polygon.io) -NX_POLYGON_API_KEY= - -# If using free ngrok account for webhooks -NGROK_AUTH_TOKEN= +######################################################################## +# AUTHENTICATION +######################################################################## # Generate a new secret using openssl rand -base64 32 NEXTAUTH_SECRET= NEXTAUTH_URL=http://localhost:4200 NX_NEXTAUTH_URL=http://localhost:4200 -NX_PLAID_SECRET= -NX_FINICITY_APP_KEY= -NX_FINICITY_PARTNER_SECRET= +######################################################################## +# WEBHOOKS +######################################################################## -# Teller API keys (https://teller.io) +# We use ngrok to expose a local development environment to the internet +# You can sign up for a free account and get an API key at https://ngrok.com +NGROK_AUTH_TOKEN= + +######################################################################## +# DATABASE +######################################################################## + +NX_DATABASE_URL=postgresql://maybe:maybe@localhost:5433/maybe_local +NX_DATABASE_SECRET= + +######################################################################## +# FINANICAL DATA SOURCES +######################################################################## + +# Market Data +# We use Polygon.io for market data. You can sign up for a free account +# and get an API key for individual use at https://polygon.io +NX_POLYGON_API_KEY= + +# Automated banking data +# We use Teller.io for automated banking data. You can sign up for a free +# account and get a free API key at https://teller.io NX_TELLER_SIGNING_SECRET= NX_TELLER_APP_ID= NX_TELLER_ENV=sandbox -# Email credentials +######################################################################## +# EMAIL +######################################################################## + +# We use Postmark for transactional emails. You can sign up for a free account +# and get a free API key at https://postmarkapp.com NX_POSTMARK_FROM_ADDRESS=account@example.com NX_POSTMARK_REPLY_TO_ADDRESS=support@example.com -NX_POSTMARK_API_TOKEN= +NX_POSTMARK_API_TOKEN= \ No newline at end of file From 2c41c7ea90bc59d69d988a00604605ee947ede03 Mon Sep 17 00:00:00 2001 From: Josh Pigford Date: Tue, 16 Jan 2024 09:14:22 -0600 Subject: [PATCH 14/15] Re-added old variables that are still required --- .env.example | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 99acc901..abd0167b 100644 --- a/.env.example +++ b/.env.example @@ -46,4 +46,15 @@ NX_TELLER_ENV=sandbox # and get a free API key at https://postmarkapp.com NX_POSTMARK_FROM_ADDRESS=account@example.com NX_POSTMARK_REPLY_TO_ADDRESS=support@example.com -NX_POSTMARK_API_TOKEN= \ No newline at end of file +NX_POSTMARK_API_TOKEN= + + +######################################################################## +# DEPRECATING +# We're in the process of removing code that requires the following +# environment variables. They will be removed in a future release, but +# for now, they are still required. +######################################################################## +NX_PLAID_SECRET= +NX_FINICITY_APP_KEY= +NX_FINICITY_PARTNER_SECRET= \ No newline at end of file From 2b7924247e2323812ef918b7ac1a074a242520ae Mon Sep 17 00:00:00 2001 From: Josh Pigford Date: Tue, 16 Jan 2024 09:34:13 -0600 Subject: [PATCH 15/15] Re-implemented some of the billing UI/UX This is in preparation for a hosted version of Maybe and ultimately will be hidden/disabled for all self-hosters. Just getting the UI bits back in there as we'd removed them when we shut down last year. --- apps/client/pages/upgrade.tsx | 33 ++++++++++--------- .../features/src/layout/DesktopLayout.tsx | 7 ++-- .../features/src/layout/MobileLayout.tsx | 13 +++++--- .../src/user-billing/BillingPreferences.tsx | 33 +++++-------------- 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/apps/client/pages/upgrade.tsx b/apps/client/pages/upgrade.tsx index a0edd511..05ca2feb 100644 --- a/apps/client/pages/upgrade.tsx +++ b/apps/client/pages/upgrade.tsx @@ -1,20 +1,23 @@ -import { Button } from '@maybe-finance/design-system' -import Link from 'next/link' +import { UpgradeTakeover } from '@maybe-finance/client/features' +import { useUserApi } from '@maybe-finance/client/shared' +import { useRouter } from 'next/router' export default function UpgradePage() { + const router = useRouter() + + const { useSubscription } = useUserApi() + const subscription = useSubscription() + return ( -
-

Signups have been disabled.

-

- Maybe will be shutting down on July 31.{' '} - - Details and FAQ - -

- -
+ + router.push( + !subscription.data || subscription.data?.subscribed + ? '/' + : '/settings?tab=billing' + ) + } + /> ) } diff --git a/libs/client/features/src/layout/DesktopLayout.tsx b/libs/client/features/src/layout/DesktopLayout.tsx index e61fbdac..21023d6b 100644 --- a/libs/client/features/src/layout/DesktopLayout.tsx +++ b/libs/client/features/src/layout/DesktopLayout.tsx @@ -23,6 +23,7 @@ import { } from 'react-icons/ri' import { Button, Tooltip } from '@maybe-finance/design-system' import { MenuPopover } from './MenuPopover' +import { UpgradePrompt } from '../user-billing' import { SidebarOnboarding } from '../onboarding' import { useSession } from 'next-auth/react' @@ -227,7 +228,7 @@ export function DesktopLayout({ children, sidebar }: DesktopLayoutProps) { className="p-3 text-base bg-gray-700 rounded-lg cursor-pointer hover:bg-gray-600" onClick={() => setOnboardingExpanded(true)} > -
+

Getting started

{onboarding.data.isComplete && ( @@ -337,6 +338,8 @@ function DefaultContent({ {onboarding && onboarding} + +

{name ?? ''}

diff --git a/libs/client/features/src/layout/MobileLayout.tsx b/libs/client/features/src/layout/MobileLayout.tsx index 206df618..585af1dd 100644 --- a/libs/client/features/src/layout/MobileLayout.tsx +++ b/libs/client/features/src/layout/MobileLayout.tsx @@ -12,6 +12,7 @@ import { Button } from '@maybe-finance/design-system' import { MenuPopover } from './MenuPopover' import Link from 'next/link' import { useRouter } from 'next/router' +import { UpgradePrompt } from '../user-billing' import { ProfileCircle } from '@maybe-finance/client/shared' import { usePopoutContext, LayoutContextProvider } from '@maybe-finance/client/shared' import classNames from 'classnames' @@ -90,10 +91,10 @@ function NavItem({ className="absolute inset-0" transition={{ duration: 0.3 }} > -
+
)} - + {label}
@@ -138,7 +139,7 @@ export function MobileLayout({ children, sidebar }: MobileLayoutProps) { >