diff --git a/apps/client/pages/_app.tsx b/apps/client/pages/_app.tsx index ae4d14db..8b8c1c4a 100644 --- a/apps/client/pages/_app.tsx +++ b/apps/client/pages/_app.tsx @@ -44,7 +44,7 @@ const WithAuth = function ({ children }: PropsWithChildren) { } }, [session, status, router]) - if (session) { + if (session && status === 'authenticated') { return ( diff --git a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts index 2dcbcb8a..17c81c4e 100644 --- a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts +++ b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts @@ -1,4 +1,5 @@ import type { User } from '@prisma/client' +import { InvestmentTransactionCategory } from '@prisma/client' import { PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' @@ -131,6 +132,7 @@ describe('balance sync strategies', () => { quantity: 10, price: 10, plaidType: 'buy', + category: InvestmentTransactionCategory.buy, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -140,6 +142,7 @@ describe('balance sync strategies', () => { quantity: 5, price: 10, plaidType: 'sell', + category: InvestmentTransactionCategory.sell, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -147,6 +150,7 @@ describe('balance sync strategies', () => { amount: 50, quantity: 50, price: 1, + category: InvestmentTransactionCategory.other, }, ], }, diff --git a/apps/server/src/app/__tests__/connection.integration.spec.ts b/apps/server/src/app/__tests__/connection.integration.spec.ts index 7eb81099..a8bfa721 100644 --- a/apps/server/src/app/__tests__/connection.integration.spec.ts +++ b/apps/server/src/app/__tests__/connection.integration.spec.ts @@ -1,21 +1,14 @@ import type { AxiosInstance } from 'axios' -import type { SharedType } from '@maybe-finance/shared' -import type { Prisma, AccountConnection, AccountSyncStatus, User } from '@prisma/client' -import type { ItemRemoveResponse } from 'plaid' +import type { Prisma, AccountConnection, User } from '@prisma/client' +import { AccountConnectionType, AccountSyncStatus } from '@prisma/client' import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import prisma from '../lib/prisma' -import { TestUtil } from '@maybe-finance/shared' import { InMemoryQueue } from '@maybe-finance/server/shared' -import { default as _plaid } from '../lib/plaid' import nock from 'nock' import { resetUser } from './utils/user' -jest.mock('../middleware/validate-plaid-jwt.ts') -jest.mock('plaid') - -// For TypeScript support -const plaid = jest.mocked(_plaid) +jest.mock('../lib/teller.ts') const authId = '__TEST_USER_ID__' let axios: AxiosInstance @@ -49,13 +42,13 @@ beforeEach(async () => { connectionData = { data: { name: 'Chase Test', - type: 'plaid' as SharedType.AccountConnectionType, - plaidItemId: 'test-plaid-item-server', - plaidInstitutionId: 'ins_3', - plaidAccessToken: - 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', - userId: user!.id, - syncStatus: 'PENDING' as AccountSyncStatus, + type: AccountConnectionType.teller, + tellerEnrollmentId: 'test-teller-item-workers', + tellerInstitutionId: 'chase_test', + tellerAccessToken: + 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', // need correct encoding here + userId: user.id, + syncStatus: AccountSyncStatus.PENDING, }, } @@ -91,16 +84,9 @@ describe('/v1/connections API', () => { }) it('DELETE /:id', async () => { - plaid.itemRemove.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - request_id: 'test request id', - }) - ) - const res = await axios.delete(`/connections/${connection.id}`) expect(res.status).toEqual(200) - expect(plaid.itemRemove).toHaveBeenCalledTimes(1) const res2 = await axios.get(`/connections/${connection.id}`) diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index 9dbe0433..fe32ffef 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,5 +1,5 @@ import type { User } from '@prisma/client' -import { Prisma, PrismaClient } from '@prisma/client' +import { InvestmentTransactionCategory, Prisma, PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import type { @@ -307,6 +307,7 @@ describe('insight service', () => { price: 100, plaidType: 'buy', plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -318,6 +319,7 @@ describe('insight service', () => { price: 200, plaidType: 'buy', plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -329,6 +331,7 @@ describe('insight service', () => { price: 0, plaidType: 'cash', plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, { accountId: account.id, @@ -340,6 +343,7 @@ describe('insight service', () => { price: 0, plaidType: 'cash', plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, ], }) diff --git a/apps/server/src/app/__tests__/net-worth.integration.spec.ts b/apps/server/src/app/__tests__/net-worth.integration.spec.ts index 5e5f7465..c17129ab 100644 --- a/apps/server/src/app/__tests__/net-worth.integration.spec.ts +++ b/apps/server/src/app/__tests__/net-worth.integration.spec.ts @@ -5,7 +5,6 @@ import { DateTime } from 'luxon' import { PgService } from '@maybe-finance/server/shared' import { AccountQueryService, UserService } from '@maybe-finance/server/features' import { resetUser } from './utils/user' -jest.mock('plaid') const prisma = new PrismaClient() diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 47e26cc3..32ed7915 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -1,5 +1,5 @@ import type { PrismaClient, User } from '@prisma/client' -import { Prisma } from '@prisma/client' +import { InvestmentTransactionCategory, Prisma } from '@prisma/client' import _ from 'lodash' import { DateTime } from 'luxon' import { parseCsv } from './csv' @@ -20,6 +20,14 @@ const portfolios: Record> = }, } +const investmentTransactionCategoryByType: Record = { + BUY: InvestmentTransactionCategory.buy, + SELL: InvestmentTransactionCategory.sell, + DIVIDEND: InvestmentTransactionCategory.dividend, + DEPOSIT: InvestmentTransactionCategory.transfer, + WITHDRAW: InvestmentTransactionCategory.transfer, +} + export async function createTestInvestmentAccount( prisma: PrismaClient, user: User, @@ -35,7 +43,7 @@ export async function createTestInvestmentAccount( join(__dirname, `../test-data/${portfolio}/holdings.csv`) ) - const [_deleted, ...securities] = await prisma.$transaction([ + const [, ...securities] = await prisma.$transaction([ prisma.security.deleteMany({ where: { symbol: { @@ -72,7 +80,7 @@ export async function createTestInvestmentAccount( .value(), ]) - const account = await prisma.account.create({ + return prisma.account.create({ data: { ...portfolios[portfolio], userId: user.id, @@ -128,12 +136,13 @@ export async function createTestInvestmentAccount( : it.type === 'SELL' ? 'sell' : undefined, + category: + investmentTransactionCategoryByType[it.type] ?? + InvestmentTransactionCategory.other, } }), }, }, }, }) - - return account } diff --git a/apps/workers/src/app/__tests__/queue.integration.spec.ts b/apps/workers/src/app/__tests__/queue.integration.spec.ts index 07882ee2..2eff8ced 100644 --- a/apps/workers/src/app/__tests__/queue.integration.spec.ts +++ b/apps/workers/src/app/__tests__/queue.integration.spec.ts @@ -1,26 +1,21 @@ // ===================================================== // Keep these imports above the rest to avoid errors // ===================================================== -import type { SharedType } from '@maybe-finance/shared' -import type { AccountsGetResponse, TransactionsGetResponse } from 'plaid' -import type { AccountConnection, User } from '@prisma/client' -import { TestUtil } from '@maybe-finance/shared' -import { PlaidTestData } from '../../../../../tools/test-data' -import { Prisma } from '@prisma/client' +import { TellerGenerator } from 'tools/generators' +import type { User, AccountConnection } from '@prisma/client' +import { AccountConnectionType } from '@prisma/client' import prisma from '../lib/prisma' -import { default as _plaid } from '../lib/plaid' -import nock from 'nock' -import { DateTime } from 'luxon' +import { default as _teller } from '../lib/teller' import { resetUser } from './helpers/user.test-helper' +import { Interval } from 'luxon' // Import the workers process import '../../main' -import { queueService, securityPricingService } from '../lib/di' - -jest.mock('plaid') +import { queueService } from '../lib/di' // For TypeScript support -const plaid = jest.mocked(_plaid) +jest.mock('../lib/teller') +const teller = jest.mocked(_teller) let user: User | null let connection: AccountConnection @@ -30,25 +25,6 @@ if (process.env.IS_VSCODE_DEBUG === 'true') { jest.setTimeout(100000) } -beforeAll(() => { - nock.disableNetConnect() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/AAPL/range/1/day')) - .reply(200, PlaidTestData.AAPL) - .persist() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/WMT/range/1/day')) - .reply(200, PlaidTestData.WMT) - .persist() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/VOO/range/1/day')) - .reply(200, PlaidTestData.VOO) - .persist() -}) - beforeEach(async () => { jest.clearAllMocks() @@ -57,10 +33,10 @@ beforeEach(async () => { connection = await prisma.accountConnection.create({ data: { name: 'Chase Test', - type: 'plaid' as SharedType.AccountConnectionType, - plaidItemId: 'test-plaid-item-workers', - plaidInstitutionId: 'ins_3', - plaidAccessToken: + type: AccountConnectionType.teller, + tellerEnrollmentId: 'test-teller-item-workers', + tellerInstitutionId: 'chase_test', + tellerAccessToken: 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', // need correct encoding here userId: user.id, syncStatus: 'PENDING', @@ -84,7 +60,7 @@ describe('Message queue tests', () => { it('Should handle sync errors', async () => { const syncQueue = queueService.getQueue('sync-account-connection') - plaid.accountsGet.mockRejectedValueOnce('forced error for Jest tests') + teller.getAccounts.mockRejectedValueOnce(new Error('forced error for Jest tests')) await syncQueue.add('sync-connection', { accountConnectionId: connection.id }) @@ -92,7 +68,7 @@ describe('Message queue tests', () => { where: { id: connection.id }, }) - expect(plaid.accountsGet).toHaveBeenCalledTimes(1) + expect(teller.getAccounts).toHaveBeenCalledTimes(1) expect(updatedConnection?.status).toEqual('ERROR') }) @@ -117,28 +93,23 @@ describe('Message queue tests', () => { const syncQueue = queueService.getQueue('sync-account-connection') // Mock will return a basic banking checking account - plaid.accountsGet.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - accounts: [PlaidTestData.checkingAccount], - item: PlaidTestData.item, - request_id: 'bkVE1BHWMAZ9Rnr', - }) as any - ) + const mockAccounts = TellerGenerator.generateAccountsWithBalances({ + count: 1, + institutionId: 'chase_test', + enrollmentId: 'test-teller-item-workers', + institutionName: 'Chase Test', + accountType: 'depository', + accountSubType: 'checking', + }) + teller.getAccounts.mockResolvedValueOnce(mockAccounts) - plaid.transactionsGet.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - accounts: [PlaidTestData.checkingAccount], - transactions: PlaidTestData.checkingTransactions, - item: PlaidTestData.item, - total_transactions: PlaidTestData.checkingTransactions.length, - request_id: '45QSn', - }) as any - ) + const mockTransactions = TellerGenerator.generateTransactions(10, mockAccounts[0].id) + teller.getTransactions.mockResolvedValueOnce(mockTransactions) await syncQueue.add('sync-connection', { accountConnectionId: connection.id }) - expect(plaid.accountsGet).toHaveBeenCalledTimes(1) - expect(plaid.transactionsGet).toHaveBeenCalledTimes(1) + expect(teller.getAccounts).toHaveBeenCalledTimes(1) + expect(teller.getTransactions).toHaveBeenCalledTimes(1) const item = await prisma.accountConnection.findUniqueOrThrow({ where: { id: connection.id }, @@ -146,7 +117,7 @@ describe('Message queue tests', () => { accounts: { include: { balances: { - where: PlaidTestData.testDates.prismaWhereFilter, + where: TellerGenerator.testDates.prismaWhereFilter, orderBy: { date: 'asc' }, }, transactions: true, @@ -162,61 +133,25 @@ describe('Message queue tests', () => { const [account] = item.accounts - expect(account.transactions).toHaveLength(PlaidTestData.checkingTransactions.length) - expect(account.balances.map((b) => b.balance)).toEqual( - [ - 3630, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5115, - 5115, - 5115, - 5089.45, - 5089.45, - PlaidTestData.checkingAccount.balances.current!, - ].map((v) => new Prisma.Decimal(v)) + const intervalDates = Interval.fromDateTimes( + TellerGenerator.lowerBound, + TellerGenerator.now ) + .splitBy({ day: 1 }) + .map((date: Interval) => date.start.toISODate()) + + const startingBalance = Number(mockAccounts[0].balance.available) + + const balances = TellerGenerator.calculateDailyBalances( + startingBalance, + mockTransactions, + intervalDates + ) + + expect(account.transactions).toHaveLength(10) + expect(account.balances.map((b) => b.balance)).toEqual(balances) expect(account.holdings).toHaveLength(0) expect(account.valuations).toHaveLength(0) expect(account.investmentTransactions).toHaveLength(0) }) - - it('Should sync valid security prices', async () => { - const security = await prisma.security.create({ - data: { - name: 'Walmart Inc.', - symbol: 'WMT', - cusip: '93114210310', - pricingLastSyncedAt: new Date(), - }, - }) - - await securityPricingService.sync(security) - - const prices = await prisma.securityPricing.findMany({ - where: { securityId: security.id }, - orderBy: { date: 'asc' }, - }) - - expect(prices).toHaveLength(PlaidTestData.WMT.results.length) - - expect( - prices.map((p) => ({ - date: DateTime.fromJSDate(p.date, { zone: 'utc' }).toISODate(), - price: p.priceClose.toNumber(), - })) - ).toEqual( - PlaidTestData.WMT.results.map((p) => ({ - date: DateTime.fromMillis(p.t, { zone: 'utc' }).toISODate(), - price: p.c, - })) - ) - }) }) diff --git a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts index 683173d6..e5d4f4b0 100644 --- a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts +++ b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts @@ -69,9 +69,8 @@ export class InvestmentTransactionBalanceSyncStrategy extends BalanceSyncStrateg WHERE it.account_id = ${pAccountId} AND it.date BETWEEN ${pStart} AND now() - AND ( -- filter for transactions that modify a position - it.plaid_type IN ('buy', 'sell', 'transfer') - ) + -- filter for transactions that modify a position + AND it.category IN ('buy', 'sell', 'transfer') GROUP BY 1, 2 ) it ON it.security_id = s.id AND it.date = d.date diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index b83100ad..04d6650a 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -242,11 +242,7 @@ export class AccountQueryService implements IAccountQueryService { it.account_id = ANY(${pAccountIds}) AND it.date BETWEEN sd.start_date AND ${pEnd} -- filter for investment_transactions that represent external flows - AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - ) + AND it.category = 'transfer' GROUP BY 1, 2 ), external_flow_totals AS ( diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index b26893c9..bfe44d9d 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -312,6 +312,7 @@ export class InsightService implements IInsightService { { plaidSubtype: 'dividend', }, + { category: 'dividend' }, ], }, }), @@ -737,11 +738,7 @@ export class InsightService implements IInsightService { LEFT JOIN account a ON a.id = it.account_id WHERE it.account_id = ${accountId} - AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer', 'send', 'request')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - ) + AND it.category = 'transfer' -- Exclude any contributions made prior to the start date since balances will be 0 AND (a.start_date is NULL OR it.date >= a.start_date) GROUP BY 1 diff --git a/libs/server/features/src/providers/plaid/plaid.etl.ts b/libs/server/features/src/providers/plaid/plaid.etl.ts index ad16b62e..66adf23a 100644 --- a/libs/server/features/src/providers/plaid/plaid.etl.ts +++ b/libs/server/features/src/providers/plaid/plaid.etl.ts @@ -13,8 +13,10 @@ import type { Item as PlaidItem, LiabilitiesObject as PlaidLiabilities, PlaidApi, + PersonalFinanceCategory, } from 'plaid' -import { Prisma } from '@prisma/client' +import { InvestmentTransactionSubtype, InvestmentTransactionType } from 'plaid' +import { Prisma, InvestmentTransactionCategory } from '@prisma/client' import { DateTime } from 'luxon' import _, { chunk } from 'lodash' import { ErrorUtil, PlaidUtil } from '@maybe-finance/server/shared' @@ -366,7 +368,7 @@ export class PlaidETL implements IETL { const txnUpsertQueries = chunk(transactions, 1_000).map((chunk) => { return this.prisma.$executeRaw` - INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category) + INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category, category) VALUES ${Prisma.join( chunk.map((plaidTransaction) => { @@ -401,7 +403,8 @@ export class PlaidETL implements IETL { ${merchant_name}, ${category ?? []}, ${category_id}, - ${personal_finance_category} + ${personal_finance_category}, + ${this.getMaybeTransactionCategory(personal_finance_category)} )` }) )} @@ -414,6 +417,7 @@ export class PlaidETL implements IETL { plaid_category = EXCLUDED.plaid_category, plaid_category_id = EXCLUDED.plaid_category_id, plaid_personal_finance_category = EXCLUDED.plaid_personal_finance_category; + category = EXCLUDED.category; ` }) @@ -444,6 +448,68 @@ export class PlaidETL implements IETL { ] } + private getMaybeTransactionCategory = (category?: PersonalFinanceCategory | null) => { + if (!category) { + return 'Other' + } + + if (category.primary === 'INCOME') { + return 'Income' + } + + if ( + ['LOAN_PAYMENTS_MORTGAGE_PAYMENT', 'RENT_AND_UTILITIES_RENT'].includes( + category.detailed + ) + ) { + return 'Housing Payments' + } + + if (category.detailed === 'LOAN_PAYMENTS_CAR_PAYMENT') { + return 'Vehicle Payments' + } + + if (category.primary === 'LOAN_PAYMENTS') { + return 'Other Payments' + } + + if (category.primary === 'HOME_IMPROVEMENT') { + return 'Home Improvement' + } + + if (category.primary === 'GENERAL_MERCHANDISE') { + return 'Shopping' + } + + if ( + category.primary === 'RENT_AND_UTILITIES' && + category.detailed !== 'RENT_AND_UTILITIES_RENT' + ) { + return 'Utilities' + } + + if (category.primary === 'FOOD_AND_DRINK') { + return 'Food and Drink' + } + + if (category.primary === 'TRANSPORTATION') { + return 'Transportation' + } + + if (category.primary === 'TRAVEL') { + return 'Travel' + } + + if ( + ['PERSONAL_CARE', 'MEDICAL'].includes(category.primary) && + category.detailed !== 'MEDICAL_VETERINARY_SERVICES' + ) { + return 'Health' + } + + return 'Other' + } + private _extractInvestmentTransactions(accessToken: string, dateRange: SharedType.DateRange) { return SharedUtil.paginate({ pageSize: 500, // https://plaid.com/docs/api/products/investments/#investments-transactions-get-request-options-count @@ -548,7 +614,7 @@ export class PlaidETL implements IETL { ...chunk(investmentTransactions, 1_000).map( (chunk) => this.prisma.$executeRaw` - INSERT INTO investment_transaction (account_id, security_id, plaid_investment_transaction_id, date, name, amount, fees, quantity, price, currency_code, plaid_type, plaid_subtype) + INSERT INTO investment_transaction (account_id, security_id, plaid_investment_transaction_id, date, name, amount, fees, quantity, price, currency_code, plaid_type, plaid_subtype, category) VALUES ${Prisma.join( chunk.map( @@ -584,7 +650,11 @@ export class PlaidETL implements IETL { ${DbUtil.toDecimal(price)}, ${currencyCode}, ${type}, - ${subtype} + ${subtype}, + ${this.getInvestmentTransactionCategoryByPlaidType( + type, + subtype + )} )` } ) @@ -602,6 +672,7 @@ export class PlaidETL implements IETL { currency_code = EXCLUDED.currency_code, plaid_type = EXCLUDED.plaid_type, plaid_subtype = EXCLUDED.plaid_subtype; + category = EXCLUDED.category; ` ), @@ -669,6 +740,63 @@ export class PlaidETL implements IETL { ] } + private getInvestmentTransactionCategoryByPlaidType = ( + type: InvestmentTransactionType, + subType: InvestmentTransactionSubtype + ): InvestmentTransactionCategory => { + if (type === InvestmentTransactionType.Buy) { + return InvestmentTransactionCategory.buy + } + + if (type === InvestmentTransactionType.Sell) { + return InvestmentTransactionCategory.sell + } + + if ( + [ + InvestmentTransactionSubtype.Dividend, + InvestmentTransactionSubtype.QualifiedDividend, + InvestmentTransactionSubtype.NonQualifiedDividend, + ].includes(subType) + ) { + return InvestmentTransactionCategory.dividend + } + + if ( + [ + InvestmentTransactionSubtype.NonResidentTax, + InvestmentTransactionSubtype.Tax, + InvestmentTransactionSubtype.TaxWithheld, + ].includes(subType) + ) { + return InvestmentTransactionCategory.tax + } + + if ( + type === InvestmentTransactionType.Fee || + [ + InvestmentTransactionSubtype.AccountFee, + InvestmentTransactionSubtype.LegalFee, + InvestmentTransactionSubtype.ManagementFee, + InvestmentTransactionSubtype.MarginExpense, + InvestmentTransactionSubtype.TransferFee, + InvestmentTransactionSubtype.TrustFee, + ].includes(subType) + ) { + return InvestmentTransactionCategory.fee + } + + if (type === InvestmentTransactionType.Cash) { + return InvestmentTransactionCategory.transfer + } + + if (type === InvestmentTransactionType.Cancel) { + return InvestmentTransactionCategory.cancel + } + + return InvestmentTransactionCategory.other + } + private async _extractHoldings(accessToken: string) { try { const { data } = await this.plaid.investmentsHoldingsGet({ access_token: accessToken }) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index 6891de97..406f70c3 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -25,6 +25,40 @@ type Connection = Pick< 'id' | 'userId' | 'tellerInstitutionId' | 'tellerAccessToken' > +const maybeCategoryByTellerCategory: Record< + Required['category'], + string +> = { + accommodation: 'Travel', + advertising: 'Other', + bar: 'Food and Drink', + charity: 'Other', + clothing: 'Shopping', + dining: 'Food and Drink', + education: 'Other', + electronics: 'Shopping', + entertainment: 'Shopping', + fuel: 'Transportation', + general: 'Other', + groceries: 'Food and Drink', + health: 'Health', + home: 'Home Improvement', + income: 'Income', + insurance: 'Other', + investment: 'Other', + loan: 'Other', + office: 'Other', + phone: 'Utilities', + service: 'Other', + shopping: 'Shopping', + software: 'Shopping', + sport: 'Shopping', + tax: 'Other', + transport: 'Transportation', + transportation: 'Transportation', + utilities: 'Utilities', +} + export class TellerETL implements IETL { public constructor( private readonly logger: Logger, @@ -204,7 +238,7 @@ export class TellerETL implements IETL { const txnUpsertQueries = _.chunk(transactions, 1_000).map((chunk) => { return this.prisma.$executeRaw` - INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category) + INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category, category) VALUES ${Prisma.join( chunk.map((tellerTransaction) => { @@ -231,7 +265,8 @@ export class TellerETL implements IETL { ${'USD'}, ${details.counterparty?.name ?? ''}, ${type}, - ${details.category ?? ''} + ${details.category ?? ''}, + ${maybeCategoryByTellerCategory[details.category ?? ''] ?? 'Other'} )` }) )} @@ -243,6 +278,7 @@ export class TellerETL implements IETL { merchant_name = EXCLUDED.merchant_name, teller_type = EXCLUDED.teller_type, teller_category = EXCLUDED.teller_category; + category = EXCLUDED.category; ` }) diff --git a/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql b/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql new file mode 100644 index 00000000..1acfb3ad --- /dev/null +++ b/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql @@ -0,0 +1,63 @@ +-- AlterTable +ALTER TABLE "transaction" + RENAME COLUMN "category" TO "category_old"; + +DROP VIEW IF EXISTS transactions_enriched; + +ALTER TABLE "transaction" + ADD COLUMN "category" TEXT NOT NULL DEFAULT 'Other'::text; + +CREATE OR REPLACE VIEW transactions_enriched AS ( + SELECT + t.id, + t.created_at as "createdAt", + t.updated_at as "updatedAt", + t.name, + t.account_id as "accountId", + t.date, + t.flow, + COALESCE( + t.type_user, + CASE + -- no matching transaction + WHEN t.match_id IS NULL THEN ( + CASE + t.flow + WHEN 'INFLOW' THEN ( + CASE + a.classification + WHEN 'asset' THEN 'INCOME' :: "TransactionType" + WHEN 'liability' THEN 'PAYMENT' :: "TransactionType" + END + ) + WHEN 'OUTFLOW' THEN 'EXPENSE' :: "TransactionType" + END + ) -- has matching transaction + ELSE ( + CASE + a.classification + WHEN 'asset' THEN 'TRANSFER' :: "TransactionType" + WHEN 'liability' THEN 'PAYMENT' :: "TransactionType" + END + ) + END + ) AS "type", + t.type_user as "typeUser", + t.amount, + t.currency_code as "currencyCode", + t.pending, + t.merchant_name as "merchantName", + t.category, + t.category_user as "categoryUser", + t.excluded, + t.match_id as "matchId", + COALESCE(ac.user_id, a.user_id) as "userId", + a.classification as "accountClassification", + a.type as "accountType" + FROM + transaction t + inner join account a on a.id = t.account_id + left join account_connection ac on a.account_connection_id = ac.id +); + +ALTER TABLE "transaction" DROP COLUMN "category_old"; diff --git a/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql b/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql new file mode 100644 index 00000000..e888285c --- /dev/null +++ b/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql @@ -0,0 +1,127 @@ +-- AlterTable +ALTER TABLE "investment_transaction" + RENAME COLUMN "category" TO "category_old"; + +DROP VIEW IF EXISTS holdings_enriched; + +ALTER TABLE "investment_transaction" + ADD COLUMN "category" "InvestmentTransactionCategory" NOT NULL DEFAULT 'other'::"InvestmentTransactionCategory"; + +CREATE OR REPLACE VIEW holdings_enriched AS ( + SELECT + h.id, + h.account_id, + h.security_id, + h.quantity, + COALESCE(pricing_latest.price_close * h.quantity * COALESCE(s.shares_per_contract, 1), h.value) AS "value", + COALESCE(h.cost_basis, tcb.cost_basis * h.quantity) AS "cost_basis", + COALESCE(h.cost_basis / h.quantity / COALESCE(s.shares_per_contract, 1), tcb.cost_basis) AS "cost_basis_per_share", + pricing_latest.price_close AS "price", + pricing_prev.price_close AS "price_prev", + h.excluded + FROM + holding h + INNER JOIN security s ON s.id = h.security_id + -- latest security pricing + LEFT JOIN LATERAL ( + SELECT + price_close + FROM + security_pricing + WHERE + security_id = h.security_id + ORDER BY + date DESC + LIMIT 1 + ) pricing_latest ON true + -- previous security pricing (for computing daily ∆) + LEFT JOIN LATERAL ( + SELECT + price_close + FROM + security_pricing + WHERE + security_id = h.security_id + ORDER BY + date DESC + LIMIT 1 + OFFSET 1 + ) pricing_prev ON true + -- calculate cost basis from transactions + LEFT JOIN ( + SELECT + it.account_id, + it.security_id, + SUM(it.quantity * it.price) / SUM(it.quantity) AS cost_basis + FROM + investment_transaction it + WHERE + it.category = 'buy' + AND it.quantity > 0 + GROUP BY + it.account_id, + it.security_id + ) tcb ON tcb.account_id = h.account_id AND tcb.security_id = s.id +); + +CREATE OR REPLACE FUNCTION calculate_return_dietz(p_account_id account.id%type, p_start date, p_end date, out percentage numeric, out amount numeric) AS $$ + DECLARE + v_start date := GREATEST(p_start, (SELECT MIN(date) FROM account_balance WHERE account_id = p_account_id)); + v_end date := p_end; + v_days int := v_end - v_start; + BEGIN + SELECT + ROUND((b1.balance - b0.balance - flows.net) / NULLIF(b0.balance + flows.weighted, 0), 4) AS "percentage", + b1.balance - b0.balance - flows.net AS "amount" + INTO + percentage, amount + FROM + account a + LEFT JOIN LATERAL ( + SELECT + COALESCE(SUM(-fw.flow), 0) AS "net", + COALESCE(SUM(-fw.flow * fw.weight), 0) AS "weighted" + FROM ( + SELECT + SUM(it.amount) AS flow, + (v_days - (it.date - v_start))::numeric / v_days AS weight + FROM + investment_transaction it + WHERE + it.account_id = a.id + AND it.date BETWEEN v_start AND v_end + -- filter for investment_transactions that represent external flows + AND it.category = 'transfer' + GROUP BY + it.date + ) fw + ) flows ON TRUE + LEFT JOIN LATERAL ( + SELECT + ab.balance AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_start + ORDER BY + ab.date DESC + LIMIT 1 + ) b0 ON TRUE + LEFT JOIN LATERAL ( + SELECT + COALESCE(ab.balance, a.current_balance) AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_end + ORDER BY + ab.date DESC + LIMIT 1 + ) b1 ON TRUE + WHERE + a.id = p_account_id; + END; +$$ LANGUAGE plpgsql STABLE; + +ALTER TABLE "investment_transaction" + DROP COLUMN "category_old"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4242132e..cccdd299 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -217,24 +217,22 @@ enum InvestmentTransactionCategory { } model InvestmentTransaction { - id Int @id @default(autoincrement()) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) - updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) - account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) - accountId Int @map("account_id") - security Security? @relation(fields: [securityId], references: [id], onDelete: Cascade) - securityId Int? @map("security_id") - date DateTime @db.Date + id Int @id @default(autoincrement()) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) + account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) + accountId Int @map("account_id") + security Security? @relation(fields: [securityId], references: [id], onDelete: Cascade) + securityId Int? @map("security_id") + date DateTime @db.Date name String - amount Decimal @db.Decimal(19, 4) - fees Decimal? @db.Decimal(19, 4) - flow TransactionFlow @default(dbgenerated("\nCASE\n WHEN (amount < (0)::numeric) THEN 'INFLOW'::\"TransactionFlow\"\n ELSE 'OUTFLOW'::\"TransactionFlow\"\nEND")) - quantity Decimal @db.Decimal(36, 18) - price Decimal @db.Decimal(23, 8) - currencyCode String @default("USD") @map("currency_code") - - // Derived from provider types - category InvestmentTransactionCategory @default(dbgenerated("\nCASE\n WHEN (plaid_type = 'buy'::text) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'sell'::text) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['dividend'::text, 'qualified dividend'::text, 'non-qualified dividend'::text])) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['non-resident tax'::text, 'tax'::text, 'tax withheld'::text])) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN ((plaid_type = 'fee'::text) OR (plaid_subtype = ANY (ARRAY['account fee'::text, 'legal fee'::text, 'management fee'::text, 'margin expense'::text, 'transfer fee'::text, 'trust fee'::text]))) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cash'::text) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n ELSE 'other'::\"InvestmentTransactionCategory\"\nEND")) + amount Decimal @db.Decimal(19, 4) + fees Decimal? @db.Decimal(19, 4) + flow TransactionFlow @default(dbgenerated("\nCASE\n WHEN (amount < (0)::numeric) THEN 'INFLOW'::\"TransactionFlow\"\n ELSE 'OUTFLOW'::\"TransactionFlow\"\nEND")) + quantity Decimal @db.Decimal(36, 18) + price Decimal @db.Decimal(23, 8) + currencyCode String @default("USD") @map("currency_code") + category InvestmentTransactionCategory @default(other) // plaid data plaidInvestmentTransactionId String? @unique @map("plaid_investment_transaction_id") @@ -312,7 +310,7 @@ model Transaction { currencyCode String @default("USD") @map("currency_code") pending Boolean @default(false) merchantName String? @map("merchant_name") - category String @default(dbgenerated("COALESCE(category_user,\nCASE\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'INCOME'::text) THEN 'Income'::text\n WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = ANY (ARRAY['LOAN_PAYMENTS_MORTGAGE_PAYMENT'::text, 'RENT_AND_UTILITIES_RENT'::text])) THEN 'Housing Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = 'LOAN_PAYMENTS_CAR_PAYMENT'::text) THEN 'Vehicle Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'LOAN_PAYMENTS'::text) THEN 'Other Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'HOME_IMPROVEMENT'::text) THEN 'Home Improvement'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'GENERAL_MERCHANDISE'::text) THEN 'Shopping'::text\n WHEN (((plaid_personal_finance_category ->> 'primary'::text) = 'RENT_AND_UTILITIES'::text) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'RENT_AND_UTILITIES_RENT'::text)) THEN 'Utilities'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'FOOD_AND_DRINK'::text) THEN 'Food and Drink'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRANSPORTATION'::text) THEN 'Transportation'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRAVEL'::text) THEN 'Travel'::text\n WHEN (((plaid_personal_finance_category ->> 'primary'::text) = ANY (ARRAY['PERSONAL_CARE'::text, 'MEDICAL'::text])) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'MEDICAL_VETERINARY_SERVICES'::text)) THEN 'Health'::text\n WHEN (teller_category = 'income'::text) THEN 'Income'::text\n WHEN (teller_category = 'home'::text) THEN 'Home Improvement'::text\n WHEN (teller_category = ANY (ARRAY['phone'::text, 'utilities'::text])) THEN 'Utilities'::text\n WHEN (teller_category = ANY (ARRAY['dining'::text, 'bar'::text, 'groceries'::text])) THEN 'Food and Drink'::text\n WHEN (teller_category = ANY (ARRAY['clothing'::text, 'entertainment'::text, 'shopping'::text, 'electronics'::text, 'software'::text, 'sport'::text])) THEN 'Shopping'::text\n WHEN (teller_category = ANY (ARRAY['transportation'::text, 'fuel'::text])) THEN 'Transportation'::text\n WHEN (teller_category = ANY (ARRAY['accommodation'::text, 'transport'::text])) THEN 'Travel'::text\n WHEN (teller_category = 'health'::text) THEN 'Health'::text\n WHEN (teller_category = ANY (ARRAY['loan'::text, 'tax'::text, 'insurance'::text, 'office'::text])) THEN 'Other Payments'::text\n ELSE 'Other'::text\nEND)")) + category String @default("Other") categoryUser String? @map("category_user") excluded Boolean @default(false) diff --git a/prisma/seed.ts b/prisma/seed.ts index adcfa65d..a8716316 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -8,17 +8,30 @@ const prisma = new PrismaClient() */ async function main() { const institutions: (Pick & { - providers: { provider: Provider; providerId: string; rank?: number }[] + providers: { provider: Provider; providerId: string; logoUrl: string; rank?: number }[] })[] = [ { id: 1, name: 'Capital One', - providers: [{ provider: 'PLAID', providerId: 'ins_9', rank: 1 }], + providers: [ + { + provider: Provider.TELLER, + providerId: 'capital_one', + logoUrl: 'https://teller.io/images/banks/capital_one.jpg', + rank: 1, + }, + ], }, { id: 2, - name: 'Discover Bank', - providers: [{ provider: 'PLAID', providerId: 'ins_33' }], + name: 'Wells Fargo', + providers: [ + { + provider: Provider.TELLER, + providerId: 'wells_fargo', + logoUrl: 'https://teller.io/images/banks/wells_fargo.jpg', + }, + ], }, ] diff --git a/tools/generators/tellerGenerator.ts b/tools/generators/tellerGenerator.ts index 8ee131b4..f0d621ac 100644 --- a/tools/generators/tellerGenerator.ts +++ b/tools/generators/tellerGenerator.ts @@ -1,5 +1,7 @@ import { faker } from '@faker-js/faker' import type { TellerTypes } from '../../libs/teller-api/src' +import type { Prisma } from '@prisma/client' +import { DateTime } from 'luxon' function generateSubType( type: TellerTypes.AccountTypes @@ -23,6 +25,8 @@ type GenerateAccountsParams = { enrollmentId: string institutionName: string institutionId: string + accountType?: TellerTypes.AccountTypes + accountSubType?: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype } export function generateAccounts({ @@ -30,12 +34,15 @@ export function generateAccounts({ enrollmentId, institutionName, institutionId, + accountType, + accountSubType, }: GenerateAccountsParams) { const accounts: TellerTypes.Account[] = [] for (let i = 0; i < count; i++) { const accountId = faker.string.uuid() const lastFour = faker.finance.creditCardNumber().slice(-4) - const type: TellerTypes.AccountTypes = faker.helpers.arrayElement(['depository', 'credit']) + const type: TellerTypes.AccountTypes = + accountType ?? faker.helpers.arrayElement(['depository', 'credit']) let subType: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype subType = generateSubType(type) @@ -99,6 +106,8 @@ type GenerateAccountsWithBalancesParams = { enrollmentId: string institutionName: string institutionId: string + accountType?: TellerTypes.AccountTypes + accountSubType?: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype } export function generateAccountsWithBalances({ @@ -106,7 +115,9 @@ export function generateAccountsWithBalances({ enrollmentId, institutionName, institutionId, -}: GenerateAccountsWithBalancesParams): TellerTypes.AccountWithBalances[] { + accountType, + accountSubType, +}: GenerateAccountsWithBalancesParams): TellerTypes.GetAccountsResponse { const accountsWithBalances: TellerTypes.AccountWithBalances[] = [] for (let i = 0; i < count; i++) { const account = generateAccounts({ @@ -114,6 +125,8 @@ export function generateAccountsWithBalances({ enrollmentId, institutionName, institutionId, + accountType, + accountSubType, })[0] const balance = generateBalance(account.id) accountsWithBalances.push({ @@ -170,7 +183,10 @@ export function generateTransactions(count: number, accountId: string): TellerTy running_balance: null, description: faker.word.words({ count: { min: 3, max: 10 } }), id: transactionId, - date: faker.date.recent({ days: 30 }).toISOString().split('T')[0], // recent date in 'YYYY-MM-DD' format + date: faker.date + .between({ from: lowerBound.toJSDate(), to: now.toJSDate() }) + .toISOString() + .split('T')[0], // recent date in 'YYYY-MM-DD' format account_id: accountId, links: { account: `https://api.teller.io/accounts/${accountId}`, @@ -246,3 +262,35 @@ export function generateConnection(): GenerateConnectionsResponse { transactions, } } + +export const now = DateTime.fromISO('2022-01-03', { zone: 'utc' }) + +export const lowerBound = DateTime.fromISO('2021-12-01', { zone: 'utc' }) + +export const testDates = { + now, + lowerBound, + totalDays: now.diff(lowerBound, 'days').days, + prismaWhereFilter: { + date: { + gte: lowerBound.toJSDate(), + lte: now.toJSDate(), + }, + } as Prisma.AccountBalanceWhereInput, +} + +export function calculateDailyBalances(startingBalance, transactions, dateInterval) { + transactions.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()) + + const balanceChanges = {} + + transactions.forEach((transaction) => { + const date = new Date(transaction.date).toISOString().split('T')[0] + balanceChanges[date] = (balanceChanges[date] || 0) + Number(transaction.amount) + }) + return dateInterval.map((date) => { + return Object.keys(balanceChanges) + .filter((d) => d <= date) + .reduce((acc, d) => acc + balanceChanges[d], startingBalance) + }) +}