From f2a5eb14953c83c42f2c762015da4267e82e80d7 Mon Sep 17 00:00:00 2001 From: Karan Handa Date: Fri, 19 Jan 2024 17:09:50 +0530 Subject: [PATCH 01/22] wait for session to load --- apps/client/pages/_app.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/client/pages/_app.tsx b/apps/client/pages/_app.tsx index d1fb65c9..0a4affac 100644 --- a/apps/client/pages/_app.tsx +++ b/apps/client/pages/_app.tsx @@ -1,4 +1,4 @@ -import { useEffect, type PropsWithChildren, type ReactElement } from 'react' +import { useEffect, useState, type PropsWithChildren, type ReactElement } from 'react' import type { AppProps } from 'next/app' import { ErrorBoundary } from 'react-error-boundary' import { Analytics } from '@vercel/analytics/react' @@ -33,14 +33,18 @@ Sentry.init({ // Providers and components only relevant to a logged-in user const WithAuth = function ({ children }: PropsWithChildren) { - const { data: session } = useSession() + const { data: session, status } = useSession() + const [isLoading, setLoading] = useState(true) const router = useRouter() useEffect(() => { + if (status === 'loading') return // Wait until the session status is not 'loading' + setLoading(false) + if (!session) { router.push('/login') } - }, [session, router]) + }, [session, status, router]) if (session) { return ( From 024f289ee9c87ffdb949a58190b109b8ee1b7e75 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 15:51:39 -0600 Subject: [PATCH 02/22] remove plaid from tests and replace db function --- apps/client/pages/_app.tsx | 8 +-- .../app/__tests__/account.integration.spec.ts | 10 --- .../balance-sync.integration.spec.ts | 7 ++- .../server/src/app/__tests__/utils/account.ts | 39 ++++++------ ...tment-transaction-balance-sync.strategy.ts | 3 +- .../src/account/account-query.service.ts | 5 +- .../features/src/account/insight.service.ts | 5 +- .../migration.sql | 7 +++ .../migration.sql | 61 +++++++++++++++++++ prisma/schema.prisma | 2 +- 10 files changed, 97 insertions(+), 50 deletions(-) create mode 100644 prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql create mode 100644 prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql diff --git a/apps/client/pages/_app.tsx b/apps/client/pages/_app.tsx index d1fb65c9..b5623f02 100644 --- a/apps/client/pages/_app.tsx +++ b/apps/client/pages/_app.tsx @@ -33,16 +33,16 @@ Sentry.init({ // Providers and components only relevant to a logged-in user const WithAuth = function ({ children }: PropsWithChildren) { - const { data: session } = useSession() + const { data: session, status } = useSession() const router = useRouter() useEffect(() => { - if (!session) { + if (!session && status === 'unauthenticated') { router.push('/login') } - }, [session, router]) + }, [session, status, router]) - if (session) { + if (session && status === 'authenticated') { return ( diff --git a/apps/server/src/app/__tests__/account.integration.spec.ts b/apps/server/src/app/__tests__/account.integration.spec.ts index d8ebb18e..83ee4a04 100644 --- a/apps/server/src/app/__tests__/account.integration.spec.ts +++ b/apps/server/src/app/__tests__/account.integration.spec.ts @@ -15,17 +15,11 @@ import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import { resetUser } from './utils/user' import { createTestInvestmentAccount } from './utils/account' -import { default as _plaid } from '../lib/plaid' -jest.mock('../middleware/validate-plaid-jwt.ts') jest.mock('bull') -jest.mock('plaid') const prisma = new PrismaClient() -// For TypeScript support -const plaid = jest.mocked(_plaid) // eslint-disable-line - const authId = '__TEST_USER_ID__' let axios: AxiosInstance let user: User @@ -107,10 +101,6 @@ describe('/v1/accounts API', () => { mask: null, isActive: true, syncStatus: 'IDLE', - plaidType: null, - plaidSubtype: null, - plaidAccountId: null, - plaidLiability: null, currencyCode: 'USD', currentBalance: new Decimal(21_000), availableBalance: null, diff --git a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts index 2dcbcb8a..1acbbb1c 100644 --- a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts +++ b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts @@ -1,4 +1,4 @@ -import type { User } from '@prisma/client' +import { InvestmentTransactionCategory, type User } from '@prisma/client' import { PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' @@ -130,7 +130,7 @@ describe('balance sync strategies', () => { amount: 100, quantity: 10, price: 10, - plaidType: 'buy', + category: InvestmentTransactionCategory.buy, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -139,7 +139,7 @@ describe('balance sync strategies', () => { amount: -50, quantity: 5, price: 10, - plaidType: 'sell', + category: InvestmentTransactionCategory.sell, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -147,6 +147,7 @@ describe('balance sync strategies', () => { amount: 50, quantity: 50, price: 1, + category: InvestmentTransactionCategory.other, }, ], }, diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 47e26cc3..088b2da8 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -35,7 +35,7 @@ export async function createTestInvestmentAccount( join(__dirname, `../test-data/${portfolio}/holdings.csv`) ) - const [_deleted, ...securities] = await prisma.$transaction([ + const [, ...securities] = await prisma.$transaction([ prisma.security.deleteMany({ where: { symbol: { @@ -99,7 +99,21 @@ export async function createTestInvestmentAccount( (s) => s.date === it.date && s.ticker === it.ticker )?.price - const isCashFlow = it.type === 'DEPOSIT' || it.type === 'WITHDRAW' + function getTransactionCategory(type: string) { + switch (type) { + case 'BUY': + return 'buy' + case 'SELL': + return 'sell' + case 'DIVIDEND': + return 'dividend' + case 'DEPOSIT': + case 'WITHDRAW': + return 'transfer' + default: + return undefined + } + } return { securityId: securities.find((s) => it.ticker === s.symbol)?.id, @@ -108,26 +122,7 @@ export async function createTestInvestmentAccount( amount: price ? new Prisma.Decimal(price).times(it.qty) : it.qty, quantity: price ? it.qty : 0, price: price ?? 0, - plaidType: - isCashFlow || it.type === 'DIVIDEND' - ? 'cash' - : it.type === 'BUY' - ? 'buy' - : it.type === 'SELL' - ? 'sell' - : undefined, - plaidSubtype: - it.type === 'DEPOSIT' - ? 'deposit' - : it.type === 'WITHDRAW' - ? 'withdrawal' - : it.type === 'DIVIDEND' - ? 'dividend' - : it.type === 'BUY' - ? 'buy' - : it.type === 'SELL' - ? 'sell' - : undefined, + category: getTransactionCategory(it.type), } }), }, diff --git a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts index 21f0d914..83cc4954 100644 --- a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts +++ b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts @@ -70,8 +70,7 @@ export class InvestmentTransactionBalanceSyncStrategy extends BalanceSyncStrateg it.account_id = ${pAccountId} AND it.date BETWEEN ${pStart} AND now() AND ( -- filter for transactions that modify a position - it.plaid_type IN ('buy', 'sell', 'transfer') - OR it.finicity_transaction_id IS NOT NULL + it.category IN ('buy', 'sell', 'transfer') ) GROUP BY 1, 2 diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index fa206904..05e7a511 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -243,10 +243,7 @@ export class AccountQueryService implements IAccountQueryService { AND it.date BETWEEN sd.start_date AND ${pEnd} -- filter for investment_transactions that represent external flows AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) + it.category = 'transfer' ) GROUP BY 1, 2 diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index 4c736ab7..f1df4568 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -747,10 +747,7 @@ export class InsightService implements IInsightService { WHERE it.account_id = ${accountId} AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer', 'send', 'request')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) + it.category = 'transfer' ) -- Exclude any contributions made prior to the start date since balances will be 0 AND (a.start_date is NULL OR it.date >= a.start_date) diff --git a/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql b/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql new file mode 100644 index 00000000..2add40ab --- /dev/null +++ b/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql @@ -0,0 +1,7 @@ +ALTER TABLE + "investment_transaction" DROP COLUMN "category"; + +ALTER TABLE + "investment_transaction" +ADD + COLUMN "category" "InvestmentTransactionCategory" DEFAULT 'other' :: "InvestmentTransactionCategory" NOT NULL; diff --git a/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql b/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql new file mode 100644 index 00000000..f47d6fee --- /dev/null +++ b/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql @@ -0,0 +1,61 @@ +CREATE OR REPLACE FUNCTION calculate_return_dietz(p_account_id account.id%type, p_start date, p_end date, out percentage numeric, out amount numeric) AS $$ + DECLARE + v_start date := GREATEST(p_start, (SELECT MIN(date) FROM account_balance WHERE account_id = p_account_id)); + v_end date := p_end; + v_days int := v_end - v_start; + BEGIN + SELECT + ROUND((b1.balance - b0.balance - flows.net) / NULLIF(b0.balance + flows.weighted, 0), 4) AS "percentage", + b1.balance - b0.balance - flows.net AS "amount" + INTO + percentage, amount + FROM + account a + LEFT JOIN LATERAL ( + SELECT + COALESCE(SUM(-fw.flow), 0) AS "net", + COALESCE(SUM(-fw.flow * fw.weight), 0) AS "weighted" + FROM ( + SELECT + SUM(it.amount) AS flow, + (v_days - (it.date - v_start))::numeric / v_days AS weight + FROM + investment_transaction it + WHERE + it.account_id = a.id + AND it.date BETWEEN v_start AND v_end + -- filter for investment_transactions that represent external flows + AND ( + it.category = 'transfer' + ) + GROUP BY + it.date + ) fw + ) flows ON TRUE + LEFT JOIN LATERAL ( + SELECT + ab.balance AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_start + ORDER BY + ab.date DESC + LIMIT 1 + ) b0 ON TRUE + LEFT JOIN LATERAL ( + SELECT + COALESCE(ab.balance, a.current_balance) AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_end + ORDER BY + ab.date DESC + LIMIT 1 + ) b1 ON TRUE + WHERE + a.id = p_account_id; + END; +$$ LANGUAGE plpgsql STABLE; + diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d358089e..9b01c4ae 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -250,7 +250,7 @@ model InvestmentTransaction { currencyCode String @default("USD") @map("currency_code") // Derived from provider types - category InvestmentTransactionCategory @default(dbgenerated("\nCASE\n WHEN (plaid_type = 'buy'::text) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'sell'::text) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['dividend'::text, 'qualified dividend'::text, 'non-qualified dividend'::text])) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['non-resident tax'::text, 'tax'::text, 'tax withheld'::text])) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN ((plaid_type = 'fee'::text) OR (plaid_subtype = ANY (ARRAY['account fee'::text, 'legal fee'::text, 'management fee'::text, 'margin expense'::text, 'transfer fee'::text, 'trust fee'::text]))) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cash'::text) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['purchased'::text, 'purchaseToClose'::text, 'purchaseToCover'::text, 'dividendReinvest'::text, 'reinvestOfIncome'::text])) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['sold'::text, 'soldToClose'::text, 'soldToOpen'::text])) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'dividend'::text) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'tax'::text) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'fee'::text) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['transfer'::text, 'contribution'::text, 'deposit'::text, 'income'::text, 'interest'::text])) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n ELSE 'other'::\"InvestmentTransactionCategory\"\nEND")) + category InvestmentTransactionCategory @default(other) // plaid data plaidInvestmentTransactionId String? @unique @map("plaid_investment_transaction_id") From 3a9e68c5ebddf2145fbd842b0437842aa3cbaac1 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 16:54:31 -0600 Subject: [PATCH 03/22] remove plaid from queue test --- .../app/__tests__/queue.integration.spec.ts | 139 ++++-------------- tools/generators/tellerGenerator.ts | 38 ++++- 2 files changed, 66 insertions(+), 111 deletions(-) diff --git a/apps/workers/src/app/__tests__/queue.integration.spec.ts b/apps/workers/src/app/__tests__/queue.integration.spec.ts index 07882ee2..4074fd12 100644 --- a/apps/workers/src/app/__tests__/queue.integration.spec.ts +++ b/apps/workers/src/app/__tests__/queue.integration.spec.ts @@ -2,25 +2,19 @@ // Keep these imports above the rest to avoid errors // ===================================================== import type { SharedType } from '@maybe-finance/shared' -import type { AccountsGetResponse, TransactionsGetResponse } from 'plaid' +import { TellerGenerator } from 'tools/generators' import type { AccountConnection, User } from '@prisma/client' -import { TestUtil } from '@maybe-finance/shared' -import { PlaidTestData } from '../../../../../tools/test-data' -import { Prisma } from '@prisma/client' import prisma from '../lib/prisma' -import { default as _plaid } from '../lib/plaid' -import nock from 'nock' -import { DateTime } from 'luxon' +import { default as _teller } from '../lib/teller' import { resetUser } from './helpers/user.test-helper' // Import the workers process import '../../main' -import { queueService, securityPricingService } from '../lib/di' - -jest.mock('plaid') +import { queueService } from '../lib/di' // For TypeScript support -const plaid = jest.mocked(_plaid) +jest.mock('../lib/teller') +const teller = jest.mocked(_teller) let user: User | null let connection: AccountConnection @@ -30,25 +24,6 @@ if (process.env.IS_VSCODE_DEBUG === 'true') { jest.setTimeout(100000) } -beforeAll(() => { - nock.disableNetConnect() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/AAPL/range/1/day')) - .reply(200, PlaidTestData.AAPL) - .persist() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/WMT/range/1/day')) - .reply(200, PlaidTestData.WMT) - .persist() - - nock('https://api.polygon.io') - .get((uri) => uri.includes('v2/aggs/ticker/VOO/range/1/day')) - .reply(200, PlaidTestData.VOO) - .persist() -}) - beforeEach(async () => { jest.clearAllMocks() @@ -57,10 +32,10 @@ beforeEach(async () => { connection = await prisma.accountConnection.create({ data: { name: 'Chase Test', - type: 'plaid' as SharedType.AccountConnectionType, - plaidItemId: 'test-plaid-item-workers', - plaidInstitutionId: 'ins_3', - plaidAccessToken: + type: 'teller' as SharedType.AccountConnectionType, + tellerEnrollmentId: 'test-teller-item-workers', + tellerInstitutionId: 'chase_test', + tellerAccessToken: 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', // need correct encoding here userId: user.id, syncStatus: 'PENDING', @@ -84,7 +59,7 @@ describe('Message queue tests', () => { it('Should handle sync errors', async () => { const syncQueue = queueService.getQueue('sync-account-connection') - plaid.accountsGet.mockRejectedValueOnce('forced error for Jest tests') + teller.getAccounts.mockRejectedValueOnce(new Error('forced error for Jest tests')) await syncQueue.add('sync-connection', { accountConnectionId: connection.id }) @@ -92,7 +67,7 @@ describe('Message queue tests', () => { where: { id: connection.id }, }) - expect(plaid.accountsGet).toHaveBeenCalledTimes(1) + expect(teller.getAccounts).toHaveBeenCalledTimes(1) expect(updatedConnection?.status).toEqual('ERROR') }) @@ -117,28 +92,23 @@ describe('Message queue tests', () => { const syncQueue = queueService.getQueue('sync-account-connection') // Mock will return a basic banking checking account - plaid.accountsGet.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - accounts: [PlaidTestData.checkingAccount], - item: PlaidTestData.item, - request_id: 'bkVE1BHWMAZ9Rnr', - }) as any - ) + const mockAccounts = TellerGenerator.generateAccountsWithBalances({ + count: 1, + institutionId: 'chase_test', + enrollmentId: 'test-teller-item-workers', + institutionName: 'Chase Test', + accountType: 'depository', + accountSubType: 'checking', + }) + teller.getAccounts.mockResolvedValueOnce(mockAccounts) - plaid.transactionsGet.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - accounts: [PlaidTestData.checkingAccount], - transactions: PlaidTestData.checkingTransactions, - item: PlaidTestData.item, - total_transactions: PlaidTestData.checkingTransactions.length, - request_id: '45QSn', - }) as any - ) + const mockTransactions = TellerGenerator.generateTransactions(10, mockAccounts[0].id) + teller.getTransactions.mockResolvedValueOnce(mockTransactions) await syncQueue.add('sync-connection', { accountConnectionId: connection.id }) - expect(plaid.accountsGet).toHaveBeenCalledTimes(1) - expect(plaid.transactionsGet).toHaveBeenCalledTimes(1) + expect(teller.getAccounts).toHaveBeenCalledTimes(1) + expect(teller.getTransactions).toHaveBeenCalledTimes(1) const item = await prisma.accountConnection.findUniqueOrThrow({ where: { id: connection.id }, @@ -146,7 +116,7 @@ describe('Message queue tests', () => { accounts: { include: { balances: { - where: PlaidTestData.testDates.prismaWhereFilter, + where: TellerGenerator.testDates.prismaWhereFilter, orderBy: { date: 'asc' }, }, transactions: true, @@ -161,62 +131,15 @@ describe('Message queue tests', () => { expect(item.accounts).toHaveLength(1) const [account] = item.accounts - - expect(account.transactions).toHaveLength(PlaidTestData.checkingTransactions.length) - expect(account.balances.map((b) => b.balance)).toEqual( - [ - 3630, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5125, - 5115, - 5115, - 5115, - 5089.45, - 5089.45, - PlaidTestData.checkingAccount.balances.current!, - ].map((v) => new Prisma.Decimal(v)) + const transactionBalance = mockTransactions.reduce( + (acc, t) => acc + t.amount, + mockAccounts[0].balance.available ) + + expect(account.transactions).toHaveLength(10) + expect(account.balances.map((b) => b.balance)).toEqual(transactionBalance) expect(account.holdings).toHaveLength(0) expect(account.valuations).toHaveLength(0) expect(account.investmentTransactions).toHaveLength(0) }) - - it('Should sync valid security prices', async () => { - const security = await prisma.security.create({ - data: { - name: 'Walmart Inc.', - symbol: 'WMT', - cusip: '93114210310', - pricingLastSyncedAt: new Date(), - }, - }) - - await securityPricingService.sync(security) - - const prices = await prisma.securityPricing.findMany({ - where: { securityId: security.id }, - orderBy: { date: 'asc' }, - }) - - expect(prices).toHaveLength(PlaidTestData.WMT.results.length) - - expect( - prices.map((p) => ({ - date: DateTime.fromJSDate(p.date, { zone: 'utc' }).toISODate(), - price: p.priceClose.toNumber(), - })) - ).toEqual( - PlaidTestData.WMT.results.map((p) => ({ - date: DateTime.fromMillis(p.t, { zone: 'utc' }).toISODate(), - price: p.c, - })) - ) - }) }) diff --git a/tools/generators/tellerGenerator.ts b/tools/generators/tellerGenerator.ts index 8ee131b4..c3f414b1 100644 --- a/tools/generators/tellerGenerator.ts +++ b/tools/generators/tellerGenerator.ts @@ -1,5 +1,7 @@ import { faker } from '@faker-js/faker' import type { TellerTypes } from '../../libs/teller-api/src' +import type { Prisma } from '@prisma/client' +import { DateTime } from 'luxon' function generateSubType( type: TellerTypes.AccountTypes @@ -23,6 +25,8 @@ type GenerateAccountsParams = { enrollmentId: string institutionName: string institutionId: string + accountType?: TellerTypes.AccountTypes + accountSubType?: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype } export function generateAccounts({ @@ -30,12 +34,15 @@ export function generateAccounts({ enrollmentId, institutionName, institutionId, + accountType, + accountSubType, }: GenerateAccountsParams) { const accounts: TellerTypes.Account[] = [] for (let i = 0; i < count; i++) { const accountId = faker.string.uuid() const lastFour = faker.finance.creditCardNumber().slice(-4) - const type: TellerTypes.AccountTypes = faker.helpers.arrayElement(['depository', 'credit']) + const type: TellerTypes.AccountTypes = + accountType ?? faker.helpers.arrayElement(['depository', 'credit']) let subType: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype subType = generateSubType(type) @@ -99,6 +106,8 @@ type GenerateAccountsWithBalancesParams = { enrollmentId: string institutionName: string institutionId: string + accountType?: TellerTypes.AccountTypes + accountSubType?: TellerTypes.DepositorySubtypes | TellerTypes.CreditSubtype } export function generateAccountsWithBalances({ @@ -106,7 +115,9 @@ export function generateAccountsWithBalances({ enrollmentId, institutionName, institutionId, -}: GenerateAccountsWithBalancesParams): TellerTypes.AccountWithBalances[] { + accountType, + accountSubType, +}: GenerateAccountsWithBalancesParams): TellerTypes.GetAccountsResponse { const accountsWithBalances: TellerTypes.AccountWithBalances[] = [] for (let i = 0; i < count; i++) { const account = generateAccounts({ @@ -114,6 +125,8 @@ export function generateAccountsWithBalances({ enrollmentId, institutionName, institutionId, + accountType, + accountSubType, })[0] const balance = generateBalance(account.id) accountsWithBalances.push({ @@ -170,7 +183,10 @@ export function generateTransactions(count: number, accountId: string): TellerTy running_balance: null, description: faker.word.words({ count: { min: 3, max: 10 } }), id: transactionId, - date: faker.date.recent({ days: 30 }).toISOString().split('T')[0], // recent date in 'YYYY-MM-DD' format + date: faker.date + .between({ from: lowerBound.toJSDate(), to: now.toJSDate() }) + .toISOString() + .split('T')[0], // recent date in 'YYYY-MM-DD' format account_id: accountId, links: { account: `https://api.teller.io/accounts/${accountId}`, @@ -246,3 +262,19 @@ export function generateConnection(): GenerateConnectionsResponse { transactions, } } + +const now = DateTime.fromISO('2022-01-03', { zone: 'utc' }) + +const lowerBound = DateTime.fromISO('2021-12-01', { zone: 'utc' }) + +export const testDates = { + now, + lowerBound, + totalDays: now.diff(lowerBound, 'days').days, + prismaWhereFilter: { + date: { + gte: lowerBound.toJSDate(), + lte: now.toJSDate(), + }, + } as Prisma.AccountBalanceWhereInput, +} From 0bb48f047f9023fe24f60ea9bf9e31387f60e4d6 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 17:03:49 -0600 Subject: [PATCH 04/22] remove plaid from net worth integration spec --- apps/server/src/app/__tests__/net-worth.integration.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/app/__tests__/net-worth.integration.spec.ts b/apps/server/src/app/__tests__/net-worth.integration.spec.ts index 5e5f7465..c17129ab 100644 --- a/apps/server/src/app/__tests__/net-worth.integration.spec.ts +++ b/apps/server/src/app/__tests__/net-worth.integration.spec.ts @@ -5,7 +5,6 @@ import { DateTime } from 'luxon' import { PgService } from '@maybe-finance/server/shared' import { AccountQueryService, UserService } from '@maybe-finance/server/features' import { resetUser } from './utils/user' -jest.mock('plaid') const prisma = new PrismaClient() From bc2b2492e7d1bc657cfbdaaa2e8a7ad0bc3030d9 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 17:52:07 -0600 Subject: [PATCH 05/22] remove plaid from more tests --- .../__tests__/connection.integration.spec.ts | 32 +++++++------------ .../__tests__/insights.integration.spec.ts | 14 +++----- prisma/seed.ts | 19 +++++++---- 3 files changed, 29 insertions(+), 36 deletions(-) diff --git a/apps/server/src/app/__tests__/connection.integration.spec.ts b/apps/server/src/app/__tests__/connection.integration.spec.ts index 7eb81099..fe3710bc 100644 --- a/apps/server/src/app/__tests__/connection.integration.spec.ts +++ b/apps/server/src/app/__tests__/connection.integration.spec.ts @@ -1,21 +1,18 @@ import type { AxiosInstance } from 'axios' import type { SharedType } from '@maybe-finance/shared' -import type { Prisma, AccountConnection, AccountSyncStatus, User } from '@prisma/client' -import type { ItemRemoveResponse } from 'plaid' +import type { Prisma, AccountConnection, User } from '@prisma/client' import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import prisma from '../lib/prisma' -import { TestUtil } from '@maybe-finance/shared' import { InMemoryQueue } from '@maybe-finance/server/shared' -import { default as _plaid } from '../lib/plaid' +import { default as _teller } from '../lib/teller' import nock from 'nock' import { resetUser } from './utils/user' -jest.mock('../middleware/validate-plaid-jwt.ts') -jest.mock('plaid') +jest.mock('../lib/teller.ts') // For TypeScript support -const plaid = jest.mocked(_plaid) +const teller = jest.mocked(_teller) const authId = '__TEST_USER_ID__' let axios: AxiosInstance @@ -49,13 +46,13 @@ beforeEach(async () => { connectionData = { data: { name: 'Chase Test', - type: 'plaid' as SharedType.AccountConnectionType, - plaidItemId: 'test-plaid-item-server', - plaidInstitutionId: 'ins_3', - plaidAccessToken: - 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', - userId: user!.id, - syncStatus: 'PENDING' as AccountSyncStatus, + type: 'teller' as SharedType.AccountConnectionType, + tellerEnrollmentId: 'test-teller-item-workers', + tellerInstitutionId: 'chase_test', + tellerAccessToken: + 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', // need correct encoding here + userId: user.id, + syncStatus: 'PENDING', }, } @@ -91,16 +88,9 @@ describe('/v1/connections API', () => { }) it('DELETE /:id', async () => { - plaid.itemRemove.mockResolvedValueOnce( - TestUtil.axiosSuccess({ - request_id: 'test request id', - }) - ) - const res = await axios.delete(`/connections/${connection.id}`) expect(res.status).toEqual(200) - expect(plaid.itemRemove).toHaveBeenCalledTimes(1) const res2 = await axios.get(`/connections/${connection.id}`) diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index 9dbe0433..89694f6f 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,4 +1,4 @@ -import type { User } from '@prisma/client' +import { InvestmentTransactionCategory, type User } from '@prisma/client' import { Prisma, PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' @@ -305,8 +305,7 @@ describe('insight service', () => { amount: 50 * 100, quantity: 50, price: 100, - plaidType: 'buy', - plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -316,8 +315,7 @@ describe('insight service', () => { amount: 10 * 200, quantity: 10, price: 200, - plaidType: 'buy', - plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -327,8 +325,7 @@ describe('insight service', () => { amount: -20.22, quantity: 0, price: 0, - plaidType: 'cash', - plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, { accountId: account.id, @@ -338,8 +335,7 @@ describe('insight service', () => { amount: -22.85, quantity: 0, price: 0, - plaidType: 'cash', - plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, ], }) diff --git a/prisma/seed.ts b/prisma/seed.ts index 7448631e..bcdaa68f 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -8,22 +8,29 @@ const prisma = new PrismaClient() */ async function main() { const institutions: (Pick & { - providers: { provider: Provider; providerId: string; rank?: number }[] + providers: { provider: Provider; providerId: string; logoUrl: string; rank?: number }[] })[] = [ { id: 1, name: 'Capital One', providers: [ - { provider: 'PLAID', providerId: 'ins_9', rank: 1 }, - { provider: 'FINICITY', providerId: '170778' }, + { + provider: 'TELLER', + providerId: 'capital_one', + logoUrl: 'https://teller.io/images/banks/capital_one.jpg', + rank: 1, + }, ], }, { id: 2, - name: 'Discover Bank', + name: 'Wells Fargo', providers: [ - { provider: 'PLAID', providerId: 'ins_33' }, - { provider: 'FINICITY', providerId: '13796', rank: 1 }, + { + provider: 'TELLER', + providerId: 'wells_fargo', + logoUrl: 'https://teller.io/images/banks/wells_fargo.jpg', + }, ], }, ] From 4bb856ffbfcc6a87842c6e483eea310b5439f815 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 18:06:42 -0600 Subject: [PATCH 06/22] add missing property --- libs/server/features/src/account/insight.service.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index f1df4568..0eb8b13a 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -315,6 +315,9 @@ export class InsightService implements IInsightService { { finicityInvestmentTransactionType: 'dividend', }, + { + category: 'dividend', + }, ], }, }), From bc4ff9bd7feea741f0376ab91ad802da772583e8 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:17:36 -0600 Subject: [PATCH 07/22] import fixes --- .../balance-sync.integration.spec.ts | 4 ++-- .../__tests__/connection.integration.spec.ts | 16 +++++++++------ .../__tests__/insights.integration.spec.ts | 4 ++-- .../app/__tests__/queue.integration.spec.ts | 20 +++++++++++++++---- prisma/seed.ts | 4 ++-- tools/generators/tellerGenerator.ts | 20 +++++++++++++++++-- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts index 1acbbb1c..0b868ebf 100644 --- a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts +++ b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts @@ -1,5 +1,5 @@ -import { InvestmentTransactionCategory, type User } from '@prisma/client' -import { PrismaClient } from '@prisma/client' +import type { User } from '@prisma/client' +import { PrismaClient, InvestmentTransactionCategory } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import { diff --git a/apps/server/src/app/__tests__/connection.integration.spec.ts b/apps/server/src/app/__tests__/connection.integration.spec.ts index fe3710bc..ac0e0df1 100644 --- a/apps/server/src/app/__tests__/connection.integration.spec.ts +++ b/apps/server/src/app/__tests__/connection.integration.spec.ts @@ -1,18 +1,22 @@ import type { AxiosInstance } from 'axios' -import type { SharedType } from '@maybe-finance/shared' -import type { Prisma, AccountConnection, User } from '@prisma/client' +import { + type Prisma, + type AccountConnection, + type User, + AccountConnectionType, + AccountSyncStatus, +} from '@prisma/client' import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import prisma from '../lib/prisma' import { InMemoryQueue } from '@maybe-finance/server/shared' -import { default as _teller } from '../lib/teller' import nock from 'nock' import { resetUser } from './utils/user' jest.mock('../lib/teller.ts') // For TypeScript support -const teller = jest.mocked(_teller) +//const teller = jest.mocked(_teller) const authId = '__TEST_USER_ID__' let axios: AxiosInstance @@ -46,13 +50,13 @@ beforeEach(async () => { connectionData = { data: { name: 'Chase Test', - type: 'teller' as SharedType.AccountConnectionType, + type: AccountConnectionType.teller, tellerEnrollmentId: 'test-teller-item-workers', tellerInstitutionId: 'chase_test', tellerAccessToken: 'U2FsdGVkX1+WMq9lfTS9Zkbgrn41+XT1hvSK5ain/udRPujzjVCAx/lyPG7EumVZA+nVKXPauGwI+d7GZgtqTA9R3iCZNusU6LFPnmFOCE4=', // need correct encoding here userId: user.id, - syncStatus: 'PENDING', + syncStatus: AccountSyncStatus.PENDING, }, } diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index 89694f6f..16a4ce96 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,5 +1,5 @@ -import { InvestmentTransactionCategory, type User } from '@prisma/client' -import { Prisma, PrismaClient } from '@prisma/client' +import type { User } from '@prisma/client' +import { Prisma, PrismaClient, InvestmentTransactionCategory } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import type { diff --git a/apps/workers/src/app/__tests__/queue.integration.spec.ts b/apps/workers/src/app/__tests__/queue.integration.spec.ts index 4074fd12..4d0b1f21 100644 --- a/apps/workers/src/app/__tests__/queue.integration.spec.ts +++ b/apps/workers/src/app/__tests__/queue.integration.spec.ts @@ -7,6 +7,7 @@ import type { AccountConnection, User } from '@prisma/client' import prisma from '../lib/prisma' import { default as _teller } from '../lib/teller' import { resetUser } from './helpers/user.test-helper' +import { Interval } from 'luxon' // Import the workers process import '../../main' @@ -131,13 +132,24 @@ describe('Message queue tests', () => { expect(item.accounts).toHaveLength(1) const [account] = item.accounts - const transactionBalance = mockTransactions.reduce( - (acc, t) => acc + t.amount, - mockAccounts[0].balance.available + + const intervalDates = Interval.fromDateTimes( + TellerGenerator.lowerBound, + TellerGenerator.now + ) + .splitBy({ day: 1 }) + .map((date: Interval) => date.start.toISODate()) + + const startingBalance = Number(mockAccounts[0].balance.available) + + const balances = TellerGenerator.calculateDailyBalances( + startingBalance, + mockTransactions, + intervalDates ) expect(account.transactions).toHaveLength(10) - expect(account.balances.map((b) => b.balance)).toEqual(transactionBalance) + expect(account.balances.map((b) => b.balance)).toEqual(balances) expect(account.holdings).toHaveLength(0) expect(account.valuations).toHaveLength(0) expect(account.investmentTransactions).toHaveLength(0) diff --git a/prisma/seed.ts b/prisma/seed.ts index bcdaa68f..a8716316 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -15,7 +15,7 @@ async function main() { name: 'Capital One', providers: [ { - provider: 'TELLER', + provider: Provider.TELLER, providerId: 'capital_one', logoUrl: 'https://teller.io/images/banks/capital_one.jpg', rank: 1, @@ -27,7 +27,7 @@ async function main() { name: 'Wells Fargo', providers: [ { - provider: 'TELLER', + provider: Provider.TELLER, providerId: 'wells_fargo', logoUrl: 'https://teller.io/images/banks/wells_fargo.jpg', }, diff --git a/tools/generators/tellerGenerator.ts b/tools/generators/tellerGenerator.ts index c3f414b1..f0d621ac 100644 --- a/tools/generators/tellerGenerator.ts +++ b/tools/generators/tellerGenerator.ts @@ -263,9 +263,9 @@ export function generateConnection(): GenerateConnectionsResponse { } } -const now = DateTime.fromISO('2022-01-03', { zone: 'utc' }) +export const now = DateTime.fromISO('2022-01-03', { zone: 'utc' }) -const lowerBound = DateTime.fromISO('2021-12-01', { zone: 'utc' }) +export const lowerBound = DateTime.fromISO('2021-12-01', { zone: 'utc' }) export const testDates = { now, @@ -278,3 +278,19 @@ export const testDates = { }, } as Prisma.AccountBalanceWhereInput, } + +export function calculateDailyBalances(startingBalance, transactions, dateInterval) { + transactions.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()) + + const balanceChanges = {} + + transactions.forEach((transaction) => { + const date = new Date(transaction.date).toISOString().split('T')[0] + balanceChanges[date] = (balanceChanges[date] || 0) + Number(transaction.amount) + }) + return dateInterval.map((date) => { + return Object.keys(balanceChanges) + .filter((d) => d <= date) + .reduce((acc, d) => acc + balanceChanges[d], startingBalance) + }) +} From 4b6a441c61f624474bf34c94fa217536afc2d8ff Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:22:58 -0600 Subject: [PATCH 08/22] missed one --- apps/workers/src/app/__tests__/queue.integration.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/workers/src/app/__tests__/queue.integration.spec.ts b/apps/workers/src/app/__tests__/queue.integration.spec.ts index 4d0b1f21..da9f25b7 100644 --- a/apps/workers/src/app/__tests__/queue.integration.spec.ts +++ b/apps/workers/src/app/__tests__/queue.integration.spec.ts @@ -1,9 +1,8 @@ // ===================================================== // Keep these imports above the rest to avoid errors // ===================================================== -import type { SharedType } from '@maybe-finance/shared' import { TellerGenerator } from 'tools/generators' -import type { AccountConnection, User } from '@prisma/client' +import { AccountConnectionType, type User, type AccountConnection } from '@prisma/client' import prisma from '../lib/prisma' import { default as _teller } from '../lib/teller' import { resetUser } from './helpers/user.test-helper' @@ -33,7 +32,7 @@ beforeEach(async () => { connection = await prisma.accountConnection.create({ data: { name: 'Chase Test', - type: 'teller' as SharedType.AccountConnectionType, + type: AccountConnectionType.teller, tellerEnrollmentId: 'test-teller-item-workers', tellerInstitutionId: 'chase_test', tellerAccessToken: From d4984fba311e1ea50072d7f472c2cc6af3f4f96a Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:27:22 -0600 Subject: [PATCH 09/22] revert changes to schema/db and undo changes to account integration spec --- .../app/__tests__/account.integration.spec.ts | 10 +++ .../migration.sql | 7 --- .../migration.sql | 61 ------------------- prisma/schema.prisma | 2 +- 4 files changed, 11 insertions(+), 69 deletions(-) delete mode 100644 prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql delete mode 100644 prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql diff --git a/apps/server/src/app/__tests__/account.integration.spec.ts b/apps/server/src/app/__tests__/account.integration.spec.ts index 83ee4a04..d8ebb18e 100644 --- a/apps/server/src/app/__tests__/account.integration.spec.ts +++ b/apps/server/src/app/__tests__/account.integration.spec.ts @@ -15,11 +15,17 @@ import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import { resetUser } from './utils/user' import { createTestInvestmentAccount } from './utils/account' +import { default as _plaid } from '../lib/plaid' +jest.mock('../middleware/validate-plaid-jwt.ts') jest.mock('bull') +jest.mock('plaid') const prisma = new PrismaClient() +// For TypeScript support +const plaid = jest.mocked(_plaid) // eslint-disable-line + const authId = '__TEST_USER_ID__' let axios: AxiosInstance let user: User @@ -101,6 +107,10 @@ describe('/v1/accounts API', () => { mask: null, isActive: true, syncStatus: 'IDLE', + plaidType: null, + plaidSubtype: null, + plaidAccountId: null, + plaidLiability: null, currencyCode: 'USD', currentBalance: new Decimal(21_000), availableBalance: null, diff --git a/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql b/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql deleted file mode 100644 index 2add40ab..00000000 --- a/prisma/migrations/20240119180135_remove_gen_statement_investment_transaction/migration.sql +++ /dev/null @@ -1,7 +0,0 @@ -ALTER TABLE - "investment_transaction" DROP COLUMN "category"; - -ALTER TABLE - "investment_transaction" -ADD - COLUMN "category" "InvestmentTransactionCategory" DEFAULT 'other' :: "InvestmentTransactionCategory" NOT NULL; diff --git a/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql b/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql deleted file mode 100644 index f47d6fee..00000000 --- a/prisma/migrations/20240119213343_replace_return_calculations_function/migration.sql +++ /dev/null @@ -1,61 +0,0 @@ -CREATE OR REPLACE FUNCTION calculate_return_dietz(p_account_id account.id%type, p_start date, p_end date, out percentage numeric, out amount numeric) AS $$ - DECLARE - v_start date := GREATEST(p_start, (SELECT MIN(date) FROM account_balance WHERE account_id = p_account_id)); - v_end date := p_end; - v_days int := v_end - v_start; - BEGIN - SELECT - ROUND((b1.balance - b0.balance - flows.net) / NULLIF(b0.balance + flows.weighted, 0), 4) AS "percentage", - b1.balance - b0.balance - flows.net AS "amount" - INTO - percentage, amount - FROM - account a - LEFT JOIN LATERAL ( - SELECT - COALESCE(SUM(-fw.flow), 0) AS "net", - COALESCE(SUM(-fw.flow * fw.weight), 0) AS "weighted" - FROM ( - SELECT - SUM(it.amount) AS flow, - (v_days - (it.date - v_start))::numeric / v_days AS weight - FROM - investment_transaction it - WHERE - it.account_id = a.id - AND it.date BETWEEN v_start AND v_end - -- filter for investment_transactions that represent external flows - AND ( - it.category = 'transfer' - ) - GROUP BY - it.date - ) fw - ) flows ON TRUE - LEFT JOIN LATERAL ( - SELECT - ab.balance AS "balance" - FROM - account_balance ab - WHERE - ab.account_id = a.id AND ab.date <= v_start - ORDER BY - ab.date DESC - LIMIT 1 - ) b0 ON TRUE - LEFT JOIN LATERAL ( - SELECT - COALESCE(ab.balance, a.current_balance) AS "balance" - FROM - account_balance ab - WHERE - ab.account_id = a.id AND ab.date <= v_end - ORDER BY - ab.date DESC - LIMIT 1 - ) b1 ON TRUE - WHERE - a.id = p_account_id; - END; -$$ LANGUAGE plpgsql STABLE; - diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9b01c4ae..d358089e 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -250,7 +250,7 @@ model InvestmentTransaction { currencyCode String @default("USD") @map("currency_code") // Derived from provider types - category InvestmentTransactionCategory @default(other) + category InvestmentTransactionCategory @default(dbgenerated("\nCASE\n WHEN (plaid_type = 'buy'::text) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'sell'::text) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['dividend'::text, 'qualified dividend'::text, 'non-qualified dividend'::text])) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['non-resident tax'::text, 'tax'::text, 'tax withheld'::text])) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN ((plaid_type = 'fee'::text) OR (plaid_subtype = ANY (ARRAY['account fee'::text, 'legal fee'::text, 'management fee'::text, 'margin expense'::text, 'transfer fee'::text, 'trust fee'::text]))) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cash'::text) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['purchased'::text, 'purchaseToClose'::text, 'purchaseToCover'::text, 'dividendReinvest'::text, 'reinvestOfIncome'::text])) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['sold'::text, 'soldToClose'::text, 'soldToOpen'::text])) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'dividend'::text) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'tax'::text) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'fee'::text) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = ANY (ARRAY['transfer'::text, 'contribution'::text, 'deposit'::text, 'income'::text, 'interest'::text])) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (finicity_investment_transaction_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n ELSE 'other'::\"InvestmentTransactionCategory\"\nEND")) // plaid data plaidInvestmentTransactionId String? @unique @map("plaid_investment_transaction_id") From 18562477bf0d18f693e7fd8ed1839b4549485dc1 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:29:18 -0600 Subject: [PATCH 10/22] rollback plaid changes in specs --- .../app/__tests__/balance-sync.integration.spec.ts | 7 +++---- .../app/__tests__/connection.integration.spec.ts | 3 --- .../src/app/__tests__/insights.integration.spec.ts | 14 +++++++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts index 0b868ebf..2dcbcb8a 100644 --- a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts +++ b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts @@ -1,5 +1,5 @@ import type { User } from '@prisma/client' -import { PrismaClient, InvestmentTransactionCategory } from '@prisma/client' +import { PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import { @@ -130,7 +130,7 @@ describe('balance sync strategies', () => { amount: 100, quantity: 10, price: 10, - category: InvestmentTransactionCategory.buy, + plaidType: 'buy', }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -139,7 +139,7 @@ describe('balance sync strategies', () => { amount: -50, quantity: 5, price: 10, - category: InvestmentTransactionCategory.sell, + plaidType: 'sell', }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -147,7 +147,6 @@ describe('balance sync strategies', () => { amount: 50, quantity: 50, price: 1, - category: InvestmentTransactionCategory.other, }, ], }, diff --git a/apps/server/src/app/__tests__/connection.integration.spec.ts b/apps/server/src/app/__tests__/connection.integration.spec.ts index ac0e0df1..f6e48a7e 100644 --- a/apps/server/src/app/__tests__/connection.integration.spec.ts +++ b/apps/server/src/app/__tests__/connection.integration.spec.ts @@ -15,9 +15,6 @@ import { resetUser } from './utils/user' jest.mock('../lib/teller.ts') -// For TypeScript support -//const teller = jest.mocked(_teller) - const authId = '__TEST_USER_ID__' let axios: AxiosInstance let user: User | null diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index 16a4ce96..9dbe0433 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,5 +1,5 @@ import type { User } from '@prisma/client' -import { Prisma, PrismaClient, InvestmentTransactionCategory } from '@prisma/client' +import { Prisma, PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import type { @@ -305,7 +305,8 @@ describe('insight service', () => { amount: 50 * 100, quantity: 50, price: 100, - category: InvestmentTransactionCategory.buy, + plaidType: 'buy', + plaidSubtype: 'buy', }, { accountId: account.id, @@ -315,7 +316,8 @@ describe('insight service', () => { amount: 10 * 200, quantity: 10, price: 200, - category: InvestmentTransactionCategory.buy, + plaidType: 'buy', + plaidSubtype: 'buy', }, { accountId: account.id, @@ -325,7 +327,8 @@ describe('insight service', () => { amount: -20.22, quantity: 0, price: 0, - category: InvestmentTransactionCategory.dividend, + plaidType: 'cash', + plaidSubtype: 'dividend', }, { accountId: account.id, @@ -335,7 +338,8 @@ describe('insight service', () => { amount: -22.85, quantity: 0, price: 0, - category: InvestmentTransactionCategory.dividend, + plaidType: 'cash', + plaidSubtype: 'dividend', }, ], }) From 187b84e1c5c59786cbc1afbe5e91ffadeb4b33e0 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:34:34 -0600 Subject: [PATCH 11/22] rollback more stuff --- .../server/src/app/__tests__/utils/account.ts | 37 +- ...tment-transaction-balance-sync.strategy.ts | 3 +- .../src/account/account-query.service.ts | 485 +++++++++--------- .../features/src/account/insight.service.ts | 8 +- 4 files changed, 271 insertions(+), 262 deletions(-) diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 088b2da8..6ba2fd23 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -99,21 +99,7 @@ export async function createTestInvestmentAccount( (s) => s.date === it.date && s.ticker === it.ticker )?.price - function getTransactionCategory(type: string) { - switch (type) { - case 'BUY': - return 'buy' - case 'SELL': - return 'sell' - case 'DIVIDEND': - return 'dividend' - case 'DEPOSIT': - case 'WITHDRAW': - return 'transfer' - default: - return undefined - } - } + const isCashFlow = it.type === 'DEPOSIT' || it.type === 'WITHDRAW' return { securityId: securities.find((s) => it.ticker === s.symbol)?.id, @@ -122,7 +108,26 @@ export async function createTestInvestmentAccount( amount: price ? new Prisma.Decimal(price).times(it.qty) : it.qty, quantity: price ? it.qty : 0, price: price ?? 0, - category: getTransactionCategory(it.type), + plaidType: + isCashFlow || it.type === 'DIVIDEND' + ? 'cash' + : it.type === 'BUY' + ? 'buy' + : it.type === 'SELL' + ? 'sell' + : undefined, + plaidSubtype: + it.type === 'DEPOSIT' + ? 'deposit' + : it.type === 'WITHDRAW' + ? 'withdrawal' + : it.type === 'DIVIDEND' + ? 'dividend' + : it.type === 'BUY' + ? 'buy' + : it.type === 'SELL' + ? 'sell' + : undefined, } }), }, diff --git a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts index 83cc4954..21f0d914 100644 --- a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts +++ b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts @@ -70,7 +70,8 @@ export class InvestmentTransactionBalanceSyncStrategy extends BalanceSyncStrateg it.account_id = ${pAccountId} AND it.date BETWEEN ${pStart} AND now() AND ( -- filter for transactions that modify a position - it.category IN ('buy', 'sell', 'transfer') + it.plaid_type IN ('buy', 'sell', 'transfer') + OR it.finicity_transaction_id IS NOT NULL ) GROUP BY 1, 2 diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index 05e7a511..b13afa78 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -117,33 +117,33 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - h.id, - h.security_id, - s.name, - s.symbol, - s.shares_per_contract, - he.quantity, - he.value, - he.cost_basis, - h.cost_basis_user, - h.cost_basis_provider, - he.cost_basis_per_share, - he.price, - he.price_prev, - he.excluded - FROM - holdings_enriched he - INNER JOIN security s ON s.id = he.security_id - INNER JOIN holding h ON h.id = he.id - WHERE - he.account_id = ${accountId} - ORDER BY - he.excluded ASC, - he.value DESC - OFFSET ${page * pageSize} - LIMIT ${pageSize}; - ` + SELECT + h.id, + h.security_id, + s.name, + s.symbol, + s.shares_per_contract, + he.quantity, + he.value, + he.cost_basis, + h.cost_basis_user, + h.cost_basis_provider, + he.cost_basis_per_share, + he.price, + he.price_prev, + he.excluded + FROM + holdings_enriched he + INNER JOIN security s ON s.id = he.security_id + INNER JOIN holding h ON h.id = he.id + WHERE + he.account_id = ${accountId} + ORDER BY + he.excluded ASC, + he.value DESC + OFFSET ${page * pageSize} + LIMIT ${pageSize}; + ` ) return rows @@ -158,53 +158,53 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH valuation_trends AS ( - SELECT - date, - COALESCE(interpolated::numeric, filled) AS amount - FROM ( - SELECT - time_bucket_gapfill('1d', v.date) AS date, - interpolate(avg(v.amount)) AS interpolated, - locf(avg(v.amount)) AS filled - FROM - valuation v - WHERE - v.account_id = ${accountId} - AND v.date BETWEEN ${pStart} AND ${pEnd} - GROUP BY - 1 - ) valuations_gapfilled - WHERE - to_char(date, 'MM-DD') = '01-01' - ), valuations_combined AS ( - SELECT - COALESCE(v.date, vt.date) AS date, - COALESCE(v.amount, vt.amount) AS amount, - v.id AS valuation_id - FROM - (SELECT * FROM valuation WHERE account_id = ${accountId}) v - FULL OUTER JOIN valuation_trends vt ON vt.date = v.date - ) + WITH valuation_trends AS ( SELECT - v.date, - v.amount, - v.valuation_id, - v.amount - v.prev_amount AS period_change, - ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, - v.amount - v.first_amount AS total_change, - ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct + date, + COALESCE(interpolated::numeric, filled) AS amount FROM ( SELECT - *, - LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, - (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount + time_bucket_gapfill('1d', v.date) AS date, + interpolate(avg(v.amount)) AS interpolated, + locf(avg(v.amount)) AS filled FROM - valuations_combined - ) v - ORDER BY - v.date ASC - ` + valuation v + WHERE + v.account_id = ${accountId} + AND v.date BETWEEN ${pStart} AND ${pEnd} + GROUP BY + 1 + ) valuations_gapfilled + WHERE + to_char(date, 'MM-DD') = '01-01' + ), valuations_combined AS ( + SELECT + COALESCE(v.date, vt.date) AS date, + COALESCE(v.amount, vt.amount) AS amount, + v.id AS valuation_id + FROM + (SELECT * FROM valuation WHERE account_id = ${accountId}) v + FULL OUTER JOIN valuation_trends vt ON vt.date = v.date + ) + SELECT + v.date, + v.amount, + v.valuation_id, + v.amount - v.prev_amount AS period_change, + ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, + v.amount - v.first_amount AS total_change, + ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct + FROM ( + SELECT + *, + LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, + (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount + FROM + valuations_combined + ) v + ORDER BY + v.date ASC + ` ) return rows @@ -220,77 +220,80 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH start_date AS ( - SELECT - a.id AS "account_id", - GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" - FROM - account a - WHERE - a.id = ANY(${pAccountIds}) - GROUP BY - 1 - ), external_flows AS ( - SELECT - it.account_id, - it.date, - SUM(it.amount) AS "amount" - FROM - investment_transaction it - LEFT JOIN start_date sd ON sd.account_id = it.account_id - WHERE - it.account_id = ANY(${pAccountIds}) - AND it.date BETWEEN sd.start_date AND ${pEnd} - -- filter for investment_transactions that represent external flows - AND ( - it.category = 'transfer' - ) - GROUP BY - 1, 2 - ), external_flow_totals AS ( - SELECT - account_id, - SUM(amount) as "amount" - FROM - external_flows - GROUP BY - 1 - ), balances AS ( - SELECT - abg.account_id, - abg.date, - abg.balance, - 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", - COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - '1d', - ${pAccountIds} - ) abg - LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date - LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id - ) + WITH start_date AS ( SELECT - b.account_id, - b.date, - b.balance, - b.contributions, - b.contributions_period, - COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" + a.id AS "account_id", + GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" FROM - balances b - LEFT JOIN ( - SELECT DISTINCT ON (account_id) - account_id, - balance - FROM - balances - ORDER BY - account_id, date ASC - ) b0 ON b0.account_id = b.account_id - ` + account a + WHERE + a.id = ANY(${pAccountIds}) + GROUP BY + 1 + ), external_flows AS ( + SELECT + it.account_id, + it.date, + SUM(it.amount) AS "amount" + FROM + investment_transaction it + LEFT JOIN start_date sd ON sd.account_id = it.account_id + WHERE + it.account_id = ANY(${pAccountIds}) + AND it.date BETWEEN sd.start_date AND ${pEnd} + -- filter for investment_transactions that represent external flows + AND ( + (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) + OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) + OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) + OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) + ) + GROUP BY + 1, 2 + ), external_flow_totals AS ( + SELECT + account_id, + SUM(amount) as "amount" + FROM + external_flows + GROUP BY + 1 + ), balances AS ( + SELECT + abg.account_id, + abg.date, + abg.balance, + 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", + COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + '1d', + ${pAccountIds} + ) abg + LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date + LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id + ) + SELECT + b.account_id, + b.date, + b.balance, + b.contributions, + b.contributions_period, + COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" + FROM + balances b + LEFT JOIN ( + SELECT DISTINCT ON (account_id) + account_id, + balance + FROM + balances + ORDER BY + account_id, date ASC + ) b0 ON b0.account_id = b.account_id + ` ) return rows @@ -310,18 +313,18 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - SELECT - abg.account_id, - abg.date, - abg.balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - ` + SELECT + abg.account_id, + abg.date, + abg.balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + ` ) return rows @@ -341,15 +344,15 @@ export class AccountQueryService implements IAccountQueryService { 'accountIds' in id ? id.accountIds : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active - )` + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -376,27 +379,27 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - abg.date, - a.category, - a.classification, - SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category), - (abg.date, a.classification), - (abg.date) - ) - ORDER BY date ASC; - ` + SELECT + abg.date, + a.category, + a.classification, + SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category), + (abg.date, a.classification), + (abg.date) + ) + ORDER BY date ASC; + ` ) // Group independent rows into NetWorthSeries objects @@ -450,15 +453,15 @@ export class AccountQueryService implements IAccountQueryService { 'accountId' in id ? [id.accountId] : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active - )` + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -466,59 +469,59 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH account_rollup AS ( - SELECT - abg.date, - a.classification, - a.category, - a.id, - SUM(abg.balance) AS balance, - CASE GROUPING(abg.date, a.classification, a.category, a.id) - WHEN 3 THEN 'classification' - WHEN 1 THEN 'category' - WHEN 0 THEN 'account' - ELSE NULL - END AS grouping - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category, a.id), - (abg.date, a.classification, a.category), - (abg.date, a.classification) - ) - ) + WITH account_rollup AS ( SELECT - ar.date, - ar.classification, - ar.category, - ar.id, - ar.balance, - ar.grouping, - CASE - WHEN a.id IS NULL THEN NULL - ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) - END AS account, - ROUND( - CASE ar.grouping - WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) - WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) - WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) - END, 4) AS rollup_pct, - ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct + abg.date, + a.classification, + a.category, + a.id, + SUM(abg.balance) AS balance, + CASE GROUPING(abg.date, a.classification, a.category, a.id) + WHEN 3 THEN 'classification' + WHEN 1 THEN 'category' + WHEN 0 THEN 'account' + ELSE NULL + END AS grouping FROM - account_rollup ar - LEFT JOIN account a ON a.id = ar.id - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - ORDER BY - ar.classification, ar.category, ar.id, ar.date; - ` + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category, a.id), + (abg.date, a.classification, a.category), + (abg.date, a.classification) + ) + ) + SELECT + ar.date, + ar.classification, + ar.category, + ar.id, + ar.balance, + ar.grouping, + CASE + WHEN a.id IS NULL THEN NULL + ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) + END AS account, + ROUND( + CASE ar.grouping + WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) + WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) + WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) + END, 4) AS rollup_pct, + ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct + FROM + account_rollup ar + LEFT JOIN account a ON a.id = ar.id + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + ORDER BY + ar.classification, ar.category, ar.id, ar.date; + ` ) return rows diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index 0eb8b13a..4c736ab7 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -315,9 +315,6 @@ export class InsightService implements IInsightService { { finicityInvestmentTransactionType: 'dividend', }, - { - category: 'dividend', - }, ], }, }), @@ -750,7 +747,10 @@ export class InsightService implements IInsightService { WHERE it.account_id = ${accountId} AND ( - it.category = 'transfer' + (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) + OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer', 'send', 'request')) + OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) + OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) ) -- Exclude any contributions made prior to the start date since balances will be 0 AND (a.start_date is NULL OR it.date >= a.start_date) From 25cdf0c1be9410678a771fdd9c26794febbbb37c Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:38:11 -0600 Subject: [PATCH 12/22] what is happening --- .../src/account/account-query.service.ts | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index b13afa78..08845d99 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -158,53 +158,53 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH valuation_trends AS ( + WITH valuation_trends AS ( + SELECT + date, + COALESCE(interpolated::numeric, filled) AS amount + FROM ( + SELECT + time_bucket_gapfill('1d', v.date) AS date, + interpolate(avg(v.amount)) AS interpolated, + locf(avg(v.amount)) AS filled + FROM + valuation v + WHERE + v.account_id = ${accountId} + AND v.date BETWEEN ${pStart} AND ${pEnd} + GROUP BY + 1 + ) valuations_gapfilled + WHERE + to_char(date, 'MM-DD') = '01-01' + ), valuations_combined AS ( + SELECT + COALESCE(v.date, vt.date) AS date, + COALESCE(v.amount, vt.amount) AS amount, + v.id AS valuation_id + FROM + (SELECT * FROM valuation WHERE account_id = ${accountId}) v + FULL OUTER JOIN valuation_trends vt ON vt.date = v.date + ) SELECT - date, - COALESCE(interpolated::numeric, filled) AS amount + v.date, + v.amount, + v.valuation_id, + v.amount - v.prev_amount AS period_change, + ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, + v.amount - v.first_amount AS total_change, + ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct FROM ( SELECT - time_bucket_gapfill('1d', v.date) AS date, - interpolate(avg(v.amount)) AS interpolated, - locf(avg(v.amount)) AS filled + *, + LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, + (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount FROM - valuation v - WHERE - v.account_id = ${accountId} - AND v.date BETWEEN ${pStart} AND ${pEnd} - GROUP BY - 1 - ) valuations_gapfilled - WHERE - to_char(date, 'MM-DD') = '01-01' - ), valuations_combined AS ( - SELECT - COALESCE(v.date, vt.date) AS date, - COALESCE(v.amount, vt.amount) AS amount, - v.id AS valuation_id - FROM - (SELECT * FROM valuation WHERE account_id = ${accountId}) v - FULL OUTER JOIN valuation_trends vt ON vt.date = v.date - ) - SELECT - v.date, - v.amount, - v.valuation_id, - v.amount - v.prev_amount AS period_change, - ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, - v.amount - v.first_amount AS total_change, - ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct - FROM ( - SELECT - *, - LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, - (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount - FROM - valuations_combined - ) v - ORDER BY - v.date ASC - ` + valuations_combined + ) v + ORDER BY + v.date ASC + ` ) return rows From c1f9664c5ea68ad498064f2c536a3e8f185b7158 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:43:48 -0600 Subject: [PATCH 13/22] fix sql formatting --- .../src/account/account-query.service.ts | 392 +++++++++--------- 1 file changed, 196 insertions(+), 196 deletions(-) diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index 08845d99..f12d5e75 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -117,32 +117,32 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - h.id, - h.security_id, - s.name, - s.symbol, - s.shares_per_contract, - he.quantity, - he.value, - he.cost_basis, - h.cost_basis_user, - h.cost_basis_provider, - he.cost_basis_per_share, - he.price, - he.price_prev, - he.excluded - FROM - holdings_enriched he - INNER JOIN security s ON s.id = he.security_id - INNER JOIN holding h ON h.id = he.id - WHERE - he.account_id = ${accountId} - ORDER BY - he.excluded ASC, - he.value DESC - OFFSET ${page * pageSize} - LIMIT ${pageSize}; + SELECT + h.id, + h.security_id, + s.name, + s.symbol, + s.shares_per_contract, + he.quantity, + he.value, + he.cost_basis, + h.cost_basis_user, + h.cost_basis_provider, + he.cost_basis_per_share, + he.price, + he.price_prev, + he.excluded + FROM + holdings_enriched he + INNER JOIN security s ON s.id = he.security_id + INNER JOIN holding h ON h.id = he.id + WHERE + he.account_id = ${accountId} + ORDER BY + he.excluded ASC, + he.value DESC + OFFSET ${page * pageSize} + LIMIT ${pageSize}; ` ) @@ -204,7 +204,7 @@ export class AccountQueryService implements IAccountQueryService { ) v ORDER BY v.date ASC - ` + ` ) return rows @@ -220,79 +220,79 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH start_date AS ( - SELECT - a.id AS "account_id", - GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" - FROM - account a - WHERE - a.id = ANY(${pAccountIds}) - GROUP BY - 1 - ), external_flows AS ( - SELECT - it.account_id, - it.date, - SUM(it.amount) AS "amount" - FROM - investment_transaction it - LEFT JOIN start_date sd ON sd.account_id = it.account_id - WHERE - it.account_id = ANY(${pAccountIds}) - AND it.date BETWEEN sd.start_date AND ${pEnd} - -- filter for investment_transactions that represent external flows - AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) - ) - GROUP BY - 1, 2 - ), external_flow_totals AS ( - SELECT - account_id, - SUM(amount) as "amount" - FROM - external_flows - GROUP BY - 1 - ), balances AS ( - SELECT - abg.account_id, - abg.date, - abg.balance, - 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", - COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - '1d', - ${pAccountIds} - ) abg - LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date - LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id - ) - SELECT - b.account_id, - b.date, - b.balance, - b.contributions, - b.contributions_period, - COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" - FROM - balances b - LEFT JOIN ( - SELECT DISTINCT ON (account_id) - account_id, - balance + WITH start_date AS ( + SELECT + a.id AS "account_id", + GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" FROM - balances - ORDER BY - account_id, date ASC - ) b0 ON b0.account_id = b.account_id + account a + WHERE + a.id = ANY(${pAccountIds}) + GROUP BY + 1 + ), external_flows AS ( + SELECT + it.account_id, + it.date, + SUM(it.amount) AS "amount" + FROM + investment_transaction it + LEFT JOIN start_date sd ON sd.account_id = it.account_id + WHERE + it.account_id = ANY(${pAccountIds}) + AND it.date BETWEEN sd.start_date AND ${pEnd} + -- filter for investment_transactions that represent external flows + AND ( + (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) + OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) + OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) + OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) + ) + GROUP BY + 1, 2 + ), external_flow_totals AS ( + SELECT + account_id, + SUM(amount) as "amount" + FROM + external_flows + GROUP BY + 1 + ), balances AS ( + SELECT + abg.account_id, + abg.date, + abg.balance, + 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", + COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + '1d', + ${pAccountIds} + ) abg + LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date + LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id + ) + SELECT + b.account_id, + b.date, + b.balance, + b.contributions, + b.contributions_period, + COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" + FROM + balances b + LEFT JOIN ( + SELECT DISTINCT ON (account_id) + account_id, + balance + FROM + balances + ORDER BY + account_id, date ASC + ) b0 ON b0.account_id = b.account_id ` ) @@ -313,17 +313,17 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - SELECT - abg.account_id, - abg.date, - abg.balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg + SELECT + abg.account_id, + abg.date, + abg.balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg ` ) @@ -344,14 +344,14 @@ export class AccountQueryService implements IAccountQueryService { 'accountIds' in id ? id.accountIds : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active )` const pStart = raw(`'${start}'`) @@ -379,26 +379,26 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - abg.date, - a.category, - a.classification, - SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category), - (abg.date, a.classification), - (abg.date) - ) - ORDER BY date ASC; + SELECT + abg.date, + a.category, + a.classification, + SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category), + (abg.date, a.classification), + (abg.date) + ) + ORDER BY date ASC; ` ) @@ -453,14 +453,14 @@ export class AccountQueryService implements IAccountQueryService { 'accountId' in id ? [id.accountId] : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active )` const pStart = raw(`'${start}'`) @@ -469,58 +469,58 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH account_rollup AS ( + WITH account_rollup AS ( + SELECT + abg.date, + a.classification, + a.category, + a.id, + SUM(abg.balance) AS balance, + CASE GROUPING(abg.date, a.classification, a.category, a.id) + WHEN 3 THEN 'classification' + WHEN 1 THEN 'category' + WHEN 0 THEN 'account' + ELSE NULL + END AS grouping + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category, a.id), + (abg.date, a.classification, a.category), + (abg.date, a.classification) + ) + ) SELECT - abg.date, - a.classification, - a.category, - a.id, - SUM(abg.balance) AS balance, - CASE GROUPING(abg.date, a.classification, a.category, a.id) - WHEN 3 THEN 'classification' - WHEN 1 THEN 'category' - WHEN 0 THEN 'account' - ELSE NULL - END AS grouping + ar.date, + ar.classification, + ar.category, + ar.id, + ar.balance, + ar.grouping, + CASE + WHEN a.id IS NULL THEN NULL + ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) + END AS account, + ROUND( + CASE ar.grouping + WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) + WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) + WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) + END, 4) AS rollup_pct, + ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category, a.id), - (abg.date, a.classification, a.category), - (abg.date, a.classification) - ) - ) - SELECT - ar.date, - ar.classification, - ar.category, - ar.id, - ar.balance, - ar.grouping, - CASE - WHEN a.id IS NULL THEN NULL - ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) - END AS account, - ROUND( - CASE ar.grouping - WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) - WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) - WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) - END, 4) AS rollup_pct, - ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct - FROM - account_rollup ar - LEFT JOIN account a ON a.id = ar.id - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - ORDER BY - ar.classification, ar.category, ar.id, ar.date; + account_rollup ar + LEFT JOIN account a ON a.id = ar.id + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + ORDER BY + ar.classification, ar.category, ar.id, ar.date; ` ) From fc3302f150cf485046965e87569210df3d66184b Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:46:01 -0600 Subject: [PATCH 14/22] spacing --- .../src/account/account-query.service.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index f12d5e75..fa206904 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -143,7 +143,7 @@ export class AccountQueryService implements IAccountQueryService { he.value DESC OFFSET ${page * pageSize} LIMIT ${pageSize}; - ` + ` ) return rows @@ -204,7 +204,7 @@ export class AccountQueryService implements IAccountQueryService { ) v ORDER BY v.date ASC - ` + ` ) return rows @@ -293,7 +293,7 @@ export class AccountQueryService implements IAccountQueryService { ORDER BY account_id, date ASC ) b0 ON b0.account_id = b.account_id - ` + ` ) return rows @@ -324,7 +324,7 @@ export class AccountQueryService implements IAccountQueryService { ${pInterval}, ${pAccountIds} ) abg - ` + ` ) return rows @@ -352,7 +352,7 @@ export class AccountQueryService implements IAccountQueryService { WHERE (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) AND a.is_active - )` + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -399,7 +399,7 @@ export class AccountQueryService implements IAccountQueryService { (abg.date) ) ORDER BY date ASC; - ` + ` ) // Group independent rows into NetWorthSeries objects @@ -461,7 +461,7 @@ export class AccountQueryService implements IAccountQueryService { WHERE (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) AND a.is_active - )` + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -521,7 +521,7 @@ export class AccountQueryService implements IAccountQueryService { LEFT JOIN account_connection ac ON ac.id = a.account_connection_id ORDER BY ar.classification, ar.category, ar.id, ar.date; - ` + ` ) return rows From 96059ad2f7de9dbd08aeec6c6d36c132e2e3130e Mon Sep 17 00:00:00 2001 From: Karan Handa Date: Sat, 20 Jan 2024 13:30:59 +0530 Subject: [PATCH 15/22] remove redundant state --- apps/client/pages/_app.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/client/pages/_app.tsx b/apps/client/pages/_app.tsx index 0a4affac..ae4d14db 100644 --- a/apps/client/pages/_app.tsx +++ b/apps/client/pages/_app.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState, type PropsWithChildren, type ReactElement } from 'react' +import { useEffect, type PropsWithChildren, type ReactElement } from 'react' import type { AppProps } from 'next/app' import { ErrorBoundary } from 'react-error-boundary' import { Analytics } from '@vercel/analytics/react' @@ -34,12 +34,10 @@ Sentry.init({ // Providers and components only relevant to a logged-in user const WithAuth = function ({ children }: PropsWithChildren) { const { data: session, status } = useSession() - const [isLoading, setLoading] = useState(true) const router = useRouter() useEffect(() => { - if (status === 'loading') return // Wait until the session status is not 'loading' - setLoading(false) + if (status === 'loading') return if (!session) { router.push('/login') From 1cc9303b8b6a98df2ebf1f931f28b470ab504db0 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Sat, 20 Jan 2024 07:48:03 -0600 Subject: [PATCH 16/22] imports --- .../src/app/__tests__/connection.integration.spec.ts | 9 ++------- apps/server/src/app/__tests__/utils/account.ts | 2 +- apps/workers/src/app/__tests__/queue.integration.spec.ts | 3 ++- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/apps/server/src/app/__tests__/connection.integration.spec.ts b/apps/server/src/app/__tests__/connection.integration.spec.ts index f6e48a7e..a8bfa721 100644 --- a/apps/server/src/app/__tests__/connection.integration.spec.ts +++ b/apps/server/src/app/__tests__/connection.integration.spec.ts @@ -1,11 +1,6 @@ import type { AxiosInstance } from 'axios' -import { - type Prisma, - type AccountConnection, - type User, - AccountConnectionType, - AccountSyncStatus, -} from '@prisma/client' +import type { Prisma, AccountConnection, User } from '@prisma/client' +import { AccountConnectionType, AccountSyncStatus } from '@prisma/client' import { startServer, stopServer } from './utils/server' import { getAxiosClient } from './utils/axios' import prisma from '../lib/prisma' diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 6ba2fd23..47e26cc3 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -35,7 +35,7 @@ export async function createTestInvestmentAccount( join(__dirname, `../test-data/${portfolio}/holdings.csv`) ) - const [, ...securities] = await prisma.$transaction([ + const [_deleted, ...securities] = await prisma.$transaction([ prisma.security.deleteMany({ where: { symbol: { diff --git a/apps/workers/src/app/__tests__/queue.integration.spec.ts b/apps/workers/src/app/__tests__/queue.integration.spec.ts index da9f25b7..2eff8ced 100644 --- a/apps/workers/src/app/__tests__/queue.integration.spec.ts +++ b/apps/workers/src/app/__tests__/queue.integration.spec.ts @@ -2,7 +2,8 @@ // Keep these imports above the rest to avoid errors // ===================================================== import { TellerGenerator } from 'tools/generators' -import { AccountConnectionType, type User, type AccountConnection } from '@prisma/client' +import type { User, AccountConnection } from '@prisma/client' +import { AccountConnectionType } from '@prisma/client' import prisma from '../lib/prisma' import { default as _teller } from '../lib/teller' import { resetUser } from './helpers/user.test-helper' From 14b45aee2bf3aef335b8baea0b261c0ef481d2b7 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Sat, 20 Jan 2024 12:06:50 -0600 Subject: [PATCH 17/22] fix sign --- .../features/src/providers/teller/teller.etl.ts | 14 +++++++++++++- libs/server/shared/src/utils/teller-utils.ts | 10 +++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index ac4e8315..51330589 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -1,4 +1,5 @@ import type { AccountConnection, PrismaClient } from '@prisma/client' +import { AccountClassification } from '@prisma/client' import type { Logger } from 'winston' import { AccountUtil, SharedUtil, type SharedType } from '@maybe-finance/shared' import type { TellerApi, TellerTypes } from '@maybe-finance/teller-api' @@ -164,6 +165,12 @@ export class TellerETL implements IETL { private async _extractTransactions(accessToken: string, accountIds: string[]) { const accountTransactions = await Promise.all( accountIds.map(async (accountId) => { + const account = await this.prisma.account.findFirst({ + where: { + tellerAccountId: accountId, + }, + }) + const transactions = await SharedUtil.withRetry( () => this.teller.getTransactions({ @@ -174,6 +181,11 @@ export class TellerETL implements IETL { maxRetries: 3, } ) + if (account!.classification === AccountClassification.liability) { + transactions.forEach((t) => { + t.amount = String(Number(t.amount) * -1) + }) + } return transactions }) @@ -214,7 +226,7 @@ export class TellerETL implements IETL { ${transactionId}, ${date}::date, ${description}, - ${DbUtil.toDecimal(Number(amount))}, + ${DbUtil.toDecimal(Number(-amount))}, ${status === 'pending'}, ${'USD'}, ${details.counterparty?.name ?? ''}, diff --git a/libs/server/shared/src/utils/teller-utils.ts b/libs/server/shared/src/utils/teller-utils.ts index 6e76cf1b..32102afb 100644 --- a/libs/server/shared/src/utils/teller-utils.ts +++ b/libs/server/shared/src/utils/teller-utils.ts @@ -1,10 +1,6 @@ -import { - Prisma, - AccountCategory, - AccountType, - type Account, - type AccountClassification, -} from '@prisma/client' +import { Prisma, AccountCategory, AccountType } from '@prisma/client' +import type { AccountClassification } from '@prisma/client' +import type { Account } from '@prisma/client' import type { TellerTypes } from '@maybe-finance/teller-api' import { Duration } from 'luxon' From cd740d57bc7fdace3d5506a607aca3a29b7c9b9e Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Sat, 20 Jan 2024 12:09:55 -0600 Subject: [PATCH 18/22] simplify --- libs/server/features/src/providers/teller/teller.etl.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index 51330589..6891de97 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -181,7 +181,7 @@ export class TellerETL implements IETL { maxRetries: 3, } ) - if (account!.classification === AccountClassification.liability) { + if (account!.classification === AccountClassification.asset) { transactions.forEach((t) => { t.amount = String(Number(t.amount) * -1) }) @@ -226,7 +226,7 @@ export class TellerETL implements IETL { ${transactionId}, ${date}::date, ${description}, - ${DbUtil.toDecimal(Number(-amount))}, + ${DbUtil.toDecimal(Number(amount))}, ${status === 'pending'}, ${'USD'}, ${details.counterparty?.name ?? ''}, From d6b4a75231926cd8b0fa4564de0c4c66e0562140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20De=20Boey?= Date: Sat, 20 Jan 2024 21:56:50 +0100 Subject: [PATCH 19/22] refactor: remove `Transaction` `category` DB generation --- .../features/src/providers/plaid/plaid.etl.ts | 69 ++++++++++++++++++- .../src/providers/teller/teller.etl.ts | 40 ++++++++++- .../migration.sql | 63 +++++++++++++++++ prisma/schema.prisma | 2 +- 4 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql diff --git a/libs/server/features/src/providers/plaid/plaid.etl.ts b/libs/server/features/src/providers/plaid/plaid.etl.ts index ad16b62e..1c8159d4 100644 --- a/libs/server/features/src/providers/plaid/plaid.etl.ts +++ b/libs/server/features/src/providers/plaid/plaid.etl.ts @@ -13,6 +13,7 @@ import type { Item as PlaidItem, LiabilitiesObject as PlaidLiabilities, PlaidApi, + PersonalFinanceCategory, } from 'plaid' import { Prisma } from '@prisma/client' import { DateTime } from 'luxon' @@ -366,7 +367,7 @@ export class PlaidETL implements IETL { const txnUpsertQueries = chunk(transactions, 1_000).map((chunk) => { return this.prisma.$executeRaw` - INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category) + INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category, category) VALUES ${Prisma.join( chunk.map((plaidTransaction) => { @@ -401,7 +402,8 @@ export class PlaidETL implements IETL { ${merchant_name}, ${category ?? []}, ${category_id}, - ${personal_finance_category} + ${personal_finance_category}, + ${this.getMaybeTransactionCategory(personal_finance_category)} )` }) )} @@ -414,6 +416,7 @@ export class PlaidETL implements IETL { plaid_category = EXCLUDED.plaid_category, plaid_category_id = EXCLUDED.plaid_category_id, plaid_personal_finance_category = EXCLUDED.plaid_personal_finance_category; + category = EXCLUDED.category; ` }) @@ -444,6 +447,68 @@ export class PlaidETL implements IETL { ] } + private getMaybeTransactionCategory = (category?: PersonalFinanceCategory | null) => { + if (!category) { + return 'Other' + } + + if (category.primary === 'INCOME') { + return 'Income' + } + + if ( + ['LOAN_PAYMENTS_MORTGAGE_PAYMENT', 'RENT_AND_UTILITIES_RENT'].includes( + category.detailed + ) + ) { + return 'Housing Payments' + } + + if (category.detailed === 'LOAN_PAYMENTS_CAR_PAYMENT') { + return 'Vehicle Payments' + } + + if (category.primary === 'LOAN_PAYMENTS') { + return 'Other Payments' + } + + if (category.primary === 'HOME_IMPROVEMENT') { + return 'Home Improvement' + } + + if (category.primary === 'GENERAL_MERCHANDISE') { + return 'Shopping' + } + + if ( + category.primary === 'RENT_AND_UTILITIES' && + category.detailed !== 'RENT_AND_UTILITIES_RENT' + ) { + return 'Utilities' + } + + if (category.primary === 'FOOD_AND_DRINK') { + return 'Food and Drink' + } + + if (category.primary === 'TRANSPORTATION') { + return 'Transportation' + } + + if (category.primary === 'TRAVEL') { + return 'Travel' + } + + if ( + ['PERSONAL_CARE', 'MEDICAL'].includes(category.primary) && + category.detailed !== 'MEDICAL_VETERINARY_SERVICES' + ) { + return 'Health' + } + + return 'Other' + } + private _extractInvestmentTransactions(accessToken: string, dateRange: SharedType.DateRange) { return SharedUtil.paginate({ pageSize: 500, // https://plaid.com/docs/api/products/investments/#investments-transactions-get-request-options-count diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index 6891de97..406f70c3 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -25,6 +25,40 @@ type Connection = Pick< 'id' | 'userId' | 'tellerInstitutionId' | 'tellerAccessToken' > +const maybeCategoryByTellerCategory: Record< + Required['category'], + string +> = { + accommodation: 'Travel', + advertising: 'Other', + bar: 'Food and Drink', + charity: 'Other', + clothing: 'Shopping', + dining: 'Food and Drink', + education: 'Other', + electronics: 'Shopping', + entertainment: 'Shopping', + fuel: 'Transportation', + general: 'Other', + groceries: 'Food and Drink', + health: 'Health', + home: 'Home Improvement', + income: 'Income', + insurance: 'Other', + investment: 'Other', + loan: 'Other', + office: 'Other', + phone: 'Utilities', + service: 'Other', + shopping: 'Shopping', + software: 'Shopping', + sport: 'Shopping', + tax: 'Other', + transport: 'Transportation', + transportation: 'Transportation', + utilities: 'Utilities', +} + export class TellerETL implements IETL { public constructor( private readonly logger: Logger, @@ -204,7 +238,7 @@ export class TellerETL implements IETL { const txnUpsertQueries = _.chunk(transactions, 1_000).map((chunk) => { return this.prisma.$executeRaw` - INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category) + INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category, category) VALUES ${Prisma.join( chunk.map((tellerTransaction) => { @@ -231,7 +265,8 @@ export class TellerETL implements IETL { ${'USD'}, ${details.counterparty?.name ?? ''}, ${type}, - ${details.category ?? ''} + ${details.category ?? ''}, + ${maybeCategoryByTellerCategory[details.category ?? ''] ?? 'Other'} )` }) )} @@ -243,6 +278,7 @@ export class TellerETL implements IETL { merchant_name = EXCLUDED.merchant_name, teller_type = EXCLUDED.teller_type, teller_category = EXCLUDED.teller_category; + category = EXCLUDED.category; ` }) diff --git a/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql b/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql new file mode 100644 index 00000000..1acfb3ad --- /dev/null +++ b/prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql @@ -0,0 +1,63 @@ +-- AlterTable +ALTER TABLE "transaction" + RENAME COLUMN "category" TO "category_old"; + +DROP VIEW IF EXISTS transactions_enriched; + +ALTER TABLE "transaction" + ADD COLUMN "category" TEXT NOT NULL DEFAULT 'Other'::text; + +CREATE OR REPLACE VIEW transactions_enriched AS ( + SELECT + t.id, + t.created_at as "createdAt", + t.updated_at as "updatedAt", + t.name, + t.account_id as "accountId", + t.date, + t.flow, + COALESCE( + t.type_user, + CASE + -- no matching transaction + WHEN t.match_id IS NULL THEN ( + CASE + t.flow + WHEN 'INFLOW' THEN ( + CASE + a.classification + WHEN 'asset' THEN 'INCOME' :: "TransactionType" + WHEN 'liability' THEN 'PAYMENT' :: "TransactionType" + END + ) + WHEN 'OUTFLOW' THEN 'EXPENSE' :: "TransactionType" + END + ) -- has matching transaction + ELSE ( + CASE + a.classification + WHEN 'asset' THEN 'TRANSFER' :: "TransactionType" + WHEN 'liability' THEN 'PAYMENT' :: "TransactionType" + END + ) + END + ) AS "type", + t.type_user as "typeUser", + t.amount, + t.currency_code as "currencyCode", + t.pending, + t.merchant_name as "merchantName", + t.category, + t.category_user as "categoryUser", + t.excluded, + t.match_id as "matchId", + COALESCE(ac.user_id, a.user_id) as "userId", + a.classification as "accountClassification", + a.type as "accountType" + FROM + transaction t + inner join account a on a.id = t.account_id + left join account_connection ac on a.account_connection_id = ac.id +); + +ALTER TABLE "transaction" DROP COLUMN "category_old"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4242132e..78f96a4c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -312,7 +312,7 @@ model Transaction { currencyCode String @default("USD") @map("currency_code") pending Boolean @default(false) merchantName String? @map("merchant_name") - category String @default(dbgenerated("COALESCE(category_user,\nCASE\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'INCOME'::text) THEN 'Income'::text\n WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = ANY (ARRAY['LOAN_PAYMENTS_MORTGAGE_PAYMENT'::text, 'RENT_AND_UTILITIES_RENT'::text])) THEN 'Housing Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = 'LOAN_PAYMENTS_CAR_PAYMENT'::text) THEN 'Vehicle Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'LOAN_PAYMENTS'::text) THEN 'Other Payments'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'HOME_IMPROVEMENT'::text) THEN 'Home Improvement'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'GENERAL_MERCHANDISE'::text) THEN 'Shopping'::text\n WHEN (((plaid_personal_finance_category ->> 'primary'::text) = 'RENT_AND_UTILITIES'::text) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'RENT_AND_UTILITIES_RENT'::text)) THEN 'Utilities'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'FOOD_AND_DRINK'::text) THEN 'Food and Drink'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRANSPORTATION'::text) THEN 'Transportation'::text\n WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRAVEL'::text) THEN 'Travel'::text\n WHEN (((plaid_personal_finance_category ->> 'primary'::text) = ANY (ARRAY['PERSONAL_CARE'::text, 'MEDICAL'::text])) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'MEDICAL_VETERINARY_SERVICES'::text)) THEN 'Health'::text\n WHEN (teller_category = 'income'::text) THEN 'Income'::text\n WHEN (teller_category = 'home'::text) THEN 'Home Improvement'::text\n WHEN (teller_category = ANY (ARRAY['phone'::text, 'utilities'::text])) THEN 'Utilities'::text\n WHEN (teller_category = ANY (ARRAY['dining'::text, 'bar'::text, 'groceries'::text])) THEN 'Food and Drink'::text\n WHEN (teller_category = ANY (ARRAY['clothing'::text, 'entertainment'::text, 'shopping'::text, 'electronics'::text, 'software'::text, 'sport'::text])) THEN 'Shopping'::text\n WHEN (teller_category = ANY (ARRAY['transportation'::text, 'fuel'::text])) THEN 'Transportation'::text\n WHEN (teller_category = ANY (ARRAY['accommodation'::text, 'transport'::text])) THEN 'Travel'::text\n WHEN (teller_category = 'health'::text) THEN 'Health'::text\n WHEN (teller_category = ANY (ARRAY['loan'::text, 'tax'::text, 'insurance'::text, 'office'::text])) THEN 'Other Payments'::text\n ELSE 'Other'::text\nEND)")) + category String @default("Other") categoryUser String? @map("category_user") excluded Boolean @default(false) From 8276c56f74839ec7f36ac357d54b47ceaafa1ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20De=20Boey?= Date: Sat, 20 Jan 2024 22:11:26 +0100 Subject: [PATCH 20/22] refactor: remove `InvestmentTransaction` `category` DB generation --- .../balance-sync.integration.spec.ts | 4 + .../__tests__/insights.integration.spec.ts | 6 +- .../server/src/app/__tests__/utils/account.ts | 19 ++- ...tment-transaction-balance-sync.strategy.ts | 5 +- .../src/account/account-query.service.ts | 6 +- .../features/src/account/insight.service.ts | 7 +- .../features/src/providers/plaid/plaid.etl.ts | 69 +++++++++- .../migration.sql | 127 ++++++++++++++++++ prisma/schema.prisma | 32 +++-- 9 files changed, 236 insertions(+), 39 deletions(-) create mode 100644 prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql diff --git a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts index 2dcbcb8a..17c81c4e 100644 --- a/apps/server/src/app/__tests__/balance-sync.integration.spec.ts +++ b/apps/server/src/app/__tests__/balance-sync.integration.spec.ts @@ -1,4 +1,5 @@ import type { User } from '@prisma/client' +import { InvestmentTransactionCategory } from '@prisma/client' import { PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' @@ -131,6 +132,7 @@ describe('balance sync strategies', () => { quantity: 10, price: 10, plaidType: 'buy', + category: InvestmentTransactionCategory.buy, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -140,6 +142,7 @@ describe('balance sync strategies', () => { quantity: 5, price: 10, plaidType: 'sell', + category: InvestmentTransactionCategory.sell, }, { date: DateTime.fromISO('2023-02-04').toJSDate(), @@ -147,6 +150,7 @@ describe('balance sync strategies', () => { amount: 50, quantity: 50, price: 1, + category: InvestmentTransactionCategory.other, }, ], }, diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index 9dbe0433..fe32ffef 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,5 +1,5 @@ import type { User } from '@prisma/client' -import { Prisma, PrismaClient } from '@prisma/client' +import { InvestmentTransactionCategory, Prisma, PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import type { @@ -307,6 +307,7 @@ describe('insight service', () => { price: 100, plaidType: 'buy', plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -318,6 +319,7 @@ describe('insight service', () => { price: 200, plaidType: 'buy', plaidSubtype: 'buy', + category: InvestmentTransactionCategory.buy, }, { accountId: account.id, @@ -329,6 +331,7 @@ describe('insight service', () => { price: 0, plaidType: 'cash', plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, { accountId: account.id, @@ -340,6 +343,7 @@ describe('insight service', () => { price: 0, plaidType: 'cash', plaidSubtype: 'dividend', + category: InvestmentTransactionCategory.dividend, }, ], }) diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 47e26cc3..32ed7915 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -1,5 +1,5 @@ import type { PrismaClient, User } from '@prisma/client' -import { Prisma } from '@prisma/client' +import { InvestmentTransactionCategory, Prisma } from '@prisma/client' import _ from 'lodash' import { DateTime } from 'luxon' import { parseCsv } from './csv' @@ -20,6 +20,14 @@ const portfolios: Record> = }, } +const investmentTransactionCategoryByType: Record = { + BUY: InvestmentTransactionCategory.buy, + SELL: InvestmentTransactionCategory.sell, + DIVIDEND: InvestmentTransactionCategory.dividend, + DEPOSIT: InvestmentTransactionCategory.transfer, + WITHDRAW: InvestmentTransactionCategory.transfer, +} + export async function createTestInvestmentAccount( prisma: PrismaClient, user: User, @@ -35,7 +43,7 @@ export async function createTestInvestmentAccount( join(__dirname, `../test-data/${portfolio}/holdings.csv`) ) - const [_deleted, ...securities] = await prisma.$transaction([ + const [, ...securities] = await prisma.$transaction([ prisma.security.deleteMany({ where: { symbol: { @@ -72,7 +80,7 @@ export async function createTestInvestmentAccount( .value(), ]) - const account = await prisma.account.create({ + return prisma.account.create({ data: { ...portfolios[portfolio], userId: user.id, @@ -128,12 +136,13 @@ export async function createTestInvestmentAccount( : it.type === 'SELL' ? 'sell' : undefined, + category: + investmentTransactionCategoryByType[it.type] ?? + InvestmentTransactionCategory.other, } }), }, }, }, }) - - return account } diff --git a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts index 683173d6..e5d4f4b0 100644 --- a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts +++ b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts @@ -69,9 +69,8 @@ export class InvestmentTransactionBalanceSyncStrategy extends BalanceSyncStrateg WHERE it.account_id = ${pAccountId} AND it.date BETWEEN ${pStart} AND now() - AND ( -- filter for transactions that modify a position - it.plaid_type IN ('buy', 'sell', 'transfer') - ) + -- filter for transactions that modify a position + AND it.category IN ('buy', 'sell', 'transfer') GROUP BY 1, 2 ) it ON it.security_id = s.id AND it.date = d.date diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index b83100ad..04d6650a 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -242,11 +242,7 @@ export class AccountQueryService implements IAccountQueryService { it.account_id = ANY(${pAccountIds}) AND it.date BETWEEN sd.start_date AND ${pEnd} -- filter for investment_transactions that represent external flows - AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - ) + AND it.category = 'transfer' GROUP BY 1, 2 ), external_flow_totals AS ( diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index b26893c9..bfe44d9d 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -312,6 +312,7 @@ export class InsightService implements IInsightService { { plaidSubtype: 'dividend', }, + { category: 'dividend' }, ], }, }), @@ -737,11 +738,7 @@ export class InsightService implements IInsightService { LEFT JOIN account a ON a.id = it.account_id WHERE it.account_id = ${accountId} - AND ( - (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) - OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer', 'send', 'request')) - OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) - ) + AND it.category = 'transfer' -- Exclude any contributions made prior to the start date since balances will be 0 AND (a.start_date is NULL OR it.date >= a.start_date) GROUP BY 1 diff --git a/libs/server/features/src/providers/plaid/plaid.etl.ts b/libs/server/features/src/providers/plaid/plaid.etl.ts index ad16b62e..9974921d 100644 --- a/libs/server/features/src/providers/plaid/plaid.etl.ts +++ b/libs/server/features/src/providers/plaid/plaid.etl.ts @@ -14,7 +14,8 @@ import type { LiabilitiesObject as PlaidLiabilities, PlaidApi, } from 'plaid' -import { Prisma } from '@prisma/client' +import { InvestmentTransactionSubtype, InvestmentTransactionType } from 'plaid' +import { Prisma, InvestmentTransactionCategory } from '@prisma/client' import { DateTime } from 'luxon' import _, { chunk } from 'lodash' import { ErrorUtil, PlaidUtil } from '@maybe-finance/server/shared' @@ -548,7 +549,7 @@ export class PlaidETL implements IETL { ...chunk(investmentTransactions, 1_000).map( (chunk) => this.prisma.$executeRaw` - INSERT INTO investment_transaction (account_id, security_id, plaid_investment_transaction_id, date, name, amount, fees, quantity, price, currency_code, plaid_type, plaid_subtype) + INSERT INTO investment_transaction (account_id, security_id, plaid_investment_transaction_id, date, name, amount, fees, quantity, price, currency_code, plaid_type, plaid_subtype, category) VALUES ${Prisma.join( chunk.map( @@ -584,7 +585,11 @@ export class PlaidETL implements IETL { ${DbUtil.toDecimal(price)}, ${currencyCode}, ${type}, - ${subtype} + ${subtype}, + ${this.getInvestmentTransactionCategoryByPlaidType( + type, + subtype + )} )` } ) @@ -602,6 +607,7 @@ export class PlaidETL implements IETL { currency_code = EXCLUDED.currency_code, plaid_type = EXCLUDED.plaid_type, plaid_subtype = EXCLUDED.plaid_subtype; + category = EXCLUDED.category; ` ), @@ -669,6 +675,63 @@ export class PlaidETL implements IETL { ] } + private getInvestmentTransactionCategoryByPlaidType = ( + type: InvestmentTransactionType, + subType: InvestmentTransactionSubtype + ): InvestmentTransactionCategory => { + if (type === InvestmentTransactionType.Buy) { + return InvestmentTransactionCategory.buy + } + + if (type === InvestmentTransactionType.Sell) { + return InvestmentTransactionCategory.sell + } + + if ( + [ + InvestmentTransactionSubtype.Dividend, + InvestmentTransactionSubtype.QualifiedDividend, + InvestmentTransactionSubtype.NonQualifiedDividend, + ].includes(subType) + ) { + return InvestmentTransactionCategory.dividend + } + + if ( + [ + InvestmentTransactionSubtype.NonResidentTax, + InvestmentTransactionSubtype.Tax, + InvestmentTransactionSubtype.TaxWithheld, + ].includes(subType) + ) { + return InvestmentTransactionCategory.tax + } + + if ( + type === InvestmentTransactionType.Fee || + [ + InvestmentTransactionSubtype.AccountFee, + InvestmentTransactionSubtype.LegalFee, + InvestmentTransactionSubtype.ManagementFee, + InvestmentTransactionSubtype.MarginExpense, + InvestmentTransactionSubtype.TransferFee, + InvestmentTransactionSubtype.TrustFee, + ].includes(subType) + ) { + return InvestmentTransactionCategory.fee + } + + if (type === InvestmentTransactionType.Cash) { + return InvestmentTransactionCategory.transfer + } + + if (type === InvestmentTransactionType.Cancel) { + return InvestmentTransactionCategory.cancel + } + + return InvestmentTransactionCategory.other + } + private async _extractHoldings(accessToken: string) { try { const { data } = await this.plaid.investmentsHoldingsGet({ access_token: accessToken }) diff --git a/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql b/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql new file mode 100644 index 00000000..e888285c --- /dev/null +++ b/prisma/migrations/20240120215821_remove_investment_transaction_category_generation/migration.sql @@ -0,0 +1,127 @@ +-- AlterTable +ALTER TABLE "investment_transaction" + RENAME COLUMN "category" TO "category_old"; + +DROP VIEW IF EXISTS holdings_enriched; + +ALTER TABLE "investment_transaction" + ADD COLUMN "category" "InvestmentTransactionCategory" NOT NULL DEFAULT 'other'::"InvestmentTransactionCategory"; + +CREATE OR REPLACE VIEW holdings_enriched AS ( + SELECT + h.id, + h.account_id, + h.security_id, + h.quantity, + COALESCE(pricing_latest.price_close * h.quantity * COALESCE(s.shares_per_contract, 1), h.value) AS "value", + COALESCE(h.cost_basis, tcb.cost_basis * h.quantity) AS "cost_basis", + COALESCE(h.cost_basis / h.quantity / COALESCE(s.shares_per_contract, 1), tcb.cost_basis) AS "cost_basis_per_share", + pricing_latest.price_close AS "price", + pricing_prev.price_close AS "price_prev", + h.excluded + FROM + holding h + INNER JOIN security s ON s.id = h.security_id + -- latest security pricing + LEFT JOIN LATERAL ( + SELECT + price_close + FROM + security_pricing + WHERE + security_id = h.security_id + ORDER BY + date DESC + LIMIT 1 + ) pricing_latest ON true + -- previous security pricing (for computing daily ∆) + LEFT JOIN LATERAL ( + SELECT + price_close + FROM + security_pricing + WHERE + security_id = h.security_id + ORDER BY + date DESC + LIMIT 1 + OFFSET 1 + ) pricing_prev ON true + -- calculate cost basis from transactions + LEFT JOIN ( + SELECT + it.account_id, + it.security_id, + SUM(it.quantity * it.price) / SUM(it.quantity) AS cost_basis + FROM + investment_transaction it + WHERE + it.category = 'buy' + AND it.quantity > 0 + GROUP BY + it.account_id, + it.security_id + ) tcb ON tcb.account_id = h.account_id AND tcb.security_id = s.id +); + +CREATE OR REPLACE FUNCTION calculate_return_dietz(p_account_id account.id%type, p_start date, p_end date, out percentage numeric, out amount numeric) AS $$ + DECLARE + v_start date := GREATEST(p_start, (SELECT MIN(date) FROM account_balance WHERE account_id = p_account_id)); + v_end date := p_end; + v_days int := v_end - v_start; + BEGIN + SELECT + ROUND((b1.balance - b0.balance - flows.net) / NULLIF(b0.balance + flows.weighted, 0), 4) AS "percentage", + b1.balance - b0.balance - flows.net AS "amount" + INTO + percentage, amount + FROM + account a + LEFT JOIN LATERAL ( + SELECT + COALESCE(SUM(-fw.flow), 0) AS "net", + COALESCE(SUM(-fw.flow * fw.weight), 0) AS "weighted" + FROM ( + SELECT + SUM(it.amount) AS flow, + (v_days - (it.date - v_start))::numeric / v_days AS weight + FROM + investment_transaction it + WHERE + it.account_id = a.id + AND it.date BETWEEN v_start AND v_end + -- filter for investment_transactions that represent external flows + AND it.category = 'transfer' + GROUP BY + it.date + ) fw + ) flows ON TRUE + LEFT JOIN LATERAL ( + SELECT + ab.balance AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_start + ORDER BY + ab.date DESC + LIMIT 1 + ) b0 ON TRUE + LEFT JOIN LATERAL ( + SELECT + COALESCE(ab.balance, a.current_balance) AS "balance" + FROM + account_balance ab + WHERE + ab.account_id = a.id AND ab.date <= v_end + ORDER BY + ab.date DESC + LIMIT 1 + ) b1 ON TRUE + WHERE + a.id = p_account_id; + END; +$$ LANGUAGE plpgsql STABLE; + +ALTER TABLE "investment_transaction" + DROP COLUMN "category_old"; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4242132e..7dcef384 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -217,24 +217,22 @@ enum InvestmentTransactionCategory { } model InvestmentTransaction { - id Int @id @default(autoincrement()) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) - updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) - account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) - accountId Int @map("account_id") - security Security? @relation(fields: [securityId], references: [id], onDelete: Cascade) - securityId Int? @map("security_id") - date DateTime @db.Date + id Int @id @default(autoincrement()) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) + account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) + accountId Int @map("account_id") + security Security? @relation(fields: [securityId], references: [id], onDelete: Cascade) + securityId Int? @map("security_id") + date DateTime @db.Date name String - amount Decimal @db.Decimal(19, 4) - fees Decimal? @db.Decimal(19, 4) - flow TransactionFlow @default(dbgenerated("\nCASE\n WHEN (amount < (0)::numeric) THEN 'INFLOW'::\"TransactionFlow\"\n ELSE 'OUTFLOW'::\"TransactionFlow\"\nEND")) - quantity Decimal @db.Decimal(36, 18) - price Decimal @db.Decimal(23, 8) - currencyCode String @default("USD") @map("currency_code") - - // Derived from provider types - category InvestmentTransactionCategory @default(dbgenerated("\nCASE\n WHEN (plaid_type = 'buy'::text) THEN 'buy'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'sell'::text) THEN 'sell'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['dividend'::text, 'qualified dividend'::text, 'non-qualified dividend'::text])) THEN 'dividend'::\"InvestmentTransactionCategory\"\n WHEN (plaid_subtype = ANY (ARRAY['non-resident tax'::text, 'tax'::text, 'tax withheld'::text])) THEN 'tax'::\"InvestmentTransactionCategory\"\n WHEN ((plaid_type = 'fee'::text) OR (plaid_subtype = ANY (ARRAY['account fee'::text, 'legal fee'::text, 'management fee'::text, 'margin expense'::text, 'transfer fee'::text, 'trust fee'::text]))) THEN 'fee'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cash'::text) THEN 'transfer'::\"InvestmentTransactionCategory\"\n WHEN (plaid_type = 'cancel'::text) THEN 'cancel'::\"InvestmentTransactionCategory\"\n ELSE 'other'::\"InvestmentTransactionCategory\"\nEND")) + amount Decimal @db.Decimal(19, 4) + fees Decimal? @db.Decimal(19, 4) + flow TransactionFlow @default(dbgenerated("\nCASE\n WHEN (amount < (0)::numeric) THEN 'INFLOW'::\"TransactionFlow\"\n ELSE 'OUTFLOW'::\"TransactionFlow\"\nEND")) + quantity Decimal @db.Decimal(36, 18) + price Decimal @db.Decimal(23, 8) + currencyCode String @default("USD") @map("currency_code") + category InvestmentTransactionCategory @default(other) // plaid data plaidInvestmentTransactionId String? @unique @map("plaid_investment_transaction_id") From 3fb1dbb20e1fa32fbbb8e9e39bc79e0095aaa325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20De=20Boey?= Date: Sun, 21 Jan 2024 01:35:19 +0100 Subject: [PATCH 21/22] feat: add `AssetClass` to `Security` --- .../__tests__/insights.integration.spec.ts | 14 ++++++--- .../features/src/account/insight.service.ts | 31 +++---------------- libs/server/features/src/plan/plan.service.ts | 2 +- libs/shared/src/types/plan-types.ts | 8 ++++- .../migration.sql | 6 ++++ prisma/schema.prisma | 23 +++++++++----- 6 files changed, 45 insertions(+), 39 deletions(-) create mode 100644 prisma/migrations/20240121003016_add_asset_class_to_security/migration.sql diff --git a/apps/server/src/app/__tests__/insights.integration.spec.ts b/apps/server/src/app/__tests__/insights.integration.spec.ts index fe32ffef..029aa73f 100644 --- a/apps/server/src/app/__tests__/insights.integration.spec.ts +++ b/apps/server/src/app/__tests__/insights.integration.spec.ts @@ -1,5 +1,5 @@ import type { User } from '@prisma/client' -import { InvestmentTransactionCategory, Prisma, PrismaClient } from '@prisma/client' +import { AssetClass, InvestmentTransactionCategory, Prisma, PrismaClient } from '@prisma/client' import { createLogger, transports } from 'winston' import { DateTime } from 'luxon' import type { @@ -202,19 +202,25 @@ describe('insight service', () => { holdings: { create: [ { - security: { create: { symbol: 'AAPL', plaidType: 'equity' } }, + security: { + create: { symbol: 'AAPL', assetClass: AssetClass.stocks }, + }, quantity: 1, costBasisUser: 100, value: 200, }, { - security: { create: { symbol: 'NFLX', plaidType: 'equity' } }, + security: { + create: { symbol: 'NFLX', assetClass: AssetClass.stocks }, + }, quantity: 10, costBasisUser: 200, value: 300, }, { - security: { create: { symbol: 'SHOP', plaidType: 'equity' } }, + security: { + create: { symbol: 'SHOP', assetClass: AssetClass.stocks }, + }, quantity: 2, costBasisUser: 100, value: 50, diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index bfe44d9d..116e588e 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -641,14 +641,7 @@ export class InsightService implements IInsightService { INNER JOIN ( SELECT id, - CASE - -- plaid - WHEN plaid_type IN ('equity', 'etf', 'mutual fund', 'derivative') THEN 'stocks' - WHEN plaid_type IN ('fixed income') THEN 'fixed_income' - WHEN plaid_type IN ('cash', 'loan') THEN 'cash' - WHEN plaid_type IN ('cryptocurrency') THEN 'crypto' - ELSE 'other' - END AS "asset_class" + asset_class FROM "security" ) s ON s.id = h.security_id @@ -694,14 +687,7 @@ export class InsightService implements IInsightService { INNER JOIN security s ON s.id = h.security_id LEFT JOIN LATERAL ( SELECT - CASE - -- plaid - WHEN s.plaid_type IN ('equity', 'etf', 'mutual fund', 'derivative') THEN 'stocks' - WHEN s.plaid_type IN ('fixed income') THEN 'fixed_income' - WHEN s.plaid_type IN ('cash', 'loan') THEN 'cash' - WHEN s.plaid_type IN ('cryptocurrency') THEN 'crypto' - ELSE 'other' - END AS "category" + asset_class AS "category" ) x ON TRUE WHERE h.account_id IN ${accountIds} @@ -828,28 +814,21 @@ export class InsightService implements IInsightService { UNION ALL -- investment accounts SELECT - s.asset_type, + s.asset_class AS "asset_type", SUM(h.value) AS "amount" FROM holdings_enriched h INNER JOIN ( SELECT id, - CASE - -- plaid - WHEN plaid_type IN ('equity', 'etf', 'mutual fund', 'derivative') THEN 'stocks' - WHEN plaid_type IN ('fixed income') THEN 'bonds' - WHEN plaid_type IN ('cash', 'loan') THEN 'cash' - WHEN plaid_type IN ('cryptocurrency') THEN 'crypto' - ELSE 'other' - END AS "asset_type" + asset_class FROM "security" ) s ON s.id = h.security_id WHERE h.account_id IN ${pAccountIds} GROUP BY - s.asset_type + s.asset_class ) x GROUP BY 1 diff --git a/libs/server/features/src/plan/plan.service.ts b/libs/server/features/src/plan/plan.service.ts index b8fa4aca..9e0ba7a7 100644 --- a/libs/server/features/src/plan/plan.service.ts +++ b/libs/server/features/src/plan/plan.service.ts @@ -33,7 +33,7 @@ const PROJECTION_ASSET_PARAMS: { [type in SharedType.ProjectionAssetType]: [mean: Decimal.Value, stddev: Decimal.Value] } = { stocks: ['0.05', '0.186'], - bonds: ['0.02', '0.052'], + fixed_income: ['0.02', '0.052'], cash: ['-0.02', '0.05'], crypto: ['1.0', '1.0'], property: ['0.1', '0.2'], diff --git a/libs/shared/src/types/plan-types.ts b/libs/shared/src/types/plan-types.ts index 9b0f8d9c..97b46e5a 100644 --- a/libs/shared/src/types/plan-types.ts +++ b/libs/shared/src/types/plan-types.ts @@ -56,7 +56,13 @@ export type PlanProjectionResponse = { }[] } -export type ProjectionAssetType = 'stocks' | 'bonds' | 'cash' | 'crypto' | 'property' | 'other' +export type ProjectionAssetType = + | 'stocks' + | 'fixed_income' + | 'cash' + | 'crypto' + | 'property' + | 'other' export type ProjectionLiabilityType = 'credit' | 'loan' | 'other' export type PlanInsights = { diff --git a/prisma/migrations/20240121003016_add_asset_class_to_security/migration.sql b/prisma/migrations/20240121003016_add_asset_class_to_security/migration.sql new file mode 100644 index 00000000..23534f6f --- /dev/null +++ b/prisma/migrations/20240121003016_add_asset_class_to_security/migration.sql @@ -0,0 +1,6 @@ +-- CreateEnum +CREATE TYPE "AssetClass" AS ENUM ('cash', 'crypto', 'fixed_income', 'stocks', 'other'); + +-- AlterTable +ALTER TABLE "security" + ADD COLUMN "asset_class" "AssetClass" NOT NULL DEFAULT 'other'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index cccdd299..bebee78d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -243,18 +243,27 @@ model InvestmentTransaction { @@map("investment_transaction") } +enum AssetClass { + cash + crypto + fixed_income + stocks + other +} + model Security { - id Int @id @default(autoincrement()) - createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) - updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) + id Int @id @default(autoincrement()) + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6) name String? symbol String? cusip String? isin String? - sharesPerContract Decimal? @map("shares_per_contract") @db.Decimal(36, 18) - currencyCode String @default("USD") @map("currency_code") - pricingLastSyncedAt DateTime? @map("pricing_last_synced_at") @db.Timestamptz(6) - isBrokerageCash Boolean @default(false) @map("is_brokerage_cash") + sharesPerContract Decimal? @map("shares_per_contract") @db.Decimal(36, 18) + currencyCode String @default("USD") @map("currency_code") + pricingLastSyncedAt DateTime? @map("pricing_last_synced_at") @db.Timestamptz(6) + isBrokerageCash Boolean @default(false) @map("is_brokerage_cash") + assetClass AssetClass @default(other) @map("asset_class") // plaid data plaidSecurityId String? @unique @map("plaid_security_id") From b066baef5faa7f6e69405e1dde7723fc09db2984 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Sat, 20 Jan 2024 23:17:02 -0600 Subject: [PATCH 22/22] fun fixes --- .../src/providers/teller/teller.etl.ts | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/libs/server/features/src/providers/teller/teller.etl.ts b/libs/server/features/src/providers/teller/teller.etl.ts index 406f70c3..6d295d26 100644 --- a/libs/server/features/src/providers/teller/teller.etl.ts +++ b/libs/server/features/src/providers/teller/teller.etl.ts @@ -100,10 +100,7 @@ export class TellerETL implements IETL { const accounts = await this._extractAccounts(accessToken) - const transactions = await this._extractTransactions( - accessToken, - accounts.map((a) => a.id) - ) + const transactions = await this._extractTransactions(accessToken, accounts) this.logger.info( `Extracted Teller data for customer ${user.tellerUserId} accounts=${accounts.length} transactions=${transactions.length}`, @@ -196,26 +193,26 @@ export class TellerETL implements IETL { ] } - private async _extractTransactions(accessToken: string, accountIds: string[]) { + private async _extractTransactions( + accessToken: string, + tellerAccounts: TellerTypes.GetAccountsResponse + ) { const accountTransactions = await Promise.all( - accountIds.map(async (accountId) => { - const account = await this.prisma.account.findFirst({ - where: { - tellerAccountId: accountId, - }, - }) + tellerAccounts.map(async (tellerAccount) => { + const type = TellerUtil.getType(tellerAccount.type) + const classification = AccountUtil.getClassification(type) const transactions = await SharedUtil.withRetry( () => this.teller.getTransactions({ - accountId, + accountId: tellerAccount.id, accessToken, }), { maxRetries: 3, } ) - if (account!.classification === AccountClassification.asset) { + if (classification === AccountClassification.asset) { transactions.forEach((t) => { t.amount = String(Number(t.amount) * -1) }) @@ -277,7 +274,7 @@ export class TellerETL implements IETL { pending = EXCLUDED.pending, merchant_name = EXCLUDED.merchant_name, teller_type = EXCLUDED.teller_type, - teller_category = EXCLUDED.teller_category; + teller_category = EXCLUDED.teller_category, category = EXCLUDED.category; ` })