From 187b84e1c5c59786cbc1afbe5e91ffadeb4b33e0 Mon Sep 17 00:00:00 2001 From: Tyler Myracle Date: Fri, 19 Jan 2024 19:34:34 -0600 Subject: [PATCH] rollback more stuff --- .../server/src/app/__tests__/utils/account.ts | 37 +- ...tment-transaction-balance-sync.strategy.ts | 3 +- .../src/account/account-query.service.ts | 485 +++++++++--------- .../features/src/account/insight.service.ts | 8 +- 4 files changed, 271 insertions(+), 262 deletions(-) diff --git a/apps/server/src/app/__tests__/utils/account.ts b/apps/server/src/app/__tests__/utils/account.ts index 088b2da8..6ba2fd23 100644 --- a/apps/server/src/app/__tests__/utils/account.ts +++ b/apps/server/src/app/__tests__/utils/account.ts @@ -99,21 +99,7 @@ export async function createTestInvestmentAccount( (s) => s.date === it.date && s.ticker === it.ticker )?.price - function getTransactionCategory(type: string) { - switch (type) { - case 'BUY': - return 'buy' - case 'SELL': - return 'sell' - case 'DIVIDEND': - return 'dividend' - case 'DEPOSIT': - case 'WITHDRAW': - return 'transfer' - default: - return undefined - } - } + const isCashFlow = it.type === 'DEPOSIT' || it.type === 'WITHDRAW' return { securityId: securities.find((s) => it.ticker === s.symbol)?.id, @@ -122,7 +108,26 @@ export async function createTestInvestmentAccount( amount: price ? new Prisma.Decimal(price).times(it.qty) : it.qty, quantity: price ? it.qty : 0, price: price ?? 0, - category: getTransactionCategory(it.type), + plaidType: + isCashFlow || it.type === 'DIVIDEND' + ? 'cash' + : it.type === 'BUY' + ? 'buy' + : it.type === 'SELL' + ? 'sell' + : undefined, + plaidSubtype: + it.type === 'DEPOSIT' + ? 'deposit' + : it.type === 'WITHDRAW' + ? 'withdrawal' + : it.type === 'DIVIDEND' + ? 'dividend' + : it.type === 'BUY' + ? 'buy' + : it.type === 'SELL' + ? 'sell' + : undefined, } }), }, diff --git a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts index 83cc4954..21f0d914 100644 --- a/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts +++ b/libs/server/features/src/account-balance/investment-transaction-balance-sync.strategy.ts @@ -70,7 +70,8 @@ export class InvestmentTransactionBalanceSyncStrategy extends BalanceSyncStrateg it.account_id = ${pAccountId} AND it.date BETWEEN ${pStart} AND now() AND ( -- filter for transactions that modify a position - it.category IN ('buy', 'sell', 'transfer') + it.plaid_type IN ('buy', 'sell', 'transfer') + OR it.finicity_transaction_id IS NOT NULL ) GROUP BY 1, 2 diff --git a/libs/server/features/src/account/account-query.service.ts b/libs/server/features/src/account/account-query.service.ts index 05e7a511..b13afa78 100644 --- a/libs/server/features/src/account/account-query.service.ts +++ b/libs/server/features/src/account/account-query.service.ts @@ -117,33 +117,33 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - h.id, - h.security_id, - s.name, - s.symbol, - s.shares_per_contract, - he.quantity, - he.value, - he.cost_basis, - h.cost_basis_user, - h.cost_basis_provider, - he.cost_basis_per_share, - he.price, - he.price_prev, - he.excluded - FROM - holdings_enriched he - INNER JOIN security s ON s.id = he.security_id - INNER JOIN holding h ON h.id = he.id - WHERE - he.account_id = ${accountId} - ORDER BY - he.excluded ASC, - he.value DESC - OFFSET ${page * pageSize} - LIMIT ${pageSize}; - ` + SELECT + h.id, + h.security_id, + s.name, + s.symbol, + s.shares_per_contract, + he.quantity, + he.value, + he.cost_basis, + h.cost_basis_user, + h.cost_basis_provider, + he.cost_basis_per_share, + he.price, + he.price_prev, + he.excluded + FROM + holdings_enriched he + INNER JOIN security s ON s.id = he.security_id + INNER JOIN holding h ON h.id = he.id + WHERE + he.account_id = ${accountId} + ORDER BY + he.excluded ASC, + he.value DESC + OFFSET ${page * pageSize} + LIMIT ${pageSize}; + ` ) return rows @@ -158,53 +158,53 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH valuation_trends AS ( - SELECT - date, - COALESCE(interpolated::numeric, filled) AS amount - FROM ( - SELECT - time_bucket_gapfill('1d', v.date) AS date, - interpolate(avg(v.amount)) AS interpolated, - locf(avg(v.amount)) AS filled - FROM - valuation v - WHERE - v.account_id = ${accountId} - AND v.date BETWEEN ${pStart} AND ${pEnd} - GROUP BY - 1 - ) valuations_gapfilled - WHERE - to_char(date, 'MM-DD') = '01-01' - ), valuations_combined AS ( - SELECT - COALESCE(v.date, vt.date) AS date, - COALESCE(v.amount, vt.amount) AS amount, - v.id AS valuation_id - FROM - (SELECT * FROM valuation WHERE account_id = ${accountId}) v - FULL OUTER JOIN valuation_trends vt ON vt.date = v.date - ) + WITH valuation_trends AS ( SELECT - v.date, - v.amount, - v.valuation_id, - v.amount - v.prev_amount AS period_change, - ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, - v.amount - v.first_amount AS total_change, - ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct + date, + COALESCE(interpolated::numeric, filled) AS amount FROM ( SELECT - *, - LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, - (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount + time_bucket_gapfill('1d', v.date) AS date, + interpolate(avg(v.amount)) AS interpolated, + locf(avg(v.amount)) AS filled FROM - valuations_combined - ) v - ORDER BY - v.date ASC - ` + valuation v + WHERE + v.account_id = ${accountId} + AND v.date BETWEEN ${pStart} AND ${pEnd} + GROUP BY + 1 + ) valuations_gapfilled + WHERE + to_char(date, 'MM-DD') = '01-01' + ), valuations_combined AS ( + SELECT + COALESCE(v.date, vt.date) AS date, + COALESCE(v.amount, vt.amount) AS amount, + v.id AS valuation_id + FROM + (SELECT * FROM valuation WHERE account_id = ${accountId}) v + FULL OUTER JOIN valuation_trends vt ON vt.date = v.date + ) + SELECT + v.date, + v.amount, + v.valuation_id, + v.amount - v.prev_amount AS period_change, + ROUND((v.amount - v.prev_amount)::numeric / NULLIF(v.prev_amount, 0), 4) AS period_change_pct, + v.amount - v.first_amount AS total_change, + ROUND((v.amount - v.first_amount)::numeric / NULLIF(v.first_amount, 0), 4) AS total_change_pct + FROM ( + SELECT + *, + LAG(amount, 1) OVER (ORDER BY date ASC) AS prev_amount, + (SELECT amount FROM valuations_combined ORDER BY date ASC LIMIT 1) AS first_amount + FROM + valuations_combined + ) v + ORDER BY + v.date ASC + ` ) return rows @@ -220,77 +220,80 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH start_date AS ( - SELECT - a.id AS "account_id", - GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" - FROM - account a - WHERE - a.id = ANY(${pAccountIds}) - GROUP BY - 1 - ), external_flows AS ( - SELECT - it.account_id, - it.date, - SUM(it.amount) AS "amount" - FROM - investment_transaction it - LEFT JOIN start_date sd ON sd.account_id = it.account_id - WHERE - it.account_id = ANY(${pAccountIds}) - AND it.date BETWEEN sd.start_date AND ${pEnd} - -- filter for investment_transactions that represent external flows - AND ( - it.category = 'transfer' - ) - GROUP BY - 1, 2 - ), external_flow_totals AS ( - SELECT - account_id, - SUM(amount) as "amount" - FROM - external_flows - GROUP BY - 1 - ), balances AS ( - SELECT - abg.account_id, - abg.date, - abg.balance, - 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", - COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - '1d', - ${pAccountIds} - ) abg - LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date - LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id - ) + WITH start_date AS ( SELECT - b.account_id, - b.date, - b.balance, - b.contributions, - b.contributions_period, - COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" + a.id AS "account_id", + GREATEST(account_value_start_date(a.id), a.start_date) AS "start_date" FROM - balances b - LEFT JOIN ( - SELECT DISTINCT ON (account_id) - account_id, - balance - FROM - balances - ORDER BY - account_id, date ASC - ) b0 ON b0.account_id = b.account_id - ` + account a + WHERE + a.id = ANY(${pAccountIds}) + GROUP BY + 1 + ), external_flows AS ( + SELECT + it.account_id, + it.date, + SUM(it.amount) AS "amount" + FROM + investment_transaction it + LEFT JOIN start_date sd ON sd.account_id = it.account_id + WHERE + it.account_id = ANY(${pAccountIds}) + AND it.date BETWEEN sd.start_date AND ${pEnd} + -- filter for investment_transactions that represent external flows + AND ( + (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) + OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer')) + OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) + OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) + ) + GROUP BY + 1, 2 + ), external_flow_totals AS ( + SELECT + account_id, + SUM(amount) as "amount" + FROM + external_flows + GROUP BY + 1 + ), balances AS ( + SELECT + abg.account_id, + abg.date, + abg.balance, + 0 - SUM(COALESCE(ef.amount, 0)) OVER (PARTITION BY abg.account_id ORDER BY abg.date ASC) AS "contributions_period", + COALESCE(-1 * (eft.amount - coalesce(SUM(ef.amount) OVER (PARTITION BY abg.account_id ORDER BY abg.date DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING), 0)), 0) AS "contributions" + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + '1d', + ${pAccountIds} + ) abg + LEFT JOIN external_flows ef ON ef.account_id = abg.account_id AND ef.date = abg.date + LEFT JOIN external_flow_totals eft ON eft.account_id = abg.account_id + ) + SELECT + b.account_id, + b.date, + b.balance, + b.contributions, + b.contributions_period, + COALESCE(ROUND((b.balance - b0.balance - b.contributions_period) / COALESCE(NULLIF(b0.balance, 0), NULLIF(b.contributions_period, 0)), 4), 0) AS "rate_of_return" + FROM + balances b + LEFT JOIN ( + SELECT DISTINCT ON (account_id) + account_id, + balance + FROM + balances + ORDER BY + account_id, date ASC + ) b0 ON b0.account_id = b.account_id + ` ) return rows @@ -310,18 +313,18 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - SELECT - abg.account_id, - abg.date, - abg.balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - ` + SELECT + abg.account_id, + abg.date, + abg.balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + ` ) return rows @@ -341,15 +344,15 @@ export class AccountQueryService implements IAccountQueryService { 'accountIds' in id ? id.accountIds : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active - )` + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -376,27 +379,27 @@ export class AccountQueryService implements IAccountQueryService { } >( sql` - SELECT - abg.date, - a.category, - a.classification, - SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category), - (abg.date, a.classification), - (abg.date) - ) - ORDER BY date ASC; - ` + SELECT + abg.date, + a.category, + a.classification, + SUM(CASE WHEN a.classification = 'asset' THEN abg.balance ELSE -abg.balance END) AS balance + FROM + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category), + (abg.date, a.classification), + (abg.date) + ) + ORDER BY date ASC; + ` ) // Group independent rows into NetWorthSeries objects @@ -450,15 +453,15 @@ export class AccountQueryService implements IAccountQueryService { 'accountId' in id ? [id.accountId] : sql`( - SELECT - array_agg(a.id) - FROM - account a - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - WHERE - (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) - AND a.is_active - )` + SELECT + array_agg(a.id) + FROM + account a + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + WHERE + (a.user_id = ${id.userId} OR ac.user_id = ${id.userId}) + AND a.is_active + )` const pStart = raw(`'${start}'`) const pEnd = raw(`'${end}'`) @@ -466,59 +469,59 @@ export class AccountQueryService implements IAccountQueryService { const { rows } = await this.pg.pool.query( sql` - WITH account_rollup AS ( - SELECT - abg.date, - a.classification, - a.category, - a.id, - SUM(abg.balance) AS balance, - CASE GROUPING(abg.date, a.classification, a.category, a.id) - WHEN 3 THEN 'classification' - WHEN 1 THEN 'category' - WHEN 0 THEN 'account' - ELSE NULL - END AS grouping - FROM - account_balances_gapfilled( - ${pStart}, - ${pEnd}, - ${pInterval}, - ${pAccountIds} - ) abg - INNER JOIN account a ON a.id = abg.account_id - GROUP BY - GROUPING SETS ( - (abg.date, a.classification, a.category, a.id), - (abg.date, a.classification, a.category), - (abg.date, a.classification) - ) - ) + WITH account_rollup AS ( SELECT - ar.date, - ar.classification, - ar.category, - ar.id, - ar.balance, - ar.grouping, - CASE - WHEN a.id IS NULL THEN NULL - ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) - END AS account, - ROUND( - CASE ar.grouping - WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) - WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) - WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) - END, 4) AS rollup_pct, - ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct + abg.date, + a.classification, + a.category, + a.id, + SUM(abg.balance) AS balance, + CASE GROUPING(abg.date, a.classification, a.category, a.id) + WHEN 3 THEN 'classification' + WHEN 1 THEN 'category' + WHEN 0 THEN 'account' + ELSE NULL + END AS grouping FROM - account_rollup ar - LEFT JOIN account a ON a.id = ar.id - LEFT JOIN account_connection ac ON ac.id = a.account_connection_id - ORDER BY - ar.classification, ar.category, ar.id, ar.date; - ` + account_balances_gapfilled( + ${pStart}, + ${pEnd}, + ${pInterval}, + ${pAccountIds} + ) abg + INNER JOIN account a ON a.id = abg.account_id + GROUP BY + GROUPING SETS ( + (abg.date, a.classification, a.category, a.id), + (abg.date, a.classification, a.category), + (abg.date, a.classification) + ) + ) + SELECT + ar.date, + ar.classification, + ar.category, + ar.id, + ar.balance, + ar.grouping, + CASE + WHEN a.id IS NULL THEN NULL + ELSE json_build_object('id', a.id, 'name', a.name, 'mask', a.mask, 'syncStatus', a.sync_status, 'connection', CASE WHEN ac.id IS NULL THEN NULL ELSE json_build_object('name', ac.name, 'syncStatus', ac.sync_status) END) + END AS account, + ROUND( + CASE ar.grouping + WHEN 'account' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification, ar.category), 0) + WHEN 'category' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date, ar.classification), 0) + WHEN 'classification' THEN COALESCE(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 0) + END, 4) AS rollup_pct, + ROUND(ar.balance / SUM(NULLIF(ar.balance, 0)) OVER (PARTITION BY ar.grouping, ar.date), 4) AS total_pct + FROM + account_rollup ar + LEFT JOIN account a ON a.id = ar.id + LEFT JOIN account_connection ac ON ac.id = a.account_connection_id + ORDER BY + ar.classification, ar.category, ar.id, ar.date; + ` ) return rows diff --git a/libs/server/features/src/account/insight.service.ts b/libs/server/features/src/account/insight.service.ts index 0eb8b13a..4c736ab7 100644 --- a/libs/server/features/src/account/insight.service.ts +++ b/libs/server/features/src/account/insight.service.ts @@ -315,9 +315,6 @@ export class InsightService implements IInsightService { { finicityInvestmentTransactionType: 'dividend', }, - { - category: 'dividend', - }, ], }, }), @@ -750,7 +747,10 @@ export class InsightService implements IInsightService { WHERE it.account_id = ${accountId} AND ( - it.category = 'transfer' + (it.plaid_type = 'cash' AND it.plaid_subtype IN ('contribution', 'deposit', 'withdrawal')) + OR (it.plaid_type = 'transfer' AND it.plaid_subtype IN ('transfer', 'send', 'request')) + OR (it.plaid_type = 'buy' AND it.plaid_subtype IN ('contribution')) + OR (it.finicity_transaction_id IS NOT NULL AND it.finicity_investment_transaction_type IN ('contribution', 'deposit', 'transfer')) ) -- Exclude any contributions made prior to the start date since balances will be 0 AND (a.start_date is NULL OR it.date >= a.start_date)