Merge remote-tracking branch 'upstream/main' into remove-plaid-from-tests

libs/server/features/src/providers/plaid/plaid.etl.ts

@@ -13,6 +13,7 @@ import type {
     Item as PlaidItem,
     LiabilitiesObject as PlaidLiabilities,
     PlaidApi,
+    PersonalFinanceCategory,
 } from 'plaid'
 import { InvestmentTransactionSubtype, InvestmentTransactionType } from 'plaid'
 import { Prisma, InvestmentTransactionCategory } from '@prisma/client'
@@ -367,7 +368,7 @@ export class PlaidETL implements IETL<Connection, PlaidRawData, PlaidData> {
 
         const txnUpsertQueries = chunk(transactions, 1_000).map((chunk) => {
             return this.prisma.$executeRaw`
-                INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category)
+                INSERT INTO transaction (account_id, plaid_transaction_id, date, name, amount, pending, currency_code, merchant_name, plaid_category, plaid_category_id, plaid_personal_finance_category, category)
                 VALUES
                     ${Prisma.join(
                         chunk.map((plaidTransaction) => {
@@ -402,7 +403,8 @@ export class PlaidETL implements IETL<Connection, PlaidRawData, PlaidData> {
                                 ${merchant_name},
                                 ${category ?? []},
                                 ${category_id},
-                                ${personal_finance_category}
+                                ${personal_finance_category},
+                                ${this.getMaybeTransactionCategory(personal_finance_category)}
                             )`
                         })
                     )}
@@ -415,6 +417,7 @@ export class PlaidETL implements IETL<Connection, PlaidRawData, PlaidData> {
                     plaid_category = EXCLUDED.plaid_category,
                     plaid_category_id = EXCLUDED.plaid_category_id,
                     plaid_personal_finance_category = EXCLUDED.plaid_personal_finance_category;
+                    category = EXCLUDED.category;
             `
         })
 
@@ -445,6 +448,68 @@ export class PlaidETL implements IETL<Connection, PlaidRawData, PlaidData> {
         ]
     }
 
+    private getMaybeTransactionCategory = (category?: PersonalFinanceCategory | null) => {
+        if (!category) {
+            return 'Other'
+        }
+
+        if (category.primary === 'INCOME') {
+            return 'Income'
+        }
+
+        if (
+            ['LOAN_PAYMENTS_MORTGAGE_PAYMENT', 'RENT_AND_UTILITIES_RENT'].includes(
+                category.detailed
+            )
+        ) {
+            return 'Housing Payments'
+        }
+
+        if (category.detailed === 'LOAN_PAYMENTS_CAR_PAYMENT') {
+            return 'Vehicle Payments'
+        }
+
+        if (category.primary === 'LOAN_PAYMENTS') {
+            return 'Other Payments'
+        }
+
+        if (category.primary === 'HOME_IMPROVEMENT') {
+            return 'Home Improvement'
+        }
+
+        if (category.primary === 'GENERAL_MERCHANDISE') {
+            return 'Shopping'
+        }
+
+        if (
+            category.primary === 'RENT_AND_UTILITIES' &&
+            category.detailed !== 'RENT_AND_UTILITIES_RENT'
+        ) {
+            return 'Utilities'
+        }
+
+        if (category.primary === 'FOOD_AND_DRINK') {
+            return 'Food and Drink'
+        }
+
+        if (category.primary === 'TRANSPORTATION') {
+            return 'Transportation'
+        }
+
+        if (category.primary === 'TRAVEL') {
+            return 'Travel'
+        }
+
+        if (
+            ['PERSONAL_CARE', 'MEDICAL'].includes(category.primary) &&
+            category.detailed !== 'MEDICAL_VETERINARY_SERVICES'
+        ) {
+            return 'Health'
+        }
+
+        return 'Other'
+    }
+
     private _extractInvestmentTransactions(accessToken: string, dateRange: SharedType.DateRange) {
         return SharedUtil.paginate({
             pageSize: 500, // https://plaid.com/docs/api/products/investments/#investments-transactions-get-request-options-count

libs/server/features/src/providers/teller/teller.etl.ts

@@ -25,6 +25,40 @@ type Connection = Pick<
     'id' | 'userId' | 'tellerInstitutionId' | 'tellerAccessToken'
 >
 
+const maybeCategoryByTellerCategory: Record<
+    Required<TellerTypes.Transaction['details']>['category'],
+    string
+> = {
+    accommodation: 'Travel',
+    advertising: 'Other',
+    bar: 'Food and Drink',
+    charity: 'Other',
+    clothing: 'Shopping',
+    dining: 'Food and Drink',
+    education: 'Other',
+    electronics: 'Shopping',
+    entertainment: 'Shopping',
+    fuel: 'Transportation',
+    general: 'Other',
+    groceries: 'Food and Drink',
+    health: 'Health',
+    home: 'Home Improvement',
+    income: 'Income',
+    insurance: 'Other',
+    investment: 'Other',
+    loan: 'Other',
+    office: 'Other',
+    phone: 'Utilities',
+    service: 'Other',
+    shopping: 'Shopping',
+    software: 'Shopping',
+    sport: 'Shopping',
+    tax: 'Other',
+    transport: 'Transportation',
+    transportation: 'Transportation',
+    utilities: 'Utilities',
+}
+
 export class TellerETL implements IETL<Connection, TellerRawData, TellerData> {
     public constructor(
         private readonly logger: Logger,
@@ -204,7 +238,7 @@ export class TellerETL implements IETL<Connection, TellerRawData, TellerData> {
 
         const txnUpsertQueries = _.chunk(transactions, 1_000).map((chunk) => {
             return this.prisma.$executeRaw`
-                INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category)
+                INSERT INTO transaction (account_id, teller_transaction_id, date, name, amount, pending, currency_code, merchant_name, teller_type, teller_category, category)
                 VALUES
                     ${Prisma.join(
                         chunk.map((tellerTransaction) => {
@@ -231,7 +265,8 @@ export class TellerETL implements IETL<Connection, TellerRawData, TellerData> {
                                 ${'USD'},
                                 ${details.counterparty?.name ?? ''},
                                 ${type},
-                                ${details.category ?? ''}
+                                ${details.category ?? ''},
+                                ${maybeCategoryByTellerCategory[details.category ?? ''] ?? 'Other'}
                             )`
                         })
                     )}
@@ -243,6 +278,7 @@ export class TellerETL implements IETL<Connection, TellerRawData, TellerData> {
                     merchant_name = EXCLUDED.merchant_name,
                     teller_type = EXCLUDED.teller_type,
                     teller_category = EXCLUDED.teller_category;
+                    category = EXCLUDED.category;
             `
         })
 

prisma/migrations/20240120213022_remove_transaction_category_generation/migration.sql

@@ -0,0 +1,63 @@
+-- AlterTable
+ALTER TABLE "transaction"
+  RENAME COLUMN "category" TO "category_old";
+
+DROP VIEW IF EXISTS transactions_enriched;
+
+ALTER TABLE "transaction"
+  ADD COLUMN "category" TEXT NOT NULL DEFAULT 'Other'::text;
+
+CREATE OR REPLACE VIEW transactions_enriched AS (
+  SELECT
+    t.id,
+    t.created_at as "createdAt",
+    t.updated_at as "updatedAt",
+    t.name,
+    t.account_id as "accountId",
+    t.date,
+    t.flow,
+    COALESCE(
+      t.type_user,
+      CASE
+        -- no matching transaction
+        WHEN t.match_id IS NULL THEN (
+          CASE
+            t.flow
+            WHEN 'INFLOW' THEN (
+              CASE
+                a.classification
+                WHEN 'asset' THEN 'INCOME' :: "TransactionType"
+                WHEN 'liability' THEN 'PAYMENT' :: "TransactionType"
+              END
+            )
+            WHEN 'OUTFLOW' THEN 'EXPENSE' :: "TransactionType"
+          END
+        ) -- has matching transaction
+        ELSE (
+          CASE
+            a.classification
+            WHEN 'asset' THEN 'TRANSFER' :: "TransactionType"
+            WHEN 'liability' THEN 'PAYMENT' :: "TransactionType"
+          END
+        )
+      END
+    ) AS "type",
+    t.type_user as "typeUser",
+    t.amount,
+    t.currency_code as "currencyCode",
+    t.pending,
+    t.merchant_name as "merchantName",
+    t.category,
+    t.category_user as "categoryUser",
+    t.excluded,
+    t.match_id as "matchId",
+    COALESCE(ac.user_id, a.user_id) as "userId",
+    a.classification as "accountClassification",
+    a.type as "accountType"
+  FROM
+    transaction t
+    inner join account a on a.id = t.account_id
+    left join account_connection ac on a.account_connection_id = ac.id
+);
+
+ALTER TABLE "transaction" DROP COLUMN "category_old";

prisma/schema.prisma

@@ -310,7 +310,7 @@ model Transaction {
   currencyCode String           @default("USD") @map("currency_code")
   pending      Boolean          @default(false)
   merchantName String?          @map("merchant_name")
-  category     String           @default(dbgenerated("COALESCE(category_user,\nCASE\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'INCOME'::text) THEN 'Income'::text\n    WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = ANY (ARRAY['LOAN_PAYMENTS_MORTGAGE_PAYMENT'::text, 'RENT_AND_UTILITIES_RENT'::text])) THEN 'Housing Payments'::text\n    WHEN ((plaid_personal_finance_category ->> 'detailed'::text) = 'LOAN_PAYMENTS_CAR_PAYMENT'::text) THEN 'Vehicle Payments'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'LOAN_PAYMENTS'::text) THEN 'Other Payments'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'HOME_IMPROVEMENT'::text) THEN 'Home Improvement'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'GENERAL_MERCHANDISE'::text) THEN 'Shopping'::text\n    WHEN (((plaid_personal_finance_category ->> 'primary'::text) = 'RENT_AND_UTILITIES'::text) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'RENT_AND_UTILITIES_RENT'::text)) THEN 'Utilities'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'FOOD_AND_DRINK'::text) THEN 'Food and Drink'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRANSPORTATION'::text) THEN 'Transportation'::text\n    WHEN ((plaid_personal_finance_category ->> 'primary'::text) = 'TRAVEL'::text) THEN 'Travel'::text\n    WHEN (((plaid_personal_finance_category ->> 'primary'::text) = ANY (ARRAY['PERSONAL_CARE'::text, 'MEDICAL'::text])) AND ((plaid_personal_finance_category ->> 'detailed'::text) <> 'MEDICAL_VETERINARY_SERVICES'::text)) THEN 'Health'::text\n    WHEN (teller_category = 'income'::text) THEN 'Income'::text\n    WHEN (teller_category = 'home'::text) THEN 'Home Improvement'::text\n    WHEN (teller_category = ANY (ARRAY['phone'::text, 'utilities'::text])) THEN 'Utilities'::text\n    WHEN (teller_category = ANY (ARRAY['dining'::text, 'bar'::text, 'groceries'::text])) THEN 'Food and Drink'::text\n    WHEN (teller_category = ANY (ARRAY['clothing'::text, 'entertainment'::text, 'shopping'::text, 'electronics'::text, 'software'::text, 'sport'::text])) THEN 'Shopping'::text\n    WHEN (teller_category = ANY (ARRAY['transportation'::text, 'fuel'::text])) THEN 'Transportation'::text\n    WHEN (teller_category = ANY (ARRAY['accommodation'::text, 'transport'::text])) THEN 'Travel'::text\n    WHEN (teller_category = 'health'::text) THEN 'Health'::text\n    WHEN (teller_category = ANY (ARRAY['loan'::text, 'tax'::text, 'insurance'::text, 'office'::text])) THEN 'Other Payments'::text\n    ELSE 'Other'::text\nEND)"))
+  category     String           @default("Other")
   categoryUser String?          @map("category_user")
   excluded     Boolean          @default(false)