1
0
Fork 0
mirror of https://github.com/maybe-finance/maybe.git synced 2025-08-10 07:55:21 +02:00

Optimize search totals query

This commit is contained in:
Zach Gollwitzer 2025-06-19 15:14:04 -04:00
parent b399cee0c3
commit f8bb58151b
10 changed files with 276 additions and 308 deletions

View file

@ -12,32 +12,25 @@ class TransactionsController < ApplicationController
def index
@q = search_params
search = Transaction::Search.new(Current.family, filters: @q)
@totals = Transaction::Totals.compute(search)
transactions_query = search.relation
@totals = search.totals
set_focused_record(transactions_query, params[:focused_record_id], default_per_page: 50)
per_page = params[:per_page].to_i.positive? ? params[:per_page].to_i : 50
items_per_page = params[:per_page].to_i.positive? ? params[:per_page].to_i : 50
current_page = params[:page].to_i.positive? ? params[:page].to_i : 1
base_scope = search.transactions_scope
.reverse_chronological
.includes(
{ entry: :account },
:category, :merchant, :tags,
transfer_as_outflow: { inflow_transaction: { entry: :account } },
transfer_as_inflow: { outflow_transaction: { entry: :account } }
)
@pagy = Pagy.new(
count: transactions_query.count,
page: current_page,
limit: items_per_page,
params: ->(p) { p.except(:focused_record_id) }
)
@pagy, @transactions = pagy(base_scope, limit: per_page, params: ->(p) { p.except(:focused_record_id) })
# Use Pagy's calculated page (which handles overflow) with our variables
@transactions = transactions_query
.reverse_chronological
.limit(items_per_page)
.offset((@pagy.page - 1) * items_per_page)
.includes(
{ entry: :account },
:category, :merchant, :tags,
transfer_as_outflow: { inflow_transaction: { entry: :account } },
transfer_as_inflow: { outflow_transaction: { entry: :account } }
)
# No performance penalty by default. Only runs queries if the record is set.
if params[:focused_record_id].present?
set_focused_record(base_scope, params[:focused_record_id], default_per_page: per_page)
end
end
def clear_filter

View file

@ -91,6 +91,7 @@ class Family < ApplicationRecord
entries.order(:date).first&.date || Date.current
end
# Used for invalidating family / balance sheet related aggregation queries
def build_cache_key(key, invalidate_on_data_updates: false)
# Our data sync process updates this timestamp whenever any family account successfully completes a data update.
# By including it in the cache key, we can expire caches every time family account data changes.
@ -103,6 +104,14 @@ class Family < ApplicationRecord
].compact.join("_")
end
# Used for invalidating entry related aggregation queries
def entries_cache_version
@entries_cache_version ||= begin
ts = entries.maximum(:updated_at)
ts.present? ? ts.to_i : 0
end
end
def self_hoster?
Rails.application.config.app_mode.self_hosted?
end

View file

@ -101,14 +101,14 @@ class IncomeStatement
def family_stats(interval: "month")
@family_stats ||= {}
@family_stats[interval] ||= Rails.cache.fetch([
"income_statement", "family_stats", family.id, interval, entries_cache_version
"income_statement", "family_stats", family.id, interval, family.entries_cache_version
]) { FamilyStats.new(family, interval:).call }
end
def category_stats(interval: "month")
@category_stats ||= {}
@category_stats[interval] ||= Rails.cache.fetch([
"income_statement", "category_stats", family.id, interval, entries_cache_version
"income_statement", "category_stats", family.id, interval, family.entries_cache_version
]) { CategoryStats.new(family, interval:).call }
end
@ -116,24 +116,11 @@ class IncomeStatement
sql_hash = Digest::MD5.hexdigest(transactions_scope.to_sql)
Rails.cache.fetch([
"income_statement", "totals_query", family.id, sql_hash, entries_cache_version
"income_statement", "totals_query", family.id, sql_hash, family.entries_cache_version
]) { Totals.new(family, transactions_scope: transactions_scope).call }
end
def monetizable_currency
family.currency
end
# Returns a monotonically increasing integer based on the most recent
# update to any Entry that belongs to the family. Incorporated into cache
# keys so they expire automatically on data changes.
def entries_cache_version
@entries_cache_version ||= begin
ts = Entry.joins(:account)
.where(accounts: { family_id: family.id })
.maximum(:updated_at)
ts.present? ? ts.to_i : 0
end
end
end

View file

@ -23,28 +23,62 @@ class Transaction::Search
super(filters)
end
# Build the complete filtered relation
def relation
query = base_relation.joins(entry: :account)
def transactions_scope
@transactions_scope ||= begin
# This already joins entries + accounts. To avoid expensive double-joins, don't join them again (causes full table scan)
query = family.transactions
query = apply_active_accounts_filter(query, active_accounts_only)
query = apply_excluded_transactions_filter(query, excluded_transactions)
query = apply_category_filter(query, categories)
query = apply_type_filter(query, types)
query = apply_merchant_filter(query, merchants)
query = apply_tag_filter(query, tags)
query = EntrySearch.apply_search_filter(query, search)
query = EntrySearch.apply_date_filters(query, start_date, end_date)
query = EntrySearch.apply_amount_filter(query, amount, amount_operator)
query = EntrySearch.apply_accounts_filter(query, accounts, account_ids)
query = apply_active_accounts_filter(query, active_accounts_only)
query = apply_excluded_transactions_filter(query, excluded_transactions)
query = apply_category_filter(query, categories)
query = apply_type_filter(query, types)
query = apply_merchant_filter(query, merchants)
query = apply_tag_filter(query, tags)
query = EntrySearch.apply_search_filter(query, search)
query = EntrySearch.apply_date_filters(query, start_date, end_date)
query = EntrySearch.apply_amount_filter(query, amount, amount_operator)
query = EntrySearch.apply_accounts_filter(query, accounts, account_ids)
query
query
end
end
# Computes totals for the specific search
def totals
@totals ||= begin
Rails.cache.fetch("transaction_search_totals/#{cache_key_base}") do
result = transactions_scope
.select(
"COALESCE(SUM(CASE WHEN entries.amount >= 0 THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total",
"COALESCE(SUM(CASE WHEN entries.amount < 0 THEN ABS(entries.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total",
"COUNT(entries.id) as transactions_count"
)
.joins(
ActiveRecord::Base.sanitize_sql_array([
"LEFT JOIN exchange_rates er ON (er.date = entries.date AND er.from_currency = entries.currency AND er.to_currency = ?)",
family.currency
])
)
.take
Totals.new(
count: result.transactions_count.to_i,
income_money: Money.new(result.income_total.to_i, family.currency),
expense_money: Money.new(result.expense_total.to_i, family.currency)
)
end
end
end
private
# Build the base relation from family context
def base_relation
family.transactions
Totals = Data.define(:count, :income_money, :expense_money)
def cache_key_base
[
family.id,
Digest::SHA256.hexdigest(attributes.sort.to_h.to_json), # cached by filters
family.entries_cache_version
].join("/")
end
def apply_active_accounts_filter(query, active_accounts_only_filter)
@ -66,7 +100,6 @@ class Transaction::Search
def apply_category_filter(query, categories)
return query unless categories.present?
non_budget_kinds = %w[transfer payment one_time]
query = query.left_joins(:category).where(
"categories.name IN (?) OR (
categories.id IS NULL AND (transactions.kind NOT IN ('transfer', 'payment'))

View file

@ -1,50 +0,0 @@
class Transaction::Totals
# Service for computing transaction totals with multi-currency support
def self.compute(search)
new(search).call
end
def initialize(search)
@search = search
end
def call
ScopeTotals.new(
transactions_count: query_result["transactions_count"].to_i,
income_money: Money.new(query_result["income_total"].to_i, query_result["currency"]),
expense_money: Money.new(query_result["expense_total"].to_i, query_result["currency"])
)
end
private
ScopeTotals = Data.define(:transactions_count, :income_money, :expense_money)
def query_result
ActiveRecord::Base.connection.select_all(sanitized_query).first
end
def sanitized_query
ActiveRecord::Base.sanitize_sql_array([ query_sql, { target_currency: @search.family.currency } ])
end
def query_sql
<<~SQL
SELECT
COALESCE(SUM(CASE WHEN ae.amount >= 0 THEN ABS(ae.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as expense_total,
COALESCE(SUM(CASE WHEN ae.amount < 0 THEN ABS(ae.amount * COALESCE(er.rate, 1)) ELSE 0 END), 0) as income_total,
COUNT(ae.id) as transactions_count,
:target_currency as currency
FROM (#{transactions_scope.to_sql}) t
JOIN entries ae ON ae.entryable_id = t.id AND ae.entryable_type = 'Transaction'
LEFT JOIN exchange_rates er ON (
er.date = ae.date AND
er.from_currency = ae.currency AND
er.to_currency = :target_currency
);
SQL
end
def transactions_scope
@search.relation
end
end

View file

@ -2,7 +2,7 @@
<div class="grid grid-cols-1 md:grid-cols-3 bg-container rounded-xl shadow-border-xs md:divide-x divide-y md:divide-y-0 divide-alpha-black-100 theme-dark:divide-alpha-white-200">
<div class="p-4 space-y-2">
<p class="text-sm text-secondary">Total transactions</p>
<p class="text-primary font-medium text-xl" id="total-transactions"><%= totals.transactions_count.round(0) %></p>
<p class="text-primary font-medium text-xl" id="total-transactions"><%= totals.count.round(0) %></p>
</div>
<div class="p-4 space-y-2">
<p class="text-sm text-secondary">Income</p>

View file

@ -14,16 +14,23 @@ namespace :benchmarking do
family = User.find_by(email: "user@maybe.local").family
scope = family.transactions.active
x.report("IncomeStatement::Totals") do
IncomeStatement::Totals.new(family, transactions_scope: scope).call
end
# x.report("IncomeStatement::Totals") do
# IncomeStatement::Totals.new(family, transactions_scope: scope).call
# end
x.report("IncomeStatement::CategoryStats") do
IncomeStatement::CategoryStats.new(family).call
end
# x.report("IncomeStatement::CategoryStats") do
# IncomeStatement::CategoryStats.new(family).call
# end
x.report("IncomeStatement::FamilyStats") do
IncomeStatement::FamilyStats.new(family).call
# x.report("IncomeStatement::FamilyStats") do
# IncomeStatement::FamilyStats.new(family).call
# end
puts family.entries.count
x.report("Transaction::Totals") do
search = Transaction::Search.new(family)
search.totals
end
x.compare!

View file

@ -148,7 +148,7 @@ class TransactionsControllerTest < ActionDispatch::IntegrationTest
assert_operator overflow_count, :>, 0, "Overflow should show some transactions"
end
test "calls Transaction::Totals service with correct search parameters" do
test "calls Transaction::Search totals method with correct search parameters" do
family = families(:empty)
sign_in users(:empty)
account = family.accounts.create! name: "Test", balance: 0, currency: "USD", accountable: Depository.new
@ -157,19 +157,19 @@ end
search = Transaction::Search.new(family)
totals = OpenStruct.new(
transactions_count: 1,
count: 1,
expense_money: Money.new(10000, "USD"),
income_money: Money.new(0, "USD")
)
Transaction::Search.expects(:new).with(family, filters: {}).returns(search)
Transaction::Totals.expects(:compute).once.with(search).returns(totals)
search.expects(:totals).once.returns(totals)
get transactions_url
assert_response :success
end
test "calls Transaction::Totals service with filtered search parameters" do
test "calls Transaction::Search totals method with filtered search parameters" do
family = families(:empty)
sign_in users(:empty)
account = family.accounts.create! name: "Test", balance: 0, currency: "USD", accountable: Depository.new
@ -179,13 +179,13 @@ end
search = Transaction::Search.new(family, filters: { "categories" => [ "Food" ], "types" => [ "expense" ] })
totals = OpenStruct.new(
transactions_count: 1,
count: 1,
expense_money: Money.new(10000, "USD"),
income_money: Money.new(0, "USD")
)
Transaction::Search.expects(:new).with(family, filters: { "categories" => [ "Food" ], "types" => [ "expense" ] }).returns(search)
Transaction::Totals.expects(:compute).once.with(search).returns(totals)
search.expects(:totals).once.returns(totals)
get transactions_url(q: { categories: [ "Food" ], types: [ "expense" ] })
assert_response :success

View file

@ -8,6 +8,9 @@ class Transaction::SearchTest < ActiveSupport::TestCase
@checking_account = accounts(:depository)
@credit_card_account = accounts(:credit_card)
@loan_account = accounts(:loan)
# Clean up existing entries/transactions from fixtures to ensure test isolation
@family.accounts.each { |account| account.entries.delete_all }
end
test "search filters by transaction types using kind enum" do
@ -44,11 +47,9 @@ class Transaction::SearchTest < ActiveSupport::TestCase
)
# Test transfer type filter (includes loan_payment)
transfer_results = Transaction::Search.new(@family, filters: { types: [ "transfer" ] }).relation
transfer_results = Transaction::Search.new(@family, filters: { types: [ "transfer" ] }).transactions_scope
transfer_ids = transfer_results.pluck(:id)
assert_includes transfer_ids, transfer_entry.entryable.id
assert_includes transfer_ids, payment_entry.entryable.id
assert_includes transfer_ids, loan_payment_entry.entryable.id
@ -56,7 +57,7 @@ class Transaction::SearchTest < ActiveSupport::TestCase
assert_not_includes transfer_ids, standard_entry.entryable.id
# Test expense type filter (excludes transfer kinds but includes one_time)
expense_results = Transaction::Search.new(@family, filters: { types: [ "expense" ] }).relation
expense_results = Transaction::Search.new(@family, filters: { types: [ "expense" ] }).transactions_scope
expense_ids = expense_results.pluck(:id)
assert_includes expense_ids, standard_entry.entryable.id
@ -72,7 +73,7 @@ class Transaction::SearchTest < ActiveSupport::TestCase
kind: "standard"
)
income_results = Transaction::Search.new(@family, filters: { types: [ "income" ] }).relation
income_results = Transaction::Search.new(@family, filters: { types: [ "income" ] }).transactions_scope
income_ids = income_results.pluck(:id)
assert_includes income_ids, income_entry.entryable.id
@ -81,7 +82,7 @@ class Transaction::SearchTest < ActiveSupport::TestCase
assert_not_includes income_ids, transfer_entry.entryable.id
# Test combined expense and income filter (excludes transfer kinds but includes one_time)
non_transfer_results = Transaction::Search.new(@family, filters: { types: [ "expense", "income" ] }).relation
non_transfer_results = Transaction::Search.new(@family, filters: { types: [ "expense", "income" ] }).transactions_scope
non_transfer_ids = non_transfer_results.pluck(:id)
assert_includes non_transfer_ids, standard_entry.entryable.id
@ -113,7 +114,7 @@ class Transaction::SearchTest < ActiveSupport::TestCase
)
# Search for uncategorized transactions
uncategorized_results = Transaction::Search.new(@family, filters: { categories: [ "Uncategorized" ] }).relation
uncategorized_results = Transaction::Search.new(@family, filters: { categories: [ "Uncategorized" ] }).transactions_scope
uncategorized_ids = uncategorized_results.pluck(:id)
# Should include standard uncategorized transactions
@ -142,7 +143,7 @@ class Transaction::SearchTest < ActiveSupport::TestCase
# Test new family-based API
search = Transaction::Search.new(@family, filters: { types: [ "expense" ] })
results = search.relation
results = search.transactions_scope
result_ids = results.pluck(:id)
# Should include expense transactions
@ -159,7 +160,173 @@ class Transaction::SearchTest < ActiveSupport::TestCase
test "family-based API requires family parameter" do
assert_raises(NoMethodError) do
search = Transaction::Search.new({ types: [ "expense" ] })
search.relation # This will fail when trying to call .transactions on a Hash
search.transactions_scope # This will fail when trying to call .transactions on a Hash
end
end
# Totals method tests (lifted from Transaction::TotalsTest)
test "totals computes basic expense and income totals" do
# Create expense transaction
expense_entry = create_transaction(
account: @checking_account,
amount: 100,
category: categories(:food_and_drink),
kind: "standard"
)
# Create income transaction
income_entry = create_transaction(
account: @checking_account,
amount: -200,
kind: "standard"
)
search = Transaction::Search.new(@family)
totals = search.totals
assert_equal 2, totals.count
assert_equal Money.new(100, "USD"), totals.expense_money # $100
assert_equal Money.new(200, "USD"), totals.income_money # $200
end
test "totals handles multi-currency transactions with exchange rates" do
# Create EUR transaction
eur_entry = create_transaction(
account: @checking_account,
amount: 100,
currency: "EUR",
kind: "standard"
)
# Create exchange rate EUR -> USD
ExchangeRate.create!(
from_currency: "EUR",
to_currency: "USD",
rate: 1.1,
date: eur_entry.date
)
# Create USD transaction
usd_entry = create_transaction(
account: @checking_account,
amount: 50,
currency: "USD",
kind: "standard"
)
search = Transaction::Search.new(@family)
totals = search.totals
assert_equal 2, totals.count
# EUR 100 * 1.1 + USD 50 = 110 + 50 = 160
assert_equal Money.new(160, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "totals handles missing exchange rates gracefully" do
# Create EUR transaction without exchange rate
eur_entry = create_transaction(
account: @checking_account,
amount: 100,
currency: "EUR",
kind: "standard"
)
search = Transaction::Search.new(@family)
totals = search.totals
assert_equal 1, totals.count
# Should use rate of 1 when exchange rate is missing
assert_equal Money.new(100, "USD"), totals.expense_money # EUR 100 * 1
assert_equal Money.new(0, "USD"), totals.income_money
end
test "totals respects category filters" do
# Create transactions in different categories
food_entry = create_transaction(
account: @checking_account,
amount: 100,
category: categories(:food_and_drink),
kind: "standard"
)
other_entry = create_transaction(
account: @checking_account,
amount: 50,
category: categories(:income),
kind: "standard"
)
# Filter by food category only
search = Transaction::Search.new(@family, filters: { categories: [ "Food & Drink" ] })
totals = search.totals
assert_equal 1, totals.count
assert_equal Money.new(100, "USD"), totals.expense_money # Only food transaction
assert_equal Money.new(0, "USD"), totals.income_money
end
test "totals respects type filters" do
# Create expense and income transactions
expense_entry = create_transaction(
account: @checking_account,
amount: 100,
kind: "standard"
)
income_entry = create_transaction(
account: @checking_account,
amount: -200,
kind: "standard"
)
# Filter by expense type only
search = Transaction::Search.new(@family, filters: { types: [ "expense" ] })
totals = search.totals
assert_equal 1, totals.count
assert_equal Money.new(100, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "totals handles empty results" do
search = Transaction::Search.new(@family)
totals = search.totals
assert_equal 0, totals.count
assert_equal Money.new(0, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "totals respects excluded transactions filter from search" do
# Create an excluded transaction (should be excluded by default)
excluded_entry = create_transaction(
account: @checking_account,
amount: 100,
kind: "standard"
)
excluded_entry.update!(excluded: true) # Marks it as excluded
# Create a normal transaction
normal_entry = create_transaction(
account: @checking_account,
amount: 50,
kind: "standard"
)
# Default behavior should exclude excluded transactions
search = Transaction::Search.new(@family)
totals = search.totals
assert_equal 1, totals.count
assert_equal Money.new(50, "USD"), totals.expense_money # Only non-excluded transaction
# Explicitly include excluded transactions
search_with_excluded = Transaction::Search.new(@family, filters: { excluded_transactions: true })
totals_with_excluded = search_with_excluded.totals
assert_equal 2, totals_with_excluded.count
assert_equal Money.new(150, "USD"), totals_with_excluded.expense_money # Both transactions
end
end

View file

@ -1,178 +0,0 @@
require "test_helper"
class Transaction::TotalsTest < ActiveSupport::TestCase
include EntriesTestHelper
setup do
@family = families(:dylan_family)
@checking_account = accounts(:depository)
@credit_card_account = accounts(:credit_card)
@loan_account = accounts(:loan)
# Clean up existing entries/transactions from fixtures to ensure test isolation
@family.accounts.each { |account| account.entries.delete_all }
@search = Transaction::Search.new(@family)
end
test "computes basic expense and income totals" do
# Create expense transaction
expense_entry = create_transaction(
account: @checking_account,
amount: 100,
category: categories(:food_and_drink),
kind: "standard"
)
# Create income transaction
income_entry = create_transaction(
account: @checking_account,
amount: -200,
kind: "standard"
)
totals = Transaction::Totals.compute(@search)
assert_equal 2, totals.transactions_count
assert_equal Money.new(100, "USD"), totals.expense_money # $100
assert_equal Money.new(200, "USD"), totals.income_money # $200
end
test "handles multi-currency transactions with exchange rates" do
# Create EUR transaction
eur_entry = create_transaction(
account: @checking_account,
amount: 100,
currency: "EUR",
kind: "standard"
)
# Create exchange rate EUR -> USD
ExchangeRate.create!(
from_currency: "EUR",
to_currency: "USD",
rate: 1.1,
date: eur_entry.date
)
# Create USD transaction
usd_entry = create_transaction(
account: @checking_account,
amount: 50,
currency: "USD",
kind: "standard"
)
totals = Transaction::Totals.compute(@search)
assert_equal 2, totals.transactions_count
# EUR 100 * 1.1 + USD 50 = 110 + 50 = 160
assert_equal Money.new(160, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "handles missing exchange rates gracefully" do
# Create EUR transaction without exchange rate
eur_entry = create_transaction(
account: @checking_account,
amount: 100,
currency: "EUR",
kind: "standard"
)
totals = Transaction::Totals.compute(@search)
assert_equal 1, totals.transactions_count
# Should use rate of 1 when exchange rate is missing
assert_equal Money.new(100, "USD"), totals.expense_money # EUR 100 * 1
assert_equal Money.new(0, "USD"), totals.income_money
end
test "respects category filters" do
# Create transactions in different categories
food_entry = create_transaction(
account: @checking_account,
amount: 100,
category: categories(:food_and_drink),
kind: "standard"
)
other_entry = create_transaction(
account: @checking_account,
amount: 50,
category: categories(:income),
kind: "standard"
)
# Filter by food category only
search = Transaction::Search.new(@family, filters: { categories: [ "Food & Drink" ] })
totals = Transaction::Totals.compute(search)
assert_equal 1, totals.transactions_count
assert_equal Money.new(100, "USD"), totals.expense_money # Only food transaction
assert_equal Money.new(0, "USD"), totals.income_money
end
test "respects type filters" do
# Create expense and income transactions
expense_entry = create_transaction(
account: @checking_account,
amount: 100,
kind: "standard"
)
income_entry = create_transaction(
account: @checking_account,
amount: -200,
kind: "standard"
)
# Filter by expense type only
search = Transaction::Search.new(@family, filters: { types: [ "expense" ] })
totals = Transaction::Totals.compute(search)
assert_equal 1, totals.transactions_count
assert_equal Money.new(100, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "handles empty results" do
totals = Transaction::Totals.compute(@search)
assert_equal 0, totals.transactions_count
assert_equal Money.new(0, "USD"), totals.expense_money
assert_equal Money.new(0, "USD"), totals.income_money
end
test "respects excluded transactions filter from search" do
# Create an excluded transaction (should be excluded by default)
excluded_entry = create_transaction(
account: @checking_account,
amount: 100,
kind: "standard"
)
excluded_entry.update!(excluded: true) # Marks it as excluded
# Create a normal transaction
normal_entry = create_transaction(
account: @checking_account,
amount: 50,
kind: "standard"
)
# Default behavior should exclude excluded transactions
totals = Transaction::Totals.compute(@search)
assert_equal 1, totals.transactions_count
assert_equal Money.new(50, "USD"), totals.expense_money # Only non-excluded transaction
# Explicitly include excluded transactions
search_with_excluded = Transaction::Search.new(@family, filters: { excluded_transactions: true })
totals_with_excluded = Transaction::Totals.compute(search_with_excluded)
assert_equal 2, totals_with_excluded.transactions_count
assert_equal Money.new(150, "USD"), totals_with_excluded.expense_money # Both transactions
end
end