diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 949a9b9d..9ae237ff 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.4.1 +ARG RUBY_VERSION=3.4.4 FROM ruby:${RUBY_VERSION}-slim-bullseye RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ @@ -10,6 +10,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ imagemagick \ iproute2 \ libpq-dev \ + libyaml-dev \ + libyaml-0-2 \ openssh-client \ postgresql-client \ vim diff --git a/.ruby-version b/.ruby-version index 47b322c9..f9892605 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.4.1 +3.4.4 diff --git a/Dockerfile b/Dockerfile index d98833b3..2092d0ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:1 # Make sure RUBY_VERSION matches the Ruby version in .ruby-version and Gemfile -ARG RUBY_VERSION=3.4.1 +ARG RUBY_VERSION=3.4.4 FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim AS base # Rails app lives here @@ -9,7 +9,7 @@ WORKDIR /rails # Install base packages RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y curl libvips postgresql-client + apt-get install --no-install-recommends -y curl libvips postgresql-client libyaml-0-2 # Set production environment ARG BUILD_COMMIT_SHA @@ -23,7 +23,7 @@ ENV RAILS_ENV="production" \ FROM base AS build # Install packages needed to build gems -RUN apt-get install --no-install-recommends -y build-essential libpq-dev git pkg-config +RUN apt-get install --no-install-recommends -y build-essential libpq-dev git pkg-config libyaml-dev # Install application gems COPY .ruby-version Gemfile Gemfile.lock ./ diff --git a/Gemfile b/Gemfile index 26b29fc9..c06ca145 100644 --- a/Gemfile +++ b/Gemfile @@ -24,11 +24,11 @@ gem "stimulus-rails" gem "turbo-rails" gem "view_component" gem "lookbook", ">= 2.3.7" - gem "hotwire_combobox" # Background Jobs gem "sidekiq" +gem "sidekiq-cron" # Monitoring gem "vernier" @@ -44,6 +44,7 @@ gem "aws-sdk-s3", "~> 1.177.0", require: false gem "image_processing", ">= 1.2" # Other +gem "ostruct" gem "bcrypt", "~> 3.1" gem "jwt" gem "faraday" @@ -63,6 +64,10 @@ gem "rotp", "~> 6.3" gem "rqrcode", "~> 3.0" gem "activerecord-import" +# State machines +gem "aasm" +gem "after_commit_everywhere", "~> 1.0" + # AI gem "ruby-openai" diff --git a/Gemfile.lock b/Gemfile.lock index 224f8f9c..6596db18 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,6 +8,8 @@ GIT GEM remote: https://rubygems.org/ specs: + aasm (5.5.0) + concurrent-ruby (~> 1.0) actioncable (7.2.2.1) actionpack (= 7.2.2.1) activesupport (= 7.2.2.1) @@ -83,17 +85,20 @@ GEM tzinfo (~> 2.0, >= 2.0.5) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + after_commit_everywhere (1.6.0) + activerecord (>= 4.2) + activesupport ast (2.4.3) aws-eventstream (1.3.2) - aws-partitions (1.1093.0) - aws-sdk-core (3.222.3) + aws-partitions (1.1105.0) + aws-sdk-core (3.224.0) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.992.0) aws-sigv4 (~> 1.9) base64 jmespath (~> 1, >= 1.6.1) logger - aws-sdk-kms (1.99.0) + aws-sdk-kms (1.101.0) aws-sdk-core (~> 3, >= 3.216.0) aws-sigv4 (~> 1.5) aws-sdk-s3 (1.177.0) @@ -115,7 +120,7 @@ GEM smart_properties bigdecimal (3.1.9) bindex (0.8.1) - bootsnap (1.18.4) + bootsnap (1.18.6) msgpack (~> 1.2) brakeman (7.0.2) racc @@ -139,6 +144,9 @@ GEM bigdecimal rexml crass (1.0.6) + cronex (0.15.0) + tzinfo + unicode (>= 0.4.4.5) css_parser (1.21.1) addressable csv (3.3.4) @@ -152,6 +160,7 @@ GEM dotenv (= 3.1.8) railties (>= 6.1) drb (2.2.1) + erb (5.0.1) erb_lint (0.9.0) activesupport better_html (>= 2.0.1) @@ -160,6 +169,8 @@ GEM rubocop (>= 1) smart_properties erubi (1.13.1) + et-orbi (1.2.11) + tzinfo event_stream_parser (1.0.0) faker (3.5.1) i18n (>= 1.8.11, < 2) @@ -182,6 +193,9 @@ GEM ffi (1.17.2-x86_64-linux-gnu) ffi (1.17.2-x86_64-linux-musl) foreman (0.88.1) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) hashdiff (1.1.2) @@ -230,10 +244,10 @@ GEM rdoc (>= 4.0.0) reline (>= 0.4.2) jmespath (1.6.2) - json (2.11.3) + json (2.12.0) jwt (2.10.1) base64 - language_server-protocol (3.17.0.4) + language_server-protocol (3.17.0.5) launchy (3.1.1) addressable (~> 2.8) childprocess (~> 5.0) @@ -256,7 +270,7 @@ GEM logtail (~> 0.1, >= 0.1.14) logtail-rack (~> 0.1) railties (>= 5.0.0) - loofah (2.24.0) + loofah (2.24.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) lookbook (2.3.9) @@ -290,7 +304,7 @@ GEM multipart-post (2.4.1) net-http (0.6.0) uri - net-imap (0.5.7) + net-imap (0.5.8) date net-protocol net-pop (0.1.2) @@ -319,13 +333,14 @@ GEM octokit (10.0.0) faraday (>= 1, < 3) sawyer (~> 0.9) + ostruct (0.6.1) pagy (9.3.4) parallel (1.27.0) parser (3.3.8.0) ast (~> 2.4.1) racc pg (1.5.9) - plaid (38.0.0) + plaid (39.0.0) faraday (>= 1.0.1, < 3.0) faraday-multipart (>= 1.0.1, < 2.0) platform_agent (1.0.1) @@ -340,17 +355,18 @@ GEM activesupport (>= 7.0.0) rack railties (>= 7.0.0) - psych (5.2.3) + psych (5.2.6) date stringio - public_suffix (6.0.1) + public_suffix (6.0.2) puma (6.6.0) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) - rack (3.1.13) + rack (3.1.15) rack-mini-profiler (3.3.1) rack (>= 1.2.0) - rack-session (2.1.0) + rack-session (2.1.1) base64 (>= 0.1.0) rack (>= 3.0.0) rack-test (2.2.0) @@ -397,9 +413,10 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - rbs (3.9.2) + rbs (3.9.4) logger - rdoc (6.13.1) + rdoc (6.14.0) + erb psych (>= 4.0.0) redcarpet (3.6.1) redis (5.4.0) @@ -416,7 +433,7 @@ GEM chunky_png (~> 1.0) rqrcode_core (~> 2.0) rqrcode_core (2.0.0) - rubocop (1.75.4) + rubocop (1.75.6) json (~> 2.3) language_server-protocol (~> 3.17.0.2) lint_roller (~> 1.1.0) @@ -434,23 +451,23 @@ GEM lint_roller (~> 1.1) rubocop (>= 1.75.0, < 2.0) rubocop-ast (>= 1.38.0, < 2.0) - rubocop-rails (2.31.0) + rubocop-rails (2.32.0) activesupport (>= 4.2.0) lint_roller (~> 1.1) rack (>= 1.1) rubocop (>= 1.75.0, < 2.0) - rubocop-ast (>= 1.38.0, < 2.0) + rubocop-ast (>= 1.44.0, < 2.0) rubocop-rails-omakase (1.1.0) rubocop (>= 1.72) rubocop-performance (>= 1.24) rubocop-rails (>= 2.30) - ruby-lsp (0.23.16) + ruby-lsp (0.23.20) language_server-protocol (~> 3.17.0) prism (>= 1.2, < 2.0) rbs (>= 3, < 4) sorbet-runtime (>= 0.5.10782) - ruby-lsp-rails (0.4.2) - ruby-lsp (>= 0.23.16, < 0.24.0) + ruby-lsp-rails (0.4.3) + ruby-lsp (>= 0.23.18, < 0.24.0) ruby-openai (8.1.0) event_stream_parser (>= 0.3.0, < 2.0.0) faraday (>= 1) @@ -471,14 +488,14 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) - sentry-rails (5.23.0) + sentry-rails (5.24.0) railties (>= 5.0) - sentry-ruby (~> 5.23.0) - sentry-ruby (5.23.0) + sentry-ruby (~> 5.24.0) + sentry-ruby (5.24.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sentry-sidekiq (5.23.0) - sentry-ruby (~> 5.23.0) + sentry-sidekiq (5.24.0) + sentry-ruby (~> 5.24.0) sidekiq (>= 3.0) sidekiq (8.0.3) connection_pool (>= 2.5.0) @@ -486,6 +503,11 @@ GEM logger (>= 1.6.2) rack (>= 3.1.0) redis-client (>= 0.23.2) + sidekiq-cron (2.3.0) + cronex (>= 0.13.0) + fugit (~> 1.8, >= 1.11.1) + globalid (>= 1.0.1) + sidekiq (>= 6.5.0) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -495,21 +517,21 @@ GEM skylight (6.0.4) activesupport (>= 5.2.0) smart_properties (1.17.0) - sorbet-runtime (0.5.12060) + sorbet-runtime (0.5.12117) stimulus-rails (1.3.4) railties (>= 6.0.0) stringio (3.1.7) stripe (15.1.0) - tailwindcss-rails (4.2.2) + tailwindcss-rails (4.2.3) railties (>= 7.0.0) tailwindcss-ruby (~> 4.0) - tailwindcss-ruby (4.1.4) - tailwindcss-ruby (4.1.4-aarch64-linux-gnu) - tailwindcss-ruby (4.1.4-aarch64-linux-musl) - tailwindcss-ruby (4.1.4-arm64-darwin) - tailwindcss-ruby (4.1.4-x86_64-darwin) - tailwindcss-ruby (4.1.4-x86_64-linux-gnu) - tailwindcss-ruby (4.1.4-x86_64-linux-musl) + tailwindcss-ruby (4.1.7) + tailwindcss-ruby (4.1.7-aarch64-linux-gnu) + tailwindcss-ruby (4.1.7-aarch64-linux-musl) + tailwindcss-ruby (4.1.7-arm64-darwin) + tailwindcss-ruby (4.1.7-x86_64-darwin) + tailwindcss-ruby (4.1.7-x86_64-linux-gnu) + tailwindcss-ruby (4.1.7-x86_64-linux-musl) terminal-table (4.0.0) unicode-display_width (>= 1.1.1, < 4) thor (1.3.2) @@ -519,6 +541,7 @@ GEM railties (>= 7.1.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + unicode (0.4.4.5) unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) @@ -526,7 +549,7 @@ GEM useragent (0.16.11) vcr (6.3.1) base64 - vernier (1.7.0) + vernier (1.7.1) view_component (3.22.0) activesupport (>= 5.2.0, < 8.1) concurrent-ruby (= 1.3.4) @@ -548,7 +571,7 @@ GEM xpath (3.2.0) nokogiri (~> 1.8) yard (0.9.37) - zeitwerk (2.7.2) + zeitwerk (2.7.3) PLATFORMS aarch64-linux-gnu @@ -561,7 +584,9 @@ PLATFORMS x86_64-linux-musl DEPENDENCIES + aasm activerecord-import + after_commit_everywhere (~> 1.0) aws-sdk-s3 (~> 1.177.0) bcrypt (~> 3.1) benchmark-ips @@ -592,6 +617,7 @@ DEPENDENCIES lucide-rails! mocha octokit + ostruct pagy pg (~> 1.5) plaid @@ -612,6 +638,7 @@ DEPENDENCIES sentry-ruby sentry-sidekiq sidekiq + sidekiq-cron simplecov skylight stimulus-rails @@ -626,7 +653,7 @@ DEPENDENCIES webmock RUBY VERSION - ruby 3.4.1p0 + ruby 3.4.4p34 BUNDLED WITH - 2.6.3 + 2.6.9 diff --git a/app/assets/tailwind/maybe-design-system.css b/app/assets/tailwind/maybe-design-system.css index 8bf9c6c8..f9dc6039 100644 --- a/app/assets/tailwind/maybe-design-system.css +++ b/app/assets/tailwind/maybe-design-system.css @@ -240,7 +240,7 @@ 100% { stroke-dashoffset: 0; } - } + } } /* Specific override for strong tags in prose under dark mode */ @@ -429,5 +429,3 @@ } } - - diff --git a/app/assets/tailwind/maybe-design-system/background-utils.css b/app/assets/tailwind/maybe-design-system/background-utils.css index 0e4b85b8..2c57f18f 100644 --- a/app/assets/tailwind/maybe-design-system/background-utils.css +++ b/app/assets/tailwind/maybe-design-system/background-utils.css @@ -85,3 +85,7 @@ background-color: var(--color-alpha-black-900); } } + +@utility bg-loader { + @apply bg-surface-inset animate-pulse; +} diff --git a/app/components/tabs_component.html.erb b/app/components/tabs_component.html.erb index 4ec901fa..bfceddad 100644 --- a/app/components/tabs_component.html.erb +++ b/app/components/tabs_component.html.erb @@ -1,6 +1,7 @@ <%= tag.div data: { controller: "tabs", testid: testid, + tabs_session_key_value: session_key, tabs_url_param_key_value: url_param_key, tabs_nav_btn_active_class: active_btn_classes, tabs_nav_btn_inactive_class: inactive_btn_classes diff --git a/app/components/tabs_component.rb b/app/components/tabs_component.rb index 4017b308..747a9420 100644 --- a/app/components/tabs_component.rb +++ b/app/components/tabs_component.rb @@ -27,11 +27,12 @@ class TabsComponent < ViewComponent::Base } } - attr_reader :active_tab, :url_param_key, :variant, :testid + attr_reader :active_tab, :url_param_key, :session_key, :variant, :testid - def initialize(active_tab:, url_param_key: nil, variant: :default, active_btn_classes: "", inactive_btn_classes: "", testid: nil) + def initialize(active_tab:, url_param_key: nil, session_key: nil, variant: :default, active_btn_classes: "", inactive_btn_classes: "", testid: nil) @active_tab = active_tab @url_param_key = url_param_key + @session_key = session_key @variant = variant.to_sym @active_btn_classes = active_btn_classes @inactive_btn_classes = inactive_btn_classes diff --git a/app/components/tabs_controller.js b/app/components/tabs_controller.js index 43a4b192..259765aa 100644 --- a/app/components/tabs_controller.js +++ b/app/components/tabs_controller.js @@ -4,7 +4,7 @@ import { Controller } from "@hotwired/stimulus"; export default class extends Controller { static classes = ["navBtnActive", "navBtnInactive"]; static targets = ["panel", "navBtn"]; - static values = { urlParamKey: String }; + static values = { sessionKey: String, urlParamKey: String }; show(e) { const btn = e.target.closest("button"); @@ -28,11 +28,30 @@ export default class extends Controller { } }); - // Update URL with the selected tab if (this.urlParamKeyValue) { const url = new URL(window.location.href); url.searchParams.set(this.urlParamKeyValue, selectedTabId); window.history.replaceState({}, "", url); } + + // Update URL with the selected tab + if (this.sessionKeyValue) { + this.#updateSessionPreference(selectedTabId); + } } + + #updateSessionPreference(selectedTabId) { + fetch("/current_session", { + method: "PUT", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + "X-CSRF-Token": document.querySelector('[name="csrf-token"]').content, + Accept: "application/json", + }, + body: new URLSearchParams({ + "current_session[tab_key]": this.sessionKeyValue, + "current_session[tab_value]": selectedTabId, + }).toString(), + }); + } } diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index f003ab31..904be2b5 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -26,14 +26,6 @@ class AccountsController < ApplicationController render layout: false end - def sync_all - unless family.syncing? - family.sync_later - end - - redirect_back_or_to accounts_path - end - private def family Current.family diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index a54dc088..260579d1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,5 +1,8 @@ class ApplicationController < ActionController::Base - include Onboardable, Localize, AutoSync, Authentication, Invitable, SelfHostable, StoreLocation, Impersonatable, Breadcrumbable, FeatureGuardable, Notifiable + include RestoreLayoutPreferences, Onboardable, Localize, AutoSync, Authentication, Invitable, + SelfHostable, StoreLocation, Impersonatable, Breadcrumbable, + FeatureGuardable, Notifiable + include Pagy::Backend before_action :detect_os diff --git a/app/controllers/concerns/auto_sync.rb b/app/controllers/concerns/auto_sync.rb index e6ced672..15cdc557 100644 --- a/app/controllers/concerns/auto_sync.rb +++ b/app/controllers/concerns/auto_sync.rb @@ -13,6 +13,7 @@ module AutoSync def family_needs_auto_sync? return false unless Current.family&.accounts&.active&.any? return false if (Current.family.last_sync_created_at&.to_date || 1.day.ago) >= Date.current + return false unless Current.family.auto_sync_on_login Rails.logger.info "Auto-syncing family #{Current.family.id}, last sync was #{Current.family.last_sync_created_at}" diff --git a/app/controllers/concerns/notifiable.rb b/app/controllers/concerns/notifiable.rb index 0d8ea384..b1689f67 100644 --- a/app/controllers/concerns/notifiable.rb +++ b/app/controllers/concerns/notifiable.rb @@ -46,8 +46,6 @@ module Notifiable [ { partial: "shared/notifications/alert", locals: { message: data } } ] when "cta" [ resolve_cta(data) ] - when "loading" - [ { partial: "shared/notifications/loading", locals: { message: data } } ] when "notice" messages = Array(data) messages.map { |message| { partial: "shared/notifications/notice", locals: { message: message } } } diff --git a/app/controllers/concerns/restore_layout_preferences.rb b/app/controllers/concerns/restore_layout_preferences.rb new file mode 100644 index 00000000..284df4cc --- /dev/null +++ b/app/controllers/concerns/restore_layout_preferences.rb @@ -0,0 +1,24 @@ +module RestoreLayoutPreferences + extend ActiveSupport::Concern + + included do + before_action :restore_active_tabs + end + + private + def restore_active_tabs + last_selected_tab = Current.session&.get_preferred_tab("account_sidebar_tab") || "asset" + + @account_group_tab = account_group_tab_param || last_selected_tab + end + + def valid_account_group_tabs + %w[asset liability all] + end + + def account_group_tab_param + param_value = params[:account_sidebar_tab] + return nil unless param_value.in?(valid_account_group_tabs) + param_value + end +end diff --git a/app/controllers/cookie_sessions_controller.rb b/app/controllers/cookie_sessions_controller.rb new file mode 100644 index 00000000..7e76636f --- /dev/null +++ b/app/controllers/cookie_sessions_controller.rb @@ -0,0 +1,22 @@ +class CookieSessionsController < ApplicationController + def update + save_kv_to_session( + cookie_session_params[:tab_key], + cookie_session_params[:tab_value] + ) + + redirect_back_or_to root_path + end + + private + def cookie_session_params + params.require(:cookie_session).permit(:tab_key, :tab_value) + end + + def save_kv_to_session(key, value) + raise "Key must be a string" unless key.is_a?(String) + raise "Value must be a string" unless value.is_a?(String) + + session["custom_#{key}"] = value + end +end diff --git a/app/controllers/current_sessions_controller.rb b/app/controllers/current_sessions_controller.rb new file mode 100644 index 00000000..303b51f0 --- /dev/null +++ b/app/controllers/current_sessions_controller.rb @@ -0,0 +1,14 @@ +class CurrentSessionsController < ApplicationController + def update + if session_params[:tab_key].present? && session_params[:tab_value].present? + Current.session.set_preferred_tab(session_params[:tab_key], session_params[:tab_value]) + end + + head :ok + end + + private + def session_params + params.require(:current_session).permit(:tab_key, :tab_value) + end +end diff --git a/app/controllers/imports_controller.rb b/app/controllers/imports_controller.rb index c1b51c23..20e5f9c4 100644 --- a/app/controllers/imports_controller.rb +++ b/app/controllers/imports_controller.rb @@ -5,6 +5,8 @@ class ImportsController < ApplicationController @import.publish_later redirect_to import_path(@import), notice: "Your import has started in the background." + rescue Import::MaxRowCountExceededError + redirect_back_or_to import_path(@import), alert: "Your import exceeds the maximum row count of #{@import.max_row_count}." end def index diff --git a/app/controllers/pages_controller.rb b/app/controllers/pages_controller.rb index c566d30e..1162483a 100644 --- a/app/controllers/pages_controller.rb +++ b/app/controllers/pages_controller.rb @@ -6,6 +6,23 @@ class PagesController < ApplicationController @balance_sheet = Current.family.balance_sheet @accounts = Current.family.accounts.active.with_attached_logo + period_param = params[:cashflow_period] + @cashflow_period = if period_param.present? + begin + Period.from_key(period_param) + rescue Period::InvalidKeyError + Period.last_30_days + end + else + Period.last_30_days + end + + family_currency = Current.family.currency + income_totals = Current.family.income_statement.income_totals(period: @cashflow_period) + expense_totals = Current.family.income_statement.expense_totals(period: @cashflow_period) + + @cashflow_sankey_data = build_cashflow_sankey_data(income_totals, expense_totals, family_currency) + @breadcrumbs = [ [ "Home", root_path ], [ "Dashboard", nil ] ] end @@ -31,4 +48,98 @@ class PagesController < ApplicationController def github_provider Provider::Registry.get_provider(:github) end + + def build_cashflow_sankey_data(income_totals, expense_totals, currency_symbol) + nodes = [] + links = [] + node_indices = {} # Memoize node indices by a unique key: "type_categoryid" + + # Helper to add/find node and return its index + add_node = ->(unique_key, display_name, value, percentage, color) { + node_indices[unique_key] ||= begin + nodes << { name: display_name, value: value.to_f.round(2), percentage: percentage.to_f.round(1), color: color } + nodes.size - 1 + end + } + + total_income_val = income_totals.total.to_f.round(2) + total_expense_val = expense_totals.total.to_f.round(2) + + # --- Create Central Cash Flow Node --- + cash_flow_idx = add_node.call("cash_flow_node", "Cash Flow", total_income_val, 0, "var(--color-success)") + + # --- Process Income Side --- + income_category_values = Hash.new(0.0) + income_totals.category_totals.each do |ct| + val = ct.total.to_f.round(2) + next if val.zero? || !ct.category.parent_id + income_category_values[ct.category.parent_id] += val + end + + income_totals.category_totals.each do |ct| + val = ct.total.to_f.round(2) + percentage_of_total_income = total_income_val.zero? ? 0 : (val / total_income_val * 100).round(1) + next if val.zero? + + node_display_name = ct.category.name + node_value_for_label = val + income_category_values[ct.category.id] # This sum is for parent node display + node_percentage_for_label = total_income_val.zero? ? 0 : (node_value_for_label / total_income_val * 100).round(1) + + node_color = ct.category.color.presence || Category::COLORS.sample + current_cat_idx = add_node.call("income_#{ct.category.id}", node_display_name, node_value_for_label, node_percentage_for_label, node_color) + + if ct.category.parent_id + parent_cat_idx = node_indices["income_#{ct.category.parent_id}"] + parent_cat_idx ||= add_node.call("income_#{ct.category.parent.id}", ct.category.parent.name, income_category_values[ct.category.parent.id], 0, ct.category.parent.color || Category::COLORS.sample) # Parent percentage will be recalc based on its total flow + links << { source: current_cat_idx, target: parent_cat_idx, value: val, color: node_color, percentage: percentage_of_total_income } + else + links << { source: current_cat_idx, target: cash_flow_idx, value: val, color: node_color, percentage: percentage_of_total_income } + end + end + + # --- Process Expense Side --- + expense_category_values = Hash.new(0.0) + expense_totals.category_totals.each do |ct| + val = ct.total.to_f.round(2) + next if val.zero? || !ct.category.parent_id + expense_category_values[ct.category.parent_id] += val + end + + expense_totals.category_totals.each do |ct| + val = ct.total.to_f.round(2) + percentage_of_total_expense = total_expense_val.zero? ? 0 : (val / total_expense_val * 100).round(1) + next if val.zero? + + node_display_name = ct.category.name + node_value_for_label = val + expense_category_values[ct.category.id] + node_percentage_for_label = total_expense_val.zero? ? 0 : (node_value_for_label / total_expense_val * 100).round(1) # Percentage relative to total expenses for expense nodes + + node_color = ct.category.color.presence || Category::UNCATEGORIZED_COLOR + current_cat_idx = add_node.call("expense_#{ct.category.id}", node_display_name, node_value_for_label, node_percentage_for_label, node_color) + + if ct.category.parent_id + parent_cat_idx = node_indices["expense_#{ct.category.parent_id}"] + parent_cat_idx ||= add_node.call("expense_#{ct.category.parent.id}", ct.category.parent.name, expense_category_values[ct.category.parent.id], 0, ct.category.parent.color || Category::UNCATEGORIZED_COLOR) + links << { source: parent_cat_idx, target: current_cat_idx, value: val, color: nodes[parent_cat_idx][:color], percentage: percentage_of_total_expense } + else + links << { source: cash_flow_idx, target: current_cat_idx, value: val, color: node_color, percentage: percentage_of_total_expense } + end + end + + # --- Process Surplus --- + leftover = (total_income_val - total_expense_val).round(2) + if leftover.positive? + percentage_of_total_income_for_surplus = total_income_val.zero? ? 0 : (leftover / total_income_val * 100).round(1) + surplus_idx = add_node.call("surplus_node", "Surplus", leftover, percentage_of_total_income_for_surplus, "var(--color-success)") + links << { source: cash_flow_idx, target: surplus_idx, value: leftover, color: "var(--color-success)", percentage: percentage_of_total_income_for_surplus } + end + + # Update Cash Flow and Income node percentages (relative to total income) + if node_indices["cash_flow_node"] + nodes[node_indices["cash_flow_node"]][:percentage] = 100.0 + end + # No primary income node anymore, percentages are on individual income cats relative to total_income_val + + { nodes: nodes, links: links, currency_symbol: Money::Currency.new(currency_symbol).symbol } + end end diff --git a/app/controllers/plaid_items_controller.rb b/app/controllers/plaid_items_controller.rb index 37efd5e3..8812cf0f 100644 --- a/app/controllers/plaid_items_controller.rb +++ b/app/controllers/plaid_items_controller.rb @@ -2,8 +2,8 @@ class PlaidItemsController < ApplicationController before_action :set_plaid_item, only: %i[destroy sync] def create - Current.family.plaid_items.create_from_public_token( - plaid_item_params[:public_token], + Current.family.create_plaid_item!( + public_token: plaid_item_params[:public_token], item_name: item_name, region: plaid_item_params[:region] ) diff --git a/app/controllers/transactions_controller.rb b/app/controllers/transactions_controller.rb index 4d47c06b..e5382e73 100644 --- a/app/controllers/transactions_controller.rb +++ b/app/controllers/transactions_controller.rb @@ -61,7 +61,7 @@ class TransactionsController < ApplicationController if @entry.save @entry.sync_account_later @entry.lock_saved_attributes! - @entry.transaction.lock!(:tag_ids) if @entry.transaction.tags.any? + @entry.transaction.lock_attr!(:tag_ids) if @entry.transaction.tags.any? flash[:notice] = "Transaction created" @@ -88,7 +88,7 @@ class TransactionsController < ApplicationController @entry.sync_account_later @entry.lock_saved_attributes! - @entry.transaction.lock!(:tag_ids) if @entry.transaction.tags.any? + @entry.transaction.lock_attr!(:tag_ids) if @entry.transaction.tags.any? respond_to do |format| format.html { redirect_back_or_to account_path(@entry.account), notice: "Transaction updated" } diff --git a/app/javascript/controllers/rule/conditions_controller.js b/app/javascript/controllers/rule/conditions_controller.js index d0c12941..1a20d00d 100644 --- a/app/javascript/controllers/rule/conditions_controller.js +++ b/app/javascript/controllers/rule/conditions_controller.js @@ -13,7 +13,7 @@ export default class extends Controller { addSubCondition() { const html = this.subConditionTemplateTarget.innerHTML.replaceAll( - "IDX_PLACEHOLDER", + "IDX_CHILD_PLACEHOLDER", this.#uniqueKey(), ); @@ -110,6 +110,6 @@ export default class extends Controller { } #uniqueKey() { - return Math.random().toString(36).substring(2, 15); + return Date.now(); } } diff --git a/app/javascript/controllers/sankey_chart_controller.js b/app/javascript/controllers/sankey_chart_controller.js new file mode 100644 index 00000000..9601b088 --- /dev/null +++ b/app/javascript/controllers/sankey_chart_controller.js @@ -0,0 +1,204 @@ +import { Controller } from "@hotwired/stimulus"; +import * as d3 from "d3"; +import { sankey, sankeyLinkHorizontal } from "d3-sankey"; + +// Connects to data-controller="sankey-chart" +export default class extends Controller { + static values = { + data: Object, + nodeWidth: { type: Number, default: 15 }, + nodePadding: { type: Number, default: 20 }, + currencySymbol: { type: String, default: "$" } + }; + + connect() { + this.resizeObserver = new ResizeObserver(() => this.#draw()); + this.resizeObserver.observe(this.element); + this.#draw(); + } + + disconnect() { + this.resizeObserver?.disconnect(); + } + + #draw() { + const { nodes = [], links = [] } = this.dataValue || {}; + + if (!nodes.length || !links.length) return; + + // Clear previous SVG + d3.select(this.element).selectAll("svg").remove(); + + const width = this.element.clientWidth || 600; + const height = this.element.clientHeight || 400; + + const svg = d3 + .select(this.element) + .append("svg") + .attr("width", width) + .attr("height", height); + + const sankeyGenerator = sankey() + .nodeWidth(this.nodeWidthValue) + .nodePadding(this.nodePaddingValue) + .extent([ + [16, 16], + [width - 16, height - 16], + ]); + + const sankeyData = sankeyGenerator({ + nodes: nodes.map((d) => Object.assign({}, d)), + links: links.map((d) => Object.assign({}, d)), + }); + + // Define gradients for links + const defs = svg.append("defs"); + + sankeyData.links.forEach((link, i) => { + const gradientId = `link-gradient-${link.source.index}-${link.target.index}-${i}`; + + const getStopColorWithOpacity = (nodeColorInput, opacity = 0.1) => { + let colorStr = nodeColorInput || "var(--color-gray-400)"; + if (colorStr === "var(--color-success)") { + colorStr = "#10A861"; // Hex for --color-green-600 + } + // Add other CSS var to hex mappings here if needed + + if (colorStr.startsWith("var(--")) { // Unmapped CSS var, use as is (likely solid) + return colorStr; + } + + const d3Color = d3.color(colorStr); + return d3Color ? d3Color.copy({ opacity: opacity }) : "var(--color-gray-400)"; + }; + + const sourceStopColor = getStopColorWithOpacity(link.source.color); + const targetStopColor = getStopColorWithOpacity(link.target.color); + + const gradient = defs.append("linearGradient") + .attr("id", gradientId) + .attr("gradientUnits", "userSpaceOnUse") + .attr("x1", link.source.x1) + .attr("x2", link.target.x0); + + gradient.append("stop") + .attr("offset", "0%") + .attr("stop-color", sourceStopColor); + + gradient.append("stop") + .attr("offset", "100%") + .attr("stop-color", targetStopColor); + }); + + // Draw links + svg + .append("g") + .attr("fill", "none") + .selectAll("path") + .data(sankeyData.links) + .join("path") + .attr("d", (d) => { + const sourceX = d.source.x1; + const targetX = d.target.x0; + const path = d3.linkHorizontal()({ + source: [sourceX, d.y0], + target: [targetX, d.y1] + }); + return path; + }) + .attr("stroke", (d, i) => `url(#link-gradient-${d.source.index}-${d.target.index}-${i})`) + .attr("stroke-width", (d) => Math.max(1, d.width)) + .append("title") + .text((d) => `${nodes[d.source.index].name} → ${nodes[d.target.index].name}: ${this.currencySymbolValue}${Number.parseFloat(d.value).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })} (${d.percentage}%)`); + + // Draw nodes + const node = svg + .append("g") + .selectAll("g") + .data(sankeyData.nodes) + .join("g"); + + const cornerRadius = 8; + + node.append("path") + .attr("d", (d) => { + const x0 = d.x0; + const y0 = d.y0; + const x1 = d.x1; + const y1 = d.y1; + const h = y1 - y0; + // const w = x1 - x0; // Not directly used in path string, but good for context + + // Dynamic corner radius based on node height, maxed at 8 + const effectiveCornerRadius = Math.max(0, Math.min(cornerRadius, h / 2)); + + const isSourceNode = d.sourceLinks && d.sourceLinks.length > 0 && (!d.targetLinks || d.targetLinks.length === 0); + const isTargetNode = d.targetLinks && d.targetLinks.length > 0 && (!d.sourceLinks || d.sourceLinks.length === 0); + + if (isSourceNode) { // Round left corners, flat right for "Total Income" + if (h < effectiveCornerRadius * 2) { + return `M ${x0},${y0} L ${x1},${y0} L ${x1},${y1} L ${x0},${y1} Z`; + } + return `M ${x0 + effectiveCornerRadius},${y0} + L ${x1},${y0} + L ${x1},${y1} + L ${x0 + effectiveCornerRadius},${y1} + Q ${x0},${y1} ${x0},${y1 - effectiveCornerRadius} + L ${x0},${y0 + effectiveCornerRadius} + Q ${x0},${y0} ${x0 + effectiveCornerRadius},${y0} Z`; + } + + if (isTargetNode) { // Flat left corners, round right for Categories/Surplus + if (h < effectiveCornerRadius * 2) { + return `M ${x0},${y0} L ${x1},${y0} L ${x1},${y1} L ${x0},${y1} Z`; + } + return `M ${x0},${y0} + L ${x1 - effectiveCornerRadius},${y0} + Q ${x1},${y0} ${x1},${y0 + effectiveCornerRadius} + L ${x1},${y1 - effectiveCornerRadius} + Q ${x1},${y1} ${x1 - effectiveCornerRadius},${y1} + L ${x0},${y1} Z`; + } + + // Fallback for intermediate nodes (e.g., "Cash Flow") - draw as a simple sharp-cornered rectangle + return `M ${x0},${y0} L ${x1},${y0} L ${x1},${y1} L ${x0},${y1} Z`; + }) + .attr("fill", (d) => d.color || "var(--color-gray-400)") + .attr("stroke", (d) => { + // If a node has an explicit color assigned (even if it's a gray variable), + // it gets no stroke. Only truly un-colored nodes (falling back to default fill) + // would get a stroke, but our current data structure assigns colors to all nodes. + if (d.color) { + return "none"; + } + return "var(--color-gray-500)"; // Fallback, likely unused with current data + }); + + const stimulusControllerInstance = this; + node + .append("text") + .attr("x", (d) => (d.x0 < width / 2 ? d.x1 + 6 : d.x0 - 6)) + .attr("y", (d) => (d.y1 + d.y0) / 2) + .attr("dy", "-0.2em") + .attr("text-anchor", (d) => (d.x0 < width / 2 ? "start" : "end")) + .attr("class", "text-xs font-medium text-primary fill-current") + .each(function (d) { + const textElement = d3.select(this); + textElement.selectAll("tspan").remove(); + + // Node Name on the first line + textElement.append("tspan") + .text(d.name); + + // Financial details on the second line + const financialDetailsTspan = textElement.append("tspan") + .attr("x", textElement.attr("x")) + .attr("dy", "1.2em") + .attr("class", "font-mono text-secondary") + .style("font-size", "0.65rem"); // Explicitly set smaller font size + + financialDetailsTspan.append("tspan") + .text(stimulusControllerInstance.currencySymbolValue + Number.parseFloat(d.value).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })); + }); + } +} \ No newline at end of file diff --git a/app/javascript/controllers/sidebar_tabs_controller.js b/app/javascript/controllers/sidebar_tabs_controller.js deleted file mode 100644 index f88a70f7..00000000 --- a/app/javascript/controllers/sidebar_tabs_controller.js +++ /dev/null @@ -1,16 +0,0 @@ -import { Controller } from "@hotwired/stimulus"; - -// Connects to data-controller="sidebar-tabs" -export default class extends Controller { - static targets = ["account"]; - - select(event) { - this.accountTargets.forEach((account) => { - if (account.contains(event.target)) { - account.classList.add("bg-container"); - } else { - account.classList.remove("bg-container"); - } - }); - } -} diff --git a/app/javascript/shims/d3-array-default.js b/app/javascript/shims/d3-array-default.js new file mode 100644 index 00000000..1b1e088e --- /dev/null +++ b/app/javascript/shims/d3-array-default.js @@ -0,0 +1,3 @@ +import * as d3Array from "d3-array-src"; +export * from "d3-array-src"; +export default d3Array; \ No newline at end of file diff --git a/app/javascript/shims/d3-shape-default.js b/app/javascript/shims/d3-shape-default.js new file mode 100644 index 00000000..23920eda --- /dev/null +++ b/app/javascript/shims/d3-shape-default.js @@ -0,0 +1,3 @@ +import * as d3Shape from "d3-shape-src"; +export * from "d3-shape-src"; +export default d3Shape; \ No newline at end of file diff --git a/app/jobs/fetch_security_info_job.rb b/app/jobs/fetch_security_info_job.rb deleted file mode 100644 index e789222f..00000000 --- a/app/jobs/fetch_security_info_job.rb +++ /dev/null @@ -1,21 +0,0 @@ -class FetchSecurityInfoJob < ApplicationJob - queue_as :low_priority - - def perform(security_id) - return unless Security.provider.present? - - security = Security.find(security_id) - - params = { - ticker: security.ticker - } - params[:mic_code] = security.exchange_mic if security.exchange_mic.present? - params[:operating_mic] = security.exchange_operating_mic if security.exchange_operating_mic.present? - - security_info_response = Security.provider.fetch_security_info(**params) - - security.update( - name: security_info_response.info.dig("name") - ) - end -end diff --git a/app/jobs/import_market_data_job.rb b/app/jobs/import_market_data_job.rb new file mode 100644 index 00000000..294d5836 --- /dev/null +++ b/app/jobs/import_market_data_job.rb @@ -0,0 +1,20 @@ +# This job runs daily at market close. See config/schedule.yml for details. +# +# The primary purpose of this job is to: +# 1. Determine what exchange rate pairs, security prices, and other market data all of our users need to view historical account balance data +# 2. For each needed rate/price, fetch from our data provider and upsert to our database +# +# Each individual account sync will still fetch any missing market data that isn't yet synced, but by running +# this job daily, we significantly reduce overlapping account syncs that both need the same market data (e.g. common security like `AAPL`) +# +class ImportMarketDataJob < ApplicationJob + queue_as :scheduled + + def perform(opts) + opts = opts.symbolize_keys + mode = opts.fetch(:mode, :full) + clear_cache = opts.fetch(:clear_cache, false) + + MarketDataImporter.new(mode: mode, clear_cache: clear_cache).import_all + end +end diff --git a/app/jobs/sync_cleaner_job.rb b/app/jobs/sync_cleaner_job.rb new file mode 100644 index 00000000..f5e42551 --- /dev/null +++ b/app/jobs/sync_cleaner_job.rb @@ -0,0 +1,7 @@ +class SyncCleanerJob < ApplicationJob + queue_as :scheduled + + def perform + Sync.clean + end +end diff --git a/app/models/account.rb b/app/models/account.rb index 352335e0..4984fb89 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -1,5 +1,5 @@ class Account < ApplicationRecord - include Syncable, Monetizable, Chartable, Linkable, Convertible, Enrichable + include Syncable, Monetizable, Chartable, Linkable, Enrichable validates :name, :balance, :currency, presence: true @@ -61,6 +61,18 @@ class Account < ApplicationRecord end end + def syncing? + self_syncing = syncs.visible.any? + + # Since Plaid Items sync as a "group", if the item is syncing, even if the account + # sync hasn't yet started (i.e. we're still fetching the Plaid data), show it as syncing in UI. + if linked? + plaid_account&.plaid_item&.syncing? || self_syncing + else + self_syncing + end + end + def institution_domain url_string = plaid_account&.plaid_item&.institution_url return nil unless url_string.present? @@ -81,21 +93,6 @@ class Account < ApplicationRecord DestroyJob.perform_later(self) end - def sync_data(sync, start_date: nil) - Rails.logger.info("Processing balances (#{linked? ? 'reverse' : 'forward'})") - sync_balances - end - - def post_sync(sync) - family.remove_syncing_notice! - - accountable.post_sync(sync) - - unless sync.child? - family.auto_match_transfers! - end - end - def current_holdings holdings.where(currency: currency, date: holdings.maximum(:date)).order(amount: :desc) end @@ -172,10 +169,4 @@ class Account < ApplicationRecord def long_subtype_label accountable_class.long_subtype_label_for(subtype) || accountable_class.display_name end - - private - def sync_balances - strategy = linked? ? :reverse : :forward - Balance::Syncer.new(self, strategy: strategy).sync_balances - end end diff --git a/app/models/account/convertible.rb b/app/models/account/convertible.rb deleted file mode 100644 index fde6fa10..00000000 --- a/app/models/account/convertible.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Account::Convertible - extend ActiveSupport::Concern - - def sync_required_exchange_rates - unless requires_exchange_rates? - Rails.logger.info("No exchange rate sync needed for account #{id}") - return - end - - affected_row_count = ExchangeRate.sync_provider_rates( - from: currency, - to: target_currency, - start_date: start_date, - ) - - Rails.logger.info("Synced #{affected_row_count} exchange rates for account #{id}") - end - - private - def target_currency - family.currency - end - - def requires_exchange_rates? - currency != target_currency - end -end diff --git a/app/models/account/market_data_importer.rb b/app/models/account/market_data_importer.rb new file mode 100644 index 00000000..af538314 --- /dev/null +++ b/app/models/account/market_data_importer.rb @@ -0,0 +1,82 @@ +class Account::MarketDataImporter + attr_reader :account + + def initialize(account) + @account = account + end + + def import_all + import_exchange_rates + import_security_prices + end + + def import_exchange_rates + return unless needs_exchange_rates? + return unless ExchangeRate.provider + + pair_dates = {} + + # 1. ENTRY-BASED PAIRS – currencies that differ from the account currency + account.entries + .where.not(currency: account.currency) + .group(:currency) + .minimum(:date) + .each do |source_currency, date| + key = [ source_currency, account.currency ] + pair_dates[key] = [ pair_dates[key], date ].compact.min + end + + # 2. ACCOUNT-BASED PAIR – convert the account currency to the family currency (if different) + if foreign_account? + key = [ account.currency, account.family.currency ] + pair_dates[key] = [ pair_dates[key], account.start_date ].compact.min + end + + pair_dates.each do |(source, target), start_date| + ExchangeRate.import_provider_rates( + from: source, + to: target, + start_date: start_date, + end_date: Date.current + ) + end + end + + def import_security_prices + return unless Security.provider + + account_securities = account.trades.map(&:security).uniq + + return if account_securities.empty? + + account_securities.each do |security| + security.import_provider_prices( + start_date: first_required_price_date(security), + end_date: Date.current + ) + + security.import_provider_details + end + end + + private + # Calculates the first date we require a price for the given security scoped to this account + def first_required_price_date(security) + account.trades.with_entry + .where(security: security) + .where(entries: { account_id: account.id }) + .minimum("entries.date") + end + + def needs_exchange_rates? + has_multi_currency_entries? || foreign_account? + end + + def has_multi_currency_entries? + account.entries.where.not(currency: account.currency).exists? + end + + def foreign_account? + account.currency != account.family.currency + end +end diff --git a/app/models/account/sync_complete_event.rb b/app/models/account/sync_complete_event.rb new file mode 100644 index 00000000..32315375 --- /dev/null +++ b/app/models/account/sync_complete_event.rb @@ -0,0 +1,63 @@ +class Account::SyncCompleteEvent + attr_reader :account + + Error = Class.new(StandardError) + + def initialize(account) + @account = account + end + + def broadcast + # Replace account row in accounts list + account.broadcast_replace_to( + account.family, + target: "account_#{account.id}", + partial: "accounts/account", + locals: { account: account } + ) + + # Replace the groups this account belongs to in the sidebar + account_group_ids.each do |id| + account.broadcast_replace_to( + account.family, + target: id, + partial: "accounts/accountable_group", + locals: { account_group: account_group, open: true } + ) + end + + # If this is a manual, unlinked account (i.e. not part of a Plaid Item), + # trigger the family sync complete broadcast so net worth graph is updated + unless account.linked? + account.family.broadcast_sync_complete + end + + # Refresh entire account page (only applies if currently viewing this account) + account.broadcast_refresh + end + + private + # The sidebar will show the account in both its classification tab and the "all" tab, + # so we need to broadcast to both. + def account_group_ids + unless account_group.present? + error = Error.new("Account #{account.id} is not part of an account group") + Rails.logger.warn(error.message) + Sentry.capture_exception(error, level: :warning) + return [] + end + + id = account_group.id + [ id, "#{account_group.classification}_#{id}" ] + end + + def account_group + family_balance_sheet.account_groups.find do |group| + group.accounts.any? { |a| a.id == account.id } + end + end + + def family_balance_sheet + account.family.balance_sheet + end +end diff --git a/app/models/account/syncer.rb b/app/models/account/syncer.rb new file mode 100644 index 00000000..ab198a95 --- /dev/null +++ b/app/models/account/syncer.rb @@ -0,0 +1,37 @@ +class Account::Syncer + attr_reader :account + + def initialize(account) + @account = account + end + + def perform_sync(sync) + Rails.logger.info("Processing balances (#{account.linked? ? 'reverse' : 'forward'})") + import_market_data + materialize_balances + end + + def perform_post_sync + account.family.auto_match_transfers! + end + + private + def materialize_balances + strategy = account.linked? ? :reverse : :forward + Balance::Materializer.new(account, strategy: strategy).materialize_balances + end + + # Syncs all the exchange rates + security prices this account needs to display historical chart data + # + # This is a *supplemental* sync. The daily market data sync should have already populated + # a majority or all of this data, so this is often a no-op. + # + # We rescue errors here because if this operation fails, we don't want to fail the entire sync since + # we have reasonable fallbacks for missing market data. + def import_market_data + Account::MarketDataImporter.new(account).import_all + rescue => e + Rails.logger.error("Error syncing market data for account #{account.id}: #{e.message}") + Sentry.capture_exception(e) + end +end diff --git a/app/models/account_import.rb b/app/models/account_import.rb index 96fdfd47..aa4c6dfe 100644 --- a/app/models/account_import.rb +++ b/app/models/account_import.rb @@ -54,4 +54,8 @@ class AccountImport < Import CSV.parse(template, headers: true) end + + def max_row_count + 50 + end end diff --git a/app/models/balance/base_calculator.rb b/app/models/balance/base_calculator.rb deleted file mode 100644 index 2d01dfe7..00000000 --- a/app/models/balance/base_calculator.rb +++ /dev/null @@ -1,35 +0,0 @@ -class Balance::BaseCalculator - attr_reader :account - - def initialize(account) - @account = account - end - - def calculate - Rails.logger.tagged(self.class.name) do - calculate_balances - end - end - - private - def sync_cache - @sync_cache ||= Balance::SyncCache.new(account) - end - - def build_balance(date, cash_balance, holdings_value) - Balance.new( - account_id: account.id, - date: date, - balance: holdings_value + cash_balance, - cash_balance: cash_balance, - currency: account.currency - ) - end - - def calculate_next_balance(prior_balance, transactions, direction: :forward) - flows = transactions.sum(&:amount) - negated = direction == :forward ? account.asset? : account.liability? - flows *= -1 if negated - prior_balance + flows - end -end diff --git a/app/models/balance/forward_calculator.rb b/app/models/balance/forward_calculator.rb index d024d2c6..4e6f2d5c 100644 --- a/app/models/balance/forward_calculator.rb +++ b/app/models/balance/forward_calculator.rb @@ -1,4 +1,16 @@ -class Balance::ForwardCalculator < Balance::BaseCalculator +class Balance::ForwardCalculator + attr_reader :account + + def initialize(account) + @account = account + end + + def calculate + Rails.logger.tagged("Balance::ForwardCalculator") do + calculate_balances + end + end + private def calculate_balances current_cash_balance = 0 @@ -25,4 +37,25 @@ class Balance::ForwardCalculator < Balance::BaseCalculator @balances end + + def sync_cache + @sync_cache ||= Balance::SyncCache.new(account) + end + + def build_balance(date, cash_balance, holdings_value) + Balance.new( + account_id: account.id, + date: date, + balance: holdings_value + cash_balance, + cash_balance: cash_balance, + currency: account.currency + ) + end + + def calculate_next_balance(prior_balance, transactions, direction: :forward) + flows = transactions.sum(&:amount) + negated = direction == :forward ? account.asset? : account.liability? + flows *= -1 if negated + prior_balance + flows + end end diff --git a/app/models/balance/syncer.rb b/app/models/balance/materializer.rb similarity index 88% rename from app/models/balance/syncer.rb rename to app/models/balance/materializer.rb index 362b87aa..75a98ffd 100644 --- a/app/models/balance/syncer.rb +++ b/app/models/balance/materializer.rb @@ -1,4 +1,4 @@ -class Balance::Syncer +class Balance::Materializer attr_reader :account, :strategy def initialize(account, strategy:) @@ -6,9 +6,9 @@ class Balance::Syncer @strategy = strategy end - def sync_balances + def materialize_balances Balance.transaction do - sync_holdings + materialize_holdings calculate_balances Rails.logger.info("Persisting #{@balances.size} balances") @@ -19,14 +19,12 @@ class Balance::Syncer if strategy == :forward update_account_info end - - account.sync_required_exchange_rates end end private - def sync_holdings - @holdings = Holding::Syncer.new(account, strategy: strategy).sync_holdings + def materialize_holdings + @holdings = Holding::Materializer.new(account, strategy: strategy).materialize_holdings end def update_account_info diff --git a/app/models/balance/reverse_calculator.rb b/app/models/balance/reverse_calculator.rb index 6a2ba70b..52a05608 100644 --- a/app/models/balance/reverse_calculator.rb +++ b/app/models/balance/reverse_calculator.rb @@ -1,4 +1,16 @@ -class Balance::ReverseCalculator < Balance::BaseCalculator +class Balance::ReverseCalculator + attr_reader :account + + def initialize(account) + @account = account + end + + def calculate + Rails.logger.tagged("Balance::ReverseCalculator") do + calculate_balances + end + end + private def calculate_balances current_cash_balance = account.cash_balance @@ -35,4 +47,25 @@ class Balance::ReverseCalculator < Balance::BaseCalculator @balances end + + def sync_cache + @sync_cache ||= Balance::SyncCache.new(account) + end + + def build_balance(date, cash_balance, holdings_value) + Balance.new( + account_id: account.id, + date: date, + balance: holdings_value + cash_balance, + cash_balance: cash_balance, + currency: account.currency + ) + end + + def calculate_next_balance(prior_balance, transactions, direction: :forward) + flows = transactions.sum(&:amount) + negated = direction == :forward ? account.asset? : account.liability? + flows *= -1 if negated + prior_balance + flows + end end diff --git a/app/models/balance_sheet.rb b/app/models/balance_sheet.rb index c289f86f..a89a9859 100644 --- a/app/models/balance_sheet.rb +++ b/app/models/balance_sheet.rb @@ -22,20 +22,25 @@ class BalanceSheet end def classification_groups + asset_groups = account_groups("asset") + liability_groups = account_groups("liability") + [ ClassificationGroup.new( key: "asset", display_name: "Assets", icon: "plus", total_money: total_assets_money, - account_groups: account_groups("asset") + account_groups: asset_groups, + syncing?: asset_groups.any?(&:syncing?) ), ClassificationGroup.new( key: "liability", display_name: "Debts", icon: "minus", total_money: total_liabilities_money, - account_groups: account_groups("liability") + account_groups: liability_groups, + syncing?: liability_groups.any?(&:syncing?) ) ] end @@ -43,13 +48,17 @@ class BalanceSheet def account_groups(classification = nil) classification_accounts = classification ? totals_query.filter { |t| t.classification == classification } : totals_query classification_total = classification_accounts.sum(&:converted_balance) - account_groups = classification_accounts.group_by(&:accountable_type).transform_keys { |k| Accountable.from_type(k) } + account_groups = classification_accounts.group_by(&:accountable_type) + .transform_keys { |k| Accountable.from_type(k) } - account_groups.map do |accountable, accounts| + groups = account_groups.map do |accountable, accounts| group_total = accounts.sum(&:converted_balance) + key = accountable.model_name.param_key + AccountGroup.new( - key: accountable.model_name.param_key, + id: classification ? "#{classification}_#{key}_group" : "#{key}_group", + key: key, name: accountable.display_name, classification: accountable.classification, total: group_total, @@ -57,6 +66,7 @@ class BalanceSheet weight: classification_total.zero? ? 0 : group_total / classification_total.to_d * 100, missing_rates?: accounts.any? { |a| a.missing_rates? }, color: accountable.color, + syncing?: accounts.any?(&:is_syncing), accounts: accounts.map do |account| account.define_singleton_method(:weight) do classification_total.zero? ? 0 : account.converted_balance / classification_total.to_d * 100 @@ -65,7 +75,13 @@ class BalanceSheet account end.sort_by(&:weight).reverse ) - end.sort_by(&:weight).reverse + end + + groups.sort_by do |group| + manual_order = Accountable::TYPES + type_name = group.key.camelize + manual_order.index(type_name) || Float::INFINITY + end end def net_worth_series(period: Period.last_30_days) @@ -76,9 +92,13 @@ class BalanceSheet family.currency end + def syncing? + classification_groups.any? { |group| group.syncing? } + end + private - ClassificationGroup = Struct.new(:key, :display_name, :icon, :total_money, :account_groups, keyword_init: true) - AccountGroup = Struct.new(:key, :name, :accountable_type, :classification, :total, :total_money, :weight, :accounts, :color, :missing_rates?, keyword_init: true) + ClassificationGroup = Struct.new(:key, :display_name, :icon, :total_money, :account_groups, :syncing?, keyword_init: true) + AccountGroup = Struct.new(:id, :key, :name, :accountable_type, :classification, :total, :total_money, :weight, :accounts, :color, :missing_rates?, :syncing?, keyword_init: true) def active_accounts family.accounts.active.with_attached_logo @@ -87,9 +107,15 @@ class BalanceSheet def totals_query @totals_query ||= active_accounts .joins(ActiveRecord::Base.sanitize_sql_array([ "LEFT JOIN exchange_rates ON exchange_rates.date = CURRENT_DATE AND accounts.currency = exchange_rates.from_currency AND exchange_rates.to_currency = ?", currency ])) + .joins(ActiveRecord::Base.sanitize_sql_array([ + "LEFT JOIN syncs ON syncs.syncable_id = accounts.id AND syncs.syncable_type = 'Account' AND syncs.status IN (?) AND syncs.created_at > ?", + %w[pending syncing], + Sync::VISIBLE_FOR.ago + ])) .select( "accounts.*", "SUM(accounts.balance * COALESCE(exchange_rates.rate, 1)) as converted_balance", + "COUNT(syncs.id) > 0 as is_syncing", ActiveRecord::Base.sanitize_sql_array([ "COUNT(CASE WHEN accounts.currency <> ? AND exchange_rates.rate IS NULL THEN 1 END) as missing_rates", currency ]) ) .group(:classification, :accountable_type, :id) diff --git a/app/models/concerns/accountable.rb b/app/models/concerns/accountable.rb index 9d0ecb34..12ee9888 100644 --- a/app/models/concerns/accountable.rb +++ b/app/models/concerns/accountable.rb @@ -68,15 +68,6 @@ module Accountable end end - def post_sync(sync) - broadcast_replace_to( - account, - target: "chart_account_#{account.id}", - partial: "accounts/show/chart", - locals: { account: account } - ) - end - def display_name self.class.display_name end diff --git a/app/models/concerns/enrichable.rb b/app/models/concerns/enrichable.rb index e5804786..4c373b01 100644 --- a/app/models/concerns/enrichable.rb +++ b/app/models/concerns/enrichable.rb @@ -22,16 +22,23 @@ module Enrichable } end - def log_enrichment!(attribute_name:, attribute_value:, source:, metadata: {}) - de = DataEnrichment.find_or_create_by!( - enrichable: self, - attribute_name: attribute_name, - source: source, - ) + # Convenience method for a single attribute + def enrich_attribute(attr, value, source:, metadata: {}) + enrich_attributes({ attr => value }, source:, metadata:) + end - de.value = attribute_value - de.metadata = metadata - de.save! + # Enriches all attributes that haven't been locked yet + def enrich_attributes(attrs, source:, metadata: {}) + enrichable_attrs = Array(attrs).reject { |k, _v| locked?(k) } + + ActiveRecord::Base.transaction do + enrichable_attrs.each do |attr, value| + self.send("#{attr}=", value) + log_enrichment(attribute_name: attr, attribute_value: value, source: source, metadata: metadata) + end + + save + end end def locked?(attr) @@ -42,21 +49,33 @@ module Enrichable !locked?(attr) end - def lock!(attr) + def lock_attr!(attr) update!(locked_attributes: locked_attributes.merge(attr.to_s => Time.current)) end - def unlock!(attr) + def unlock_attr!(attr) update!(locked_attributes: locked_attributes.except(attr.to_s)) end def lock_saved_attributes! saved_changes.keys.reject { |attr| ignored_enrichable_attributes.include?(attr) }.each do |attr| - lock!(attr) + lock_attr!(attr) end end private + def log_enrichment(attribute_name:, attribute_value:, source:, metadata: {}) + de = DataEnrichment.find_or_create_by( + enrichable: self, + attribute_name: attribute_name, + source: source, + ) + + de.value = attribute_value + de.metadata = metadata + de.save + end + def ignored_enrichable_attributes %w[id updated_at created_at] end diff --git a/app/models/concerns/syncable.rb b/app/models/concerns/syncable.rb index ce3c30fd..72556bf7 100644 --- a/app/models/concerns/syncable.rb +++ b/app/models/concerns/syncable.rb @@ -6,24 +6,44 @@ module Syncable end def syncing? - syncs.where(status: [ :syncing, :pending ]).any? + raise NotImplementedError, "Subclasses must implement the syncing? method" end - def sync_later(start_date: nil, parent_sync: nil) - new_sync = syncs.create!(start_date: start_date, parent: parent_sync) - SyncJob.perform_later(new_sync) + # Schedules a sync for syncable. If there is an existing sync pending/syncing for this syncable, + # we do not create a new sync, and attempt to expand the sync window if needed. + def sync_later(parent_sync: nil, window_start_date: nil, window_end_date: nil) + Sync.transaction do + with_lock do + sync = self.syncs.incomplete.first + + if sync + Rails.logger.info("There is an existing sync, expanding window if needed (#{sync.id})") + sync.expand_window_if_needed(window_start_date, window_end_date) + else + sync = self.syncs.create!( + parent: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date + ) + + SyncJob.perform_later(sync) + end + + sync + end + end end - def sync(start_date: nil) - syncs.create!(start_date: start_date).perform + def perform_sync(sync) + syncer.perform_sync(sync) end - def sync_data(sync, start_date: nil) - raise NotImplementedError, "Subclasses must implement the `sync_data` method" + def perform_post_sync + syncer.perform_post_sync end - def post_sync(sync) - # no-op, syncable can optionally provide implementation + def broadcast_sync_complete + sync_broadcaster.broadcast end def sync_error @@ -31,7 +51,7 @@ module Syncable end def last_synced_at - latest_sync&.last_ran_at + latest_sync&.completed_at end def last_sync_created_at @@ -40,6 +60,14 @@ module Syncable private def latest_sync - syncs.order(created_at: :desc).first + syncs.ordered.first + end + + def syncer + self.class::Syncer.new(self) + end + + def sync_broadcaster + self.class::SyncCompleteEvent.new(self) end end diff --git a/app/models/entry.rb b/app/models/entry.rb index c07f27cf..5b14987a 100644 --- a/app/models/entry.rb +++ b/app/models/entry.rb @@ -45,7 +45,7 @@ class Entry < ApplicationRecord def sync_account_later sync_start_date = [ date_previously_was, date ].compact.min unless destroyed? - account.sync_later(start_date: sync_start_date) + account.sync_later(window_start_date: sync_start_date) end def entryable_name_short @@ -85,7 +85,7 @@ class Entry < ApplicationRecord entry.update! bulk_attributes entry.lock_saved_attributes! - entry.entryable.lock!(:tag_ids) if entry.transaction? && entry.transaction.tags.any? + entry.entryable.lock_attr!(:tag_ids) if entry.transaction? && entry.transaction.tags.any? end end diff --git a/app/models/exchange_rate/importer.rb b/app/models/exchange_rate/importer.rb new file mode 100644 index 00000000..0975f2ed --- /dev/null +++ b/app/models/exchange_rate/importer.rb @@ -0,0 +1,156 @@ +class ExchangeRate::Importer + MissingExchangeRateError = Class.new(StandardError) + MissingStartRateError = Class.new(StandardError) + + def initialize(exchange_rate_provider:, from:, to:, start_date:, end_date:, clear_cache: false) + @exchange_rate_provider = exchange_rate_provider + @from = from + @to = to + @start_date = start_date + @end_date = normalize_end_date(end_date) + @clear_cache = clear_cache + end + + # Constructs a daily series of rates for the given currency pair for date range + def import_provider_rates + if !clear_cache && all_rates_exist? + Rails.logger.info("No new rates to sync for #{from} to #{to} between #{start_date} and #{end_date}, skipping") + return + end + + if provider_rates.empty? + Rails.logger.warn("Could not fetch rates for #{from} to #{to} between #{start_date} and #{end_date} because provider returned no rates") + return + end + + prev_rate_value = start_rate_value + + unless prev_rate_value.present? + error = MissingStartRateError.new("Could not find a start rate for #{from} to #{to} between #{start_date} and #{end_date}") + Rails.logger.error(error.message) + Sentry.capture_exception(error) + return + end + + gapfilled_rates = effective_start_date.upto(end_date).map do |date| + db_rate_value = db_rates[date]&.rate + provider_rate_value = provider_rates[date]&.rate + + chosen_rate = if clear_cache + provider_rate_value || db_rate_value # overwrite when possible + else + db_rate_value || provider_rate_value # fill gaps + end + + # Gapfill with LOCF strategy (last observation carried forward) + if chosen_rate.nil? + chosen_rate = prev_rate_value + end + + prev_rate_value = chosen_rate + + { + from_currency: from, + to_currency: to, + date: date, + rate: chosen_rate + } + end + + upsert_rows(gapfilled_rates) + end + + private + attr_reader :exchange_rate_provider, :from, :to, :start_date, :end_date, :clear_cache + + def upsert_rows(rows) + batch_size = 200 + + total_upsert_count = 0 + + rows.each_slice(batch_size) do |batch| + upserted_ids = ExchangeRate.upsert_all( + batch, + unique_by: %i[from_currency to_currency date], + returning: [ "id" ] + ) + + total_upsert_count += upserted_ids.count + end + + total_upsert_count + end + + # Since provider may not return values on weekends and holidays, we grab the first rate from the provider that is on or before the start date + def start_rate_value + provider_rate_value = provider_rates.select { |date, _| date <= start_date }.max_by { |date, _| date }&.last + db_rate_value = db_rates[start_date]&.rate + provider_rate_value || db_rate_value + end + + # No need to fetch/upsert rates for dates that we already have in the DB + def effective_start_date + return start_date if clear_cache + + first_missing_date = nil + + start_date.upto(end_date) do |date| + unless db_rates.key?(date) + first_missing_date = date + break + end + end + + first_missing_date || end_date + end + + def provider_rates + @provider_rates ||= begin + # Always fetch with a 5 day buffer to ensure we have a starting rate (for weekends and holidays) + provider_fetch_start_date = effective_start_date - 5.days + + provider_response = exchange_rate_provider.fetch_exchange_rates( + from: from, + to: to, + start_date: provider_fetch_start_date, + end_date: end_date + ) + + if provider_response.success? + provider_response.data.index_by(&:date) + else + message = "#{exchange_rate_provider.class.name} could not fetch exchange rate pair from: #{from} to: #{to} between: #{effective_start_date} and: #{Date.current}. Provider error: #{provider_response.error.message}" + Rails.logger.warn(message) + Sentry.capture_exception(MissingExchangeRateError.new(message), level: :warning) + {} + end + end + end + + def all_rates_exist? + db_count == expected_count + end + + def expected_count + (start_date..end_date).count + end + + def db_count + db_rates.count + end + + def db_rates + @db_rates ||= ExchangeRate.where(from_currency: from, to_currency: to, date: start_date..end_date) + .order(:date) + .to_a + .index_by(&:date) + end + + # Normalizes an end date so that it never exceeds today's date in the + # America/New_York timezone. If the caller passes a future date we clamp + # it to today so that upstream provider calls remain valid and predictable. + def normalize_end_date(requested_end_date) + today_est = Date.current.in_time_zone("America/New_York").to_date + [ requested_end_date, today_est ].min + end +end diff --git a/app/models/exchange_rate/provided.rb b/app/models/exchange_rate/provided.rb index dbe87133..defee421 100644 --- a/app/models/exchange_rate/provided.rb +++ b/app/models/exchange_rate/provided.rb @@ -27,29 +27,21 @@ module ExchangeRate::Provided rate end - def sync_provider_rates(from:, to:, start_date:, end_date: Date.current) + # @return [Integer] The number of exchange rates synced + def import_provider_rates(from:, to:, start_date:, end_date:, clear_cache: false) unless provider.present? - Rails.logger.warn("No provider configured for ExchangeRate.sync_provider_rates") + Rails.logger.warn("No provider configured for ExchangeRate.import_provider_rates") return 0 end - fetched_rates = provider.fetch_exchange_rates(from: from, to: to, start_date: start_date, end_date: end_date) - - unless fetched_rates.success? - Rails.logger.error("Provider error for ExchangeRate.sync_provider_rates: #{fetched_rates.error}") - return 0 - end - - rates_data = fetched_rates.data.map do |rate| - { - from_currency: rate.from, - to_currency: rate.to, - date: rate.date, - rate: rate.rate - } - end - - ExchangeRate.upsert_all(rates_data, unique_by: %i[from_currency to_currency date]) + ExchangeRate::Importer.new( + exchange_rate_provider: provider, + from: from, + to: to, + start_date: start_date, + end_date: end_date, + clear_cache: clear_cache + ).import_provider_rates end end end diff --git a/app/models/family.rb b/app/models/family.rb index a3a73eec..cd068cae 100644 --- a/app/models/family.rb +++ b/app/models/family.rb @@ -1,5 +1,5 @@ class Family < ApplicationRecord - include Syncable, AutoTransferMatchable, Subscribeable + include PlaidConnectable, Syncable, AutoTransferMatchable, Subscribeable DATE_FORMATS = [ [ "MM-DD-YYYY", "%m-%d-%Y" ], @@ -15,7 +15,6 @@ class Family < ApplicationRecord has_many :users, dependent: :destroy has_many :accounts, dependent: :destroy - has_many :plaid_items, dependent: :destroy has_many :invitations, dependent: :destroy has_many :imports, dependent: :destroy @@ -36,6 +35,15 @@ class Family < ApplicationRecord validates :locale, inclusion: { in: I18n.available_locales.map(&:to_s) } validates :date_format, inclusion: { in: DATE_FORMATS.map(&:last) } + # If any accounts or plaid items are syncing, the family is also syncing, even if a formal "Family Sync" is not running. + def syncing? + Sync.joins("LEFT JOIN plaid_items ON plaid_items.id = syncs.syncable_id AND syncs.syncable_type = 'PlaidItem'") + .joins("LEFT JOIN accounts ON accounts.id = syncs.syncable_id AND syncs.syncable_type = 'Account'") + .where("syncs.syncable_id = ? OR accounts.family_id = ? OR plaid_items.family_id = ?", id, id, id) + .visible + .exists? + end + def assigned_merchants merchant_ids = transactions.where.not(merchant_id: nil).pluck(:merchant_id).uniq Merchant.where(id: merchant_ids) @@ -65,64 +73,10 @@ class Family < ApplicationRecord @income_statement ||= IncomeStatement.new(self) end - def sync_data(sync, start_date: nil) - # We don't rely on this value to guard the app, but keep it eventually consistent - sync_trial_status! - - Rails.logger.info("Syncing accounts for family #{id}") - accounts.manual.each do |account| - account.sync_later(start_date: start_date, parent_sync: sync) - end - - Rails.logger.info("Applying rules for family #{id}") - rules.each do |rule| - rule.apply_later - end - end - - def remove_syncing_notice! - broadcast_remove target: "syncing-notice" - end - - def post_sync(sync) - auto_match_transfers! - broadcast_refresh - end - - # If family has any syncs pending/syncing within the last 10 minutes, we show a persistent "syncing" notice. - # Ignore syncs older than 10 minutes as they are considered "stale" - def syncing? - Sync.where( - "(syncable_type = 'Family' AND syncable_id = ?) OR - (syncable_type = 'Account' AND syncable_id IN (SELECT id FROM accounts WHERE family_id = ? AND plaid_account_id IS NULL)) OR - (syncable_type = 'PlaidItem' AND syncable_id IN (SELECT id FROM plaid_items WHERE family_id = ?))", - id, id, id - ).where(status: [ "pending", "syncing" ], created_at: 10.minutes.ago..).exists? - end - def eu? country != "US" && country != "CA" end - def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil) - provider = if region.to_sym == :eu - Provider::Registry.get_provider(:plaid_eu) - else - Provider::Registry.get_provider(:plaid_us) - end - - # early return when no provider - return nil unless provider - - provider.get_link_token( - user_id: id, - webhooks_url: webhooks_url, - redirect_url: redirect_url, - accountable_type: accountable_type, - access_token: access_token - ).link_token - end - def requires_data_provider? # If family has any trades, they need a provider for historical prices return true if trades.any? diff --git a/app/models/family/auto_categorizer.rb b/app/models/family/auto_categorizer.rb index c35aa3b9..25fde493 100644 --- a/app/models/family/auto_categorizer.rb +++ b/app/models/family/auto_categorizer.rb @@ -27,23 +27,19 @@ class Family::AutoCategorizer end scope.each do |transaction| - transaction.lock!(:category_id) - auto_categorization = result.data.find { |c| c.transaction_id == transaction.id } category_id = user_categories_input.find { |c| c[:name] == auto_categorization&.category_name }&.dig(:id) if category_id.present? - Family.transaction do - transaction.log_enrichment!( - attribute_name: "category_id", - attribute_value: category_id, - source: "ai", - ) - - transaction.update!(category_id: category_id) - end + transaction.enrich_attribute( + :category_id, + category_id, + source: "ai" + ) end + + transaction.lock_attr!(:category_id) end end diff --git a/app/models/family/auto_merchant_detector.rb b/app/models/family/auto_merchant_detector.rb index 4b791e7a..39e58a3a 100644 --- a/app/models/family/auto_merchant_detector.rb +++ b/app/models/family/auto_merchant_detector.rb @@ -27,8 +27,6 @@ class Family::AutoMerchantDetector end scope.each do |transaction| - transaction.lock!(:merchant_id) - auto_detection = result.data.find { |c| c.transaction_id == transaction.id } merchant_id = user_merchants_input.find { |m| m[:name] == auto_detection&.business_name }&.dig(:id) @@ -46,16 +44,16 @@ class Family::AutoMerchantDetector merchant_id = merchant_id || ai_provider_merchant&.id if merchant_id.present? - Family.transaction do - transaction.log_enrichment!( - attribute_name: "merchant_id", - attribute_value: merchant_id, - source: "ai", - ) + transaction.enrich_attribute( + :merchant_id, + merchant_id, + source: "ai" + ) - transaction.update!(merchant_id: merchant_id) - end end + + # We lock the attribute so that this Rule doesn't try to run again + transaction.lock_attr!(:merchant_id) end end diff --git a/app/models/family/plaid_connectable.rb b/app/models/family/plaid_connectable.rb new file mode 100644 index 00000000..f2a997c8 --- /dev/null +++ b/app/models/family/plaid_connectable.rb @@ -0,0 +1,51 @@ +module Family::PlaidConnectable + extend ActiveSupport::Concern + + included do + has_many :plaid_items, dependent: :destroy + end + + def create_plaid_item!(public_token:, item_name:, region:) + provider = plaid_provider_for_region(region) + + public_token_response = provider.exchange_public_token(public_token) + + plaid_item = plaid_items.create!( + name: item_name, + plaid_id: public_token_response.item_id, + access_token: public_token_response.access_token, + plaid_region: region + ) + + plaid_item.sync_later + + plaid_item + end + + def get_link_token(webhooks_url:, redirect_url:, accountable_type: nil, region: :us, access_token: nil) + return nil unless plaid_us || plaid_eu + + provider = plaid_provider_for_region(region) + + provider.get_link_token( + user_id: self.id, + webhooks_url: webhooks_url, + redirect_url: redirect_url, + accountable_type: accountable_type, + access_token: access_token + ).link_token + end + + private + def plaid_us + @plaid ||= Provider::Registry.get_provider(:plaid_us) + end + + def plaid_eu + @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu) + end + + def plaid_provider_for_region(region) + region.to_sym == :eu ? plaid_eu : plaid_us + end +end diff --git a/app/models/family/subscribeable.rb b/app/models/family/subscribeable.rb index 4c8624b7..94ee0971 100644 --- a/app/models/family/subscribeable.rb +++ b/app/models/family/subscribeable.rb @@ -72,10 +72,9 @@ module Family::Subscribeable (1 - days_left_in_trial.to_f / Subscription::TRIAL_DAYS) * 100 end - private - def sync_trial_status! - if subscription&.status == "trialing" && days_left_in_trial < 0 - subscription.update!(status: "paused") - end + def sync_trial_status! + if subscription&.status == "trialing" && days_left_in_trial < 0 + subscription.update!(status: "paused") end + end end diff --git a/app/models/family/sync_complete_event.rb b/app/models/family/sync_complete_event.rb new file mode 100644 index 00000000..628841d0 --- /dev/null +++ b/app/models/family/sync_complete_event.rb @@ -0,0 +1,21 @@ +class Family::SyncCompleteEvent + attr_reader :family + + def initialize(family) + @family = family + end + + def broadcast + family.broadcast_replace( + target: "balance-sheet", + partial: "pages/dashboard/balance_sheet", + locals: { balance_sheet: family.balance_sheet } + ) + + family.broadcast_replace( + target: "net-worth-chart", + partial: "pages/dashboard/net_worth_chart", + locals: { balance_sheet: family.balance_sheet, period: Period.last_30_days } + ) + end +end diff --git a/app/models/family/syncer.rb b/app/models/family/syncer.rb new file mode 100644 index 00000000..30ce2ad5 --- /dev/null +++ b/app/models/family/syncer.rb @@ -0,0 +1,31 @@ +class Family::Syncer + attr_reader :family + + def initialize(family) + @family = family + end + + def perform_sync(sync) + # We don't rely on this value to guard the app, but keep it eventually consistent + family.sync_trial_status! + + Rails.logger.info("Applying rules for family #{family.id}") + family.rules.each do |rule| + rule.apply_later + end + + # Schedule child syncs + child_syncables.each do |syncable| + syncable.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date) + end + end + + def perform_post_sync + family.auto_match_transfers! + end + + private + def child_syncables + family.plaid_items + family.accounts.manual + end +end diff --git a/app/models/holding/base_calculator.rb b/app/models/holding/base_calculator.rb deleted file mode 100644 index d9b85d03..00000000 --- a/app/models/holding/base_calculator.rb +++ /dev/null @@ -1,62 +0,0 @@ -class Holding::BaseCalculator - attr_reader :account - - def initialize(account) - @account = account - end - - def calculate - Rails.logger.tagged(self.class.name) do - holdings = calculate_holdings - Holding.gapfill(holdings) - end - end - - private - def portfolio_cache - @portfolio_cache ||= Holding::PortfolioCache.new(account) - end - - def empty_portfolio - securities = portfolio_cache.get_securities - securities.each_with_object({}) { |security, hash| hash[security.id] = 0 } - end - - def generate_starting_portfolio - empty_portfolio - end - - def transform_portfolio(previous_portfolio, trade_entries, direction: :forward) - new_quantities = previous_portfolio.dup - - trade_entries.each do |trade_entry| - trade = trade_entry.entryable - security_id = trade.security_id - qty_change = trade.qty - qty_change = qty_change * -1 if direction == :reverse - new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change - end - - new_quantities - end - - def build_holdings(portfolio, date, price_source: nil) - portfolio.map do |security_id, qty| - price = portfolio_cache.get_price(security_id, date, source: price_source) - - if price.nil? - next - end - - Holding.new( - account_id: account.id, - security_id: security_id, - date: date, - qty: qty, - price: price.price, - currency: price.currency, - amount: qty * price.price - ) - end.compact - end -end diff --git a/app/models/holding/forward_calculator.rb b/app/models/holding/forward_calculator.rb index d2f2e8d7..43f91f7a 100644 --- a/app/models/holding/forward_calculator.rb +++ b/app/models/holding/forward_calculator.rb @@ -1,10 +1,12 @@ -class Holding::ForwardCalculator < Holding::BaseCalculator - private - def portfolio_cache - @portfolio_cache ||= Holding::PortfolioCache.new(account) - end +class Holding::ForwardCalculator + attr_reader :account - def calculate_holdings + def initialize(account) + @account = account + end + + def calculate + Rails.logger.tagged("Holding::ForwardCalculator") do current_portfolio = generate_starting_portfolio next_portfolio = {} holdings = [] @@ -16,6 +18,55 @@ class Holding::ForwardCalculator < Holding::BaseCalculator current_portfolio = next_portfolio end - holdings + Holding.gapfill(holdings) + end + end + + private + def portfolio_cache + @portfolio_cache ||= Holding::PortfolioCache.new(account) + end + + def empty_portfolio + securities = portfolio_cache.get_securities + securities.each_with_object({}) { |security, hash| hash[security.id] = 0 } + end + + def generate_starting_portfolio + empty_portfolio + end + + def transform_portfolio(previous_portfolio, trade_entries, direction: :forward) + new_quantities = previous_portfolio.dup + + trade_entries.each do |trade_entry| + trade = trade_entry.entryable + security_id = trade.security_id + qty_change = trade.qty + qty_change = qty_change * -1 if direction == :reverse + new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change + end + + new_quantities + end + + def build_holdings(portfolio, date, price_source: nil) + portfolio.map do |security_id, qty| + price = portfolio_cache.get_price(security_id, date, source: price_source) + + if price.nil? + next + end + + Holding.new( + account_id: account.id, + security_id: security_id, + date: date, + qty: qty, + price: price.price, + currency: price.currency, + amount: qty * price.price + ) + end.compact end end diff --git a/app/models/holding/syncer.rb b/app/models/holding/materializer.rb similarity index 87% rename from app/models/holding/syncer.rb rename to app/models/holding/materializer.rb index 345f2a3f..e4931128 100644 --- a/app/models/holding/syncer.rb +++ b/app/models/holding/materializer.rb @@ -1,10 +1,12 @@ -class Holding::Syncer +# "Materializes" holdings (similar to a DB materialized view, but done at the app level) +# into a series of records we can easily query and join with other data. +class Holding::Materializer def initialize(account, strategy:) @account = account @strategy = strategy end - def sync_holdings + def materialize_holdings calculate_holdings Rails.logger.info("Persisting #{@holdings.size} holdings") diff --git a/app/models/holding/portfolio_cache.rb b/app/models/holding/portfolio_cache.rb index 2d67a1d8..9ffed15b 100644 --- a/app/models/holding/portfolio_cache.rb +++ b/app/models/holding/portfolio_cache.rb @@ -83,9 +83,6 @@ class Holding::PortfolioCache securities.each do |security| Rails.logger.info "Loading security: ID=#{security.id} Ticker=#{security.ticker}" - # Load prices from provider to DB - security.sync_provider_prices(start_date: account.start_date) - # High priority prices from DB (synced from provider) db_prices = security.prices.where(date: account.start_date..Date.current).map do |price| PriceWithPriority.new( diff --git a/app/models/holding/reverse_calculator.rb b/app/models/holding/reverse_calculator.rb index 62e2dc95..f52184d7 100644 --- a/app/models/holding/reverse_calculator.rb +++ b/app/models/holding/reverse_calculator.rb @@ -1,4 +1,17 @@ -class Holding::ReverseCalculator < Holding::BaseCalculator +class Holding::ReverseCalculator + attr_reader :account + + def initialize(account) + @account = account + end + + def calculate + Rails.logger.tagged("Holding::ReverseCalculator") do + holdings = calculate_holdings + Holding.gapfill(holdings) + end + end + private # Reverse calculators will use the existing holdings as a source of security ids and prices # since it is common for a provider to supply "current day" holdings but not all the historical @@ -25,6 +38,11 @@ class Holding::ReverseCalculator < Holding::BaseCalculator holdings end + def empty_portfolio + securities = portfolio_cache.get_securities + securities.each_with_object({}) { |security, hash| hash[security.id] = 0 } + end + # Since this is a reverse sync, we start with today's holdings def generate_starting_portfolio holding_quantities = empty_portfolio @@ -37,4 +55,38 @@ class Holding::ReverseCalculator < Holding::BaseCalculator holding_quantities end + + def transform_portfolio(previous_portfolio, trade_entries, direction: :forward) + new_quantities = previous_portfolio.dup + + trade_entries.each do |trade_entry| + trade = trade_entry.entryable + security_id = trade.security_id + qty_change = trade.qty + qty_change = qty_change * -1 if direction == :reverse + new_quantities[security_id] = (new_quantities[security_id] || 0) + qty_change + end + + new_quantities + end + + def build_holdings(portfolio, date, price_source: nil) + portfolio.map do |security_id, qty| + price = portfolio_cache.get_price(security_id, date, source: price_source) + + if price.nil? + next + end + + Holding.new( + account_id: account.id, + security_id: security_id, + date: date, + qty: qty, + price: price.price, + currency: price.currency, + amount: qty * price.price + ) + end.compact + end end diff --git a/app/models/import.rb b/app/models/import.rb index 3ea68015..b0a02ea0 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -1,4 +1,6 @@ class Import < ApplicationRecord + MaxRowCountExceededError = Class.new(StandardError) + TYPES = %w[TransactionImport TradeImport AccountImport MintImport].freeze SIGNAGE_CONVENTIONS = %w[inflows_positive inflows_negative] SEPARATORS = [ [ "Comma (,)", "," ], [ "Semicolon (;)", ";" ] ].freeze @@ -52,6 +54,7 @@ class Import < ApplicationRecord end def publish_later + raise MaxRowCountExceededError if row_count_exceeded? raise "Import is not publishable" unless publishable? update! status: :importing @@ -60,9 +63,11 @@ class Import < ApplicationRecord end def publish + raise MaxRowCountExceededError if row_count_exceeded? + import! - family.sync + family.sync_later update! status: :complete rescue => error @@ -220,7 +225,15 @@ class Import < ApplicationRecord ) end + def max_row_count + 10000 + end + private + def row_count_exceeded? + rows.count > max_row_count + end + def import! # no-op, subclasses can implement for customization of algorithm end diff --git a/app/models/market_data_importer.rb b/app/models/market_data_importer.rb new file mode 100644 index 00000000..9eaf5964 --- /dev/null +++ b/app/models/market_data_importer.rb @@ -0,0 +1,132 @@ +class MarketDataImporter + # By default, our graphs show 1M as the view, so by fetching 31 days, + # we ensure we can always show an accurate default graph + SNAPSHOT_DAYS = 31 + + InvalidModeError = Class.new(StandardError) + + def initialize(mode: :full, clear_cache: false) + @mode = set_mode!(mode) + @clear_cache = clear_cache + end + + def import_all + import_security_prices + import_exchange_rates + end + + # Syncs historical security prices (and details) + def import_security_prices + unless Security.provider + Rails.logger.warn("No provider configured for MarketDataImporter.import_security_prices, skipping sync") + return + end + + Security.where.not(exchange_operating_mic: nil).find_each do |security| + security.import_provider_prices( + start_date: get_first_required_price_date(security), + end_date: end_date, + clear_cache: clear_cache + ) + + security.import_provider_details(clear_cache: clear_cache) + end + end + + def import_exchange_rates + unless ExchangeRate.provider + Rails.logger.warn("No provider configured for MarketDataImporter.import_exchange_rates, skipping sync") + return + end + + required_exchange_rate_pairs.each do |pair| + # pair is a Hash with keys :source, :target, and :start_date + start_date = snapshot? ? default_start_date : pair[:start_date] + + ExchangeRate.import_provider_rates( + from: pair[:source], + to: pair[:target], + start_date: start_date, + end_date: end_date, + clear_cache: clear_cache + ) + end + end + + private + attr_reader :mode, :clear_cache + + def snapshot? + mode.to_sym == :snapshot + end + + # Builds a unique list of currency pairs with the earliest date we need + # exchange rates for. + # + # Returns: Array of Hashes – [{ source:, target:, start_date: }, ...] + def required_exchange_rate_pairs + pair_dates = {} # { [source, target] => earliest_date } + + # 1. ENTRY-BASED PAIRS – we need rates from the first entry date + Entry.joins(:account) + .where.not("entries.currency = accounts.currency") + .group("entries.currency", "accounts.currency") + .minimum("entries.date") + .each do |(source, target), date| + key = [ source, target ] + pair_dates[key] = [ pair_dates[key], date ].compact.min + end + + # 2. ACCOUNT-BASED PAIRS – use the account's oldest entry date + account_first_entry_dates = Entry.group(:account_id).minimum(:date) + + Account.joins(:family) + .where.not("families.currency = accounts.currency") + .select("accounts.id, accounts.currency AS source, families.currency AS target") + .find_each do |account| + earliest_entry_date = account_first_entry_dates[account.id] + + chosen_date = [ earliest_entry_date, default_start_date ].compact.min + + key = [ account.source, account.target ] + pair_dates[key] = [ pair_dates[key], chosen_date ].compact.min + end + + # Convert to array of hashes for ease of use + pair_dates.map do |(source, target), date| + { source: source, target: target, start_date: date } + end + end + + def get_first_required_price_date(security) + return default_start_date if snapshot? + + Trade.with_entry.where(security: security).minimum(:date) || default_start_date + end + + # An approximation that grabs more than we likely need, but simplifies the logic + def get_first_required_exchange_rate_date(from_currency:) + return default_start_date if snapshot? + + Entry.where(currency: from_currency).minimum(:date) || default_start_date + end + + def default_start_date + SNAPSHOT_DAYS.days.ago.to_date + end + + # Since we're querying market data from a US-based API, end date should always be today (EST) + def end_date + Date.current.in_time_zone("America/New_York").to_date + end + + def set_mode!(mode) + valid_modes = [ :full, :snapshot ] + + unless valid_modes.include?(mode.to_sym) + raise InvalidModeError, "Invalid mode for MarketDataImporter, can only be :full or :snapshot, but was #{mode}" + end + + mode.to_sym + end +end diff --git a/app/models/plaid_account.rb b/app/models/plaid_account.rb index 4f60013e..4730985d 100644 --- a/app/models/plaid_account.rb +++ b/app/models/plaid_account.rb @@ -20,7 +20,10 @@ class PlaidAccount < ApplicationRecord internal_account = family.accounts.find_or_initialize_by(plaid_account_id: plaid_account.id) - internal_account.name = plaid_data.name + # Only set the name for new records or if the name is not locked + if internal_account.new_record? || internal_account.enrichable?(:name) + internal_account.name = plaid_data.name + end internal_account.balance = plaid_data.balances.current || plaid_data.balances.available internal_account.currency = plaid_data.balances.iso_currency_code internal_account.accountable = TYPE_MAPPING[plaid_data.type].new diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 2226f12f..e693e69a 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -1,5 +1,5 @@ class PlaidItem < ApplicationRecord - include Provided, Syncable + include Syncable enum :plaid_region, { us: "us", eu: "eu" } enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -22,39 +22,6 @@ class PlaidItem < ApplicationRecord scope :ordered, -> { order(created_at: :desc) } scope :needs_update, -> { where(status: :requires_update) } - class << self - def create_from_public_token(token, item_name:, region:) - response = plaid_provider_for_region(region).exchange_public_token(token) - - new_plaid_item = create!( - name: item_name, - plaid_id: response.item_id, - access_token: response.access_token, - plaid_region: region - ) - - new_plaid_item.sync_later - end - end - - def sync_data(sync, start_date: nil) - begin - Rails.logger.info("Fetching and loading Plaid data") - fetch_and_load_plaid_data(sync) - update!(status: :good) if requires_update? - - # Schedule account syncs - accounts.each do |account| - account.sync_later(start_date: start_date, parent_sync: sync) - end - - Rails.logger.info("Plaid data fetched and loaded") - rescue Plaid::ApiError => e - handle_plaid_error(e) - raise e - end - end - def get_update_link_token(webhooks_url:, redirect_url:) begin family.get_link_token( @@ -76,9 +43,8 @@ class PlaidItem < ApplicationRecord end end - def post_sync(sync) - auto_match_categories! - family.broadcast_refresh + def build_category_alias_matcher(user_categories) + Provider::Plaid::CategoryAliasMatcher.new(user_categories) end def destroy_later @@ -86,6 +52,14 @@ class PlaidItem < ApplicationRecord DestroyJob.perform_later(self) end + def syncing? + Sync.joins("LEFT JOIN accounts a ON a.id = syncs.syncable_id AND syncs.syncable_type = 'Account'") + .joins("LEFT JOIN plaid_accounts pa ON pa.id = a.plaid_account_id") + .where("syncs.syncable_id = ? OR pa.plaid_item_id = ?", id, id) + .visible + .exists? + end + def auto_match_categories! if family.categories.none? family.categories.bootstrap! @@ -103,137 +77,25 @@ class PlaidItem < ApplicationRecord category = alias_matcher.match(transaction.plaid_category_detailed) if category.present? - PlaidItem.transaction do - transaction.log_enrichment!( - attribute_name: "category_id", - attribute_value: category.id, - source: "plaid" - ) - transaction.set_category!(category) + # Matcher could either return a string or a Category object + user_category = if category.is_a?(String) + family.categories.find_or_create_by!(name: category) + else + category end + + transaction.enrich_attribute(:category_id, user_category.id, source: "plaid") end end end end private - def fetch_and_load_plaid_data(sync) - data = {} - - # Log what we're about to fetch - Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)" - - item = plaid_provider.get_item(access_token).item - update!(available_products: item.available_products, billed_products: item.billed_products) - - # Institution details - if item.institution_id.present? - begin - Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}" - institution = plaid_provider.get_institution(item.institution_id) - update!( - institution_id: item.institution_id, - institution_url: institution.institution.url, - institution_color: institution.institution.primary_color - ) - rescue Plaid::ApiError => e - Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}" - end - end - - # Accounts - fetched_accounts = plaid_provider.get_item_accounts(self).accounts - data[:accounts] = fetched_accounts || [] - sync.update!(data: data) - Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})" - - internal_plaid_accounts = fetched_accounts.map do |account| - internal_plaid_account = plaid_accounts.find_or_create_from_plaid_data!(account, family) - internal_plaid_account.sync_account_data!(account) - internal_plaid_account - end - - # Transactions - fetched_transactions = safe_fetch_plaid_data(:get_item_transactions) - data[:transactions] = fetched_transactions || [] - sync.update!(data: data) - - if fetched_transactions - Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id } - modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id } - removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id } - - internal_plaid_account.sync_transactions!(added:, modified:, removed:) - end - - update!(next_cursor: fetched_transactions.cursor) - end - end - - # Investments - fetched_investments = safe_fetch_plaid_data(:get_item_investments) - data[:investments] = fetched_investments || [] - sync.update!(data: data) - - if fetched_investments - Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id } - holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id } - securities = fetched_investments.securities - - internal_plaid_account.sync_investments!(transactions:, holdings:, securities:) - end - end - end - - # Liabilities - fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities) - data[:liabilities] = fetched_liabilities || [] - sync.update!(data: data) - - if fetched_liabilities - Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})" - transaction do - internal_plaid_accounts.each do |internal_plaid_account| - credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id } - mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id } - student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id } - - internal_plaid_account.sync_credit_data!(credit) if credit - internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage - internal_plaid_account.sync_student_loan_data!(student) if student - end - end - end - end - - def safe_fetch_plaid_data(method) - begin - plaid_provider.send(method, self) - rescue Plaid::ApiError => e - Rails.logger.warn("Error fetching #{method} for item #{id}: #{e.message}") - nil - end - end - def remove_plaid_item plaid_provider.remove_item(access_token) rescue StandardError => e Rails.logger.warn("Failed to remove Plaid item #{id}: #{e.message}") end - def handle_plaid_error(error) - error_body = JSON.parse(error.response_body) - - if error_body["error_code"] == "ITEM_LOGIN_REQUIRED" - update!(status: :requires_update) - end - end - class PlaidConnectionLostError < StandardError; end end diff --git a/app/models/plaid_item/provided.rb b/app/models/plaid_item/provided.rb deleted file mode 100644 index f2e8ee8f..00000000 --- a/app/models/plaid_item/provided.rb +++ /dev/null @@ -1,30 +0,0 @@ -module PlaidItem::Provided - extend ActiveSupport::Concern - - class_methods do - def plaid_us_provider - Provider::Registry.get_provider(:plaid_us) - end - - def plaid_eu_provider - Provider::Registry.get_provider(:plaid_eu) - end - - def plaid_provider_for_region(region) - region.to_sym == :eu ? plaid_eu_provider : plaid_us_provider - end - end - - def build_category_alias_matcher(user_categories) - Provider::Plaid::CategoryAliasMatcher.new(user_categories) - end - - private - def eu? - raise "eu? is not implemented for #{self.class.name}" - end - - def plaid_provider - eu? ? self.class.plaid_eu_provider : self.class.plaid_us_provider - end -end diff --git a/app/models/plaid_item/sync_complete_event.rb b/app/models/plaid_item/sync_complete_event.rb new file mode 100644 index 00000000..ca008a0a --- /dev/null +++ b/app/models/plaid_item/sync_complete_event.rb @@ -0,0 +1,22 @@ +class PlaidItem::SyncCompleteEvent + attr_reader :plaid_item + + def initialize(plaid_item) + @plaid_item = plaid_item + end + + def broadcast + plaid_item.accounts.each do |account| + account.broadcast_sync_complete + end + + plaid_item.broadcast_replace_to( + plaid_item.family, + target: "plaid_item_#{plaid_item.id}", + partial: "plaid_items/plaid_item", + locals: { plaid_item: plaid_item } + ) + + plaid_item.family.broadcast_sync_complete + end +end diff --git a/app/models/plaid_item/syncer.rb b/app/models/plaid_item/syncer.rb new file mode 100644 index 00000000..e32b9ed4 --- /dev/null +++ b/app/models/plaid_item/syncer.rb @@ -0,0 +1,149 @@ +class PlaidItem::Syncer + attr_reader :plaid_item + + def initialize(plaid_item) + @plaid_item = plaid_item + end + + def perform_sync(sync) + begin + Rails.logger.info("Fetching and loading Plaid data") + fetch_and_load_plaid_data + plaid_item.update!(status: :good) if plaid_item.requires_update? + + plaid_item.accounts.each do |account| + account.sync_later(parent_sync: sync, window_start_date: sync.window_start_date, window_end_date: sync.window_end_date) + end + + Rails.logger.info("Plaid data fetched and loaded") + rescue Plaid::ApiError => e + handle_plaid_error(e) + raise e + end + end + + def perform_post_sync + plaid_item.auto_match_categories! + end + + private + def plaid + plaid_item.plaid_region == "eu" ? plaid_eu : plaid_us + end + + def plaid_eu + @plaid_eu ||= Provider::Registry.get_provider(:plaid_eu) + end + + def plaid_us + @plaid_us ||= Provider::Registry.get_provider(:plaid_us) + end + + def safe_fetch_plaid_data(method) + begin + plaid.send(method, plaid_item) + rescue Plaid::ApiError => e + Rails.logger.warn("Error fetching #{method} for item #{plaid_item.id}: #{e.message}") + nil + end + end + + def handle_plaid_error(error) + error_body = JSON.parse(error.response_body) + + if error_body["error_code"] == "ITEM_LOGIN_REQUIRED" + plaid_item.update!(status: :requires_update) + end + end + + def fetch_and_load_plaid_data + data = {} + + # Log what we're about to fetch + Rails.logger.info "Starting Plaid data fetch (accounts, transactions, investments, liabilities)" + + item = plaid.get_item(plaid_item.access_token).item + plaid_item.update!(available_products: item.available_products, billed_products: item.billed_products) + + # Institution details + if item.institution_id.present? + begin + Rails.logger.info "Fetching Plaid institution details for #{item.institution_id}" + institution = plaid.get_institution(item.institution_id) + plaid_item.update!( + institution_id: item.institution_id, + institution_url: institution.institution.url, + institution_color: institution.institution.primary_color + ) + rescue Plaid::ApiError => e + Rails.logger.warn "Failed to fetch Plaid institution details: #{e.message}" + end + end + + # Accounts + fetched_accounts = plaid.get_item_accounts(plaid_item).accounts + data[:accounts] = fetched_accounts || [] + Rails.logger.info "Processing Plaid accounts (count: #{fetched_accounts.size})" + + internal_plaid_accounts = fetched_accounts.map do |account| + internal_plaid_account = plaid_item.plaid_accounts.find_or_create_from_plaid_data!(account, plaid_item.family) + internal_plaid_account.sync_account_data!(account) + internal_plaid_account + end + + # Transactions + fetched_transactions = safe_fetch_plaid_data(:get_item_transactions) + data[:transactions] = fetched_transactions || [] + + if fetched_transactions + Rails.logger.info "Processing Plaid transactions (added: #{fetched_transactions.added.size}, modified: #{fetched_transactions.modified.size}, removed: #{fetched_transactions.removed.size})" + PlaidItem.transaction do + internal_plaid_accounts.each do |internal_plaid_account| + added = fetched_transactions.added.select { |t| t.account_id == internal_plaid_account.plaid_id } + modified = fetched_transactions.modified.select { |t| t.account_id == internal_plaid_account.plaid_id } + removed = fetched_transactions.removed.select { |t| t.account_id == internal_plaid_account.plaid_id } + + internal_plaid_account.sync_transactions!(added:, modified:, removed:) + end + + plaid_item.update!(next_cursor: fetched_transactions.cursor) + end + end + + # Investments + fetched_investments = safe_fetch_plaid_data(:get_item_investments) + data[:investments] = fetched_investments || [] + + if fetched_investments + Rails.logger.info "Processing Plaid investments (transactions: #{fetched_investments.transactions.size}, holdings: #{fetched_investments.holdings.size}, securities: #{fetched_investments.securities.size})" + PlaidItem.transaction do + internal_plaid_accounts.each do |internal_plaid_account| + transactions = fetched_investments.transactions.select { |t| t.account_id == internal_plaid_account.plaid_id } + holdings = fetched_investments.holdings.select { |h| h.account_id == internal_plaid_account.plaid_id } + securities = fetched_investments.securities + + internal_plaid_account.sync_investments!(transactions:, holdings:, securities:) + end + end + end + + # Liabilities + fetched_liabilities = safe_fetch_plaid_data(:get_item_liabilities) + data[:liabilities] = fetched_liabilities || [] + + if fetched_liabilities + Rails.logger.info "Processing Plaid liabilities (credit: #{fetched_liabilities.credit&.size || 0}, mortgage: #{fetched_liabilities.mortgage&.size || 0}, student: #{fetched_liabilities.student&.size || 0})" + PlaidItem.transaction do + internal_plaid_accounts.each do |internal_plaid_account| + credit = fetched_liabilities.credit&.find { |l| l.account_id == internal_plaid_account.plaid_id } + mortgage = fetched_liabilities.mortgage&.find { |l| l.account_id == internal_plaid_account.plaid_id } + student = fetched_liabilities.student&.find { |l| l.account_id == internal_plaid_account.plaid_id } + + internal_plaid_account.sync_credit_data!(credit) if credit + internal_plaid_account.sync_mortgage_data!(mortgage) if mortgage + internal_plaid_account.sync_student_loan_data!(student) if student + end + end + end + end +end diff --git a/app/models/provider.rb b/app/models/provider.rb index c90866e7..e9702349 100644 --- a/app/models/provider.rb +++ b/app/models/provider.rb @@ -36,8 +36,6 @@ class Provider default_error_transformer(error) end - Sentry.capture_exception(transformed_error) - Response.new( success?: false, data: nil, diff --git a/app/models/provider/security_concept.rb b/app/models/provider/security_concept.rb index 1fc915e7..d54b2011 100644 --- a/app/models/provider/security_concept.rb +++ b/app/models/provider/security_concept.rb @@ -2,22 +2,22 @@ module Provider::SecurityConcept extend ActiveSupport::Concern Security = Data.define(:symbol, :name, :logo_url, :exchange_operating_mic) - SecurityInfo = Data.define(:symbol, :name, :links, :logo_url, :description, :kind) - Price = Data.define(:security, :date, :price, :currency) + SecurityInfo = Data.define(:symbol, :name, :links, :logo_url, :description, :kind, :exchange_operating_mic) + Price = Data.define(:symbol, :date, :price, :currency, :exchange_operating_mic) def search_securities(symbol, country_code: nil, exchange_operating_mic: nil) raise NotImplementedError, "Subclasses must implement #search_securities" end - def fetch_security_info(security) + def fetch_security_info(symbol:, exchange_operating_mic:) raise NotImplementedError, "Subclasses must implement #fetch_security_info" end - def fetch_security_price(security, date:) + def fetch_security_price(symbol:, exchange_operating_mic:, date:) raise NotImplementedError, "Subclasses must implement #fetch_security_price" end - def fetch_security_prices(security, start_date:, end_date:) + def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:) raise NotImplementedError, "Subclasses must implement #fetch_security_prices" end end diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb index ff75ff49..8d76fc72 100644 --- a/app/models/provider/synth.rb +++ b/app/models/provider/synth.rb @@ -3,6 +3,8 @@ class Provider::Synth < Provider # Subclass so errors caught in this provider are raised as Provider::Synth::Error Error = Class.new(Provider::Error) + InvalidExchangeRateError = Class.new(Error) + InvalidSecurityPriceError = Class.new(Error) def initialize(api_key) @api_key = api_key @@ -48,7 +50,7 @@ class Provider::Synth < Provider rates = JSON.parse(response.body).dig("data", "rates") - Rate.new(date:, from:, to:, rate: rates.dig(to)) + Rate.new(date: date.to_date, from:, to:, rate: rates.dig(to)) end end @@ -65,8 +67,20 @@ class Provider::Synth < Provider end data.paginated.map do |rate| - Rate.new(date: rate.dig("date"), from:, to:, rate: rate.dig("rates", to)) - end + date = rate.dig("date") + rate = rate.dig("rates", to) + + if date.nil? || rate.nil? + Rails.logger.warn("#{self.class.name} returned invalid rate data for pair from: #{from} to: #{to} on: #{date}. Rate data: #{rate.inspect}") + Sentry.capture_exception(InvalidExchangeRateError.new("#{self.class.name} returned invalid rate data"), level: :warning) do |scope| + scope.set_context("rate", { from: from, to: to, date: date }) + end + + next + end + + Rate.new(date: date.to_date, from:, to:, rate:) + end.compact end end @@ -97,65 +111,75 @@ class Provider::Synth < Provider end end - def fetch_security_info(security) + def fetch_security_info(symbol:, exchange_operating_mic:) with_provider_response do - response = client.get("#{base_url}/tickers/#{security.ticker}") do |req| - req.params["mic_code"] = security.exchange_mic if security.exchange_mic.present? - req.params["operating_mic"] = security.exchange_operating_mic if security.exchange_operating_mic.present? + response = client.get("#{base_url}/tickers/#{symbol}") do |req| + req.params["operating_mic"] = exchange_operating_mic end data = JSON.parse(response.body).dig("data") SecurityInfo.new( - symbol: data.dig("ticker"), + symbol: symbol, name: data.dig("name"), links: data.dig("links"), logo_url: data.dig("logo_url"), description: data.dig("description"), - kind: data.dig("kind") + kind: data.dig("kind"), + exchange_operating_mic: exchange_operating_mic ) end end - def fetch_security_price(security, date:) + def fetch_security_price(symbol:, exchange_operating_mic:, date:) with_provider_response do - historical_data = fetch_security_prices(security, start_date: date, end_date: date) + historical_data = fetch_security_prices(symbol:, exchange_operating_mic:, start_date: date, end_date: date) - raise ProviderError, "No prices found for security #{security.ticker} on date #{date}" if historical_data.data.empty? + raise ProviderError, "No prices found for security #{symbol} on date #{date}" if historical_data.data.empty? historical_data.data.first end end - def fetch_security_prices(security, start_date:, end_date:) + def fetch_security_prices(symbol:, exchange_operating_mic:, start_date:, end_date:) with_provider_response do params = { start_date: start_date, - end_date: end_date + end_date: end_date, + operating_mic_code: exchange_operating_mic } - params[:operating_mic_code] = security.exchange_operating_mic if security.exchange_operating_mic.present? - data = paginate( - "#{base_url}/tickers/#{security.ticker}/open-close", + "#{base_url}/tickers/#{symbol}/open-close", params ) do |body| body.dig("prices") end currency = data.first_page.dig("currency") - country_code = data.first_page.dig("exchange", "country_code") - exchange_mic = data.first_page.dig("exchange", "mic_code") exchange_operating_mic = data.first_page.dig("exchange", "operating_mic_code") data.paginated.map do |price| + date = price.dig("date") + price = price.dig("close") || price.dig("open") + + if date.nil? || price.nil? + Rails.logger.warn("#{self.class.name} returned invalid price data for security #{symbol} on: #{date}. Price data: #{price.inspect}") + Sentry.capture_exception(InvalidSecurityPriceError.new("#{self.class.name} returned invalid security price data"), level: :warning) do |scope| + scope.set_context("security", { symbol: symbol, date: date }) + end + + next + end + Price.new( - security: security, - date: price.dig("date"), - price: price.dig("close") || price.dig("open"), - currency: currency + symbol: symbol, + date: date.to_date, + price: price, + currency: currency, + exchange_operating_mic: exchange_operating_mic ) - end + end.compact end end diff --git a/app/models/rule/action_executor/set_transaction_category.rb b/app/models/rule/action_executor/set_transaction_category.rb index ef186d96..6360e45a 100644 --- a/app/models/rule/action_executor/set_transaction_category.rb +++ b/app/models/rule/action_executor/set_transaction_category.rb @@ -17,15 +17,11 @@ class Rule::ActionExecutor::SetTransactionCategory < Rule::ActionExecutor end scope.each do |txn| - Rule.transaction do - txn.log_enrichment!( - attribute_name: "category_id", - attribute_value: category.id, - source: "rule" - ) - - txn.update!(category: category) - end + txn.enrich_attribute( + :category_id, + category.id, + source: "rule" + ) end end end diff --git a/app/models/rule/action_executor/set_transaction_merchant.rb b/app/models/rule/action_executor/set_transaction_merchant.rb index 492ece52..f343a79f 100644 --- a/app/models/rule/action_executor/set_transaction_merchant.rb +++ b/app/models/rule/action_executor/set_transaction_merchant.rb @@ -17,14 +17,11 @@ class Rule::ActionExecutor::SetTransactionMerchant < Rule::ActionExecutor end scope.each do |txn| - Rule.transaction do - txn.log_enrichment!( - attribute_name: "merchant_id", - attribute_value: merchant.id, - source: "rule" - ) - txn.update!(merchant: merchant) - end + txn.enrich_attribute( + :merchant_id, + merchant.id, + source: "rule" + ) end end end diff --git a/app/models/rule/action_executor/set_transaction_name.rb b/app/models/rule/action_executor/set_transaction_name.rb index 39f3ee26..1dd89fa3 100644 --- a/app/models/rule/action_executor/set_transaction_name.rb +++ b/app/models/rule/action_executor/set_transaction_name.rb @@ -16,14 +16,11 @@ class Rule::ActionExecutor::SetTransactionName < Rule::ActionExecutor end scope.each do |txn| - Rule.transaction do - txn.entry.log_enrichment!( - attribute_name: "name", - attribute_value: value, - source: "rule" - ) - txn.entry.update!(name: value) - end + txn.entry.enrich_attribute( + :name, + value, + source: "rule" + ) end end end diff --git a/app/models/rule/action_executor/set_transaction_tags.rb b/app/models/rule/action_executor/set_transaction_tags.rb index 4d539496..d74029ca 100644 --- a/app/models/rule/action_executor/set_transaction_tags.rb +++ b/app/models/rule/action_executor/set_transaction_tags.rb @@ -17,15 +17,11 @@ class Rule::ActionExecutor::SetTransactionTags < Rule::ActionExecutor end rows = scope.each do |txn| - Rule.transaction do - txn.log_enrichment!( - attribute_name: "tag_ids", - attribute_value: [ tag.id ], - source: "rule" - ) - - txn.update!(tag_ids: [ tag.id ]) - end + txn.enrich_attribute( + :tag_ids, + [ tag.id ], + source: "rule" + ) end end end diff --git a/app/models/security/price/importer.rb b/app/models/security/price/importer.rb new file mode 100644 index 00000000..bcee3762 --- /dev/null +++ b/app/models/security/price/importer.rb @@ -0,0 +1,155 @@ +class Security::Price::Importer + MissingSecurityPriceError = Class.new(StandardError) + MissingStartPriceError = Class.new(StandardError) + + def initialize(security:, security_provider:, start_date:, end_date:, clear_cache: false) + @security = security + @security_provider = security_provider + @start_date = start_date + @end_date = normalize_end_date(end_date) + @clear_cache = clear_cache + end + + # Constructs a daily series of prices for a single security over the date range. + # Returns the number of rows upserted. + def import_provider_prices + if !clear_cache && all_prices_exist? + Rails.logger.info("No new prices to sync for #{security.ticker} between #{start_date} and #{end_date}, skipping") + return 0 + end + + if provider_prices.empty? + Rails.logger.warn("Could not fetch prices for #{security.ticker} between #{start_date} and #{end_date} because provider returned no prices") + return 0 + end + + prev_price_value = start_price_value + + unless prev_price_value.present? + Rails.logger.error("Could not find a start price for #{security.ticker} on or before #{start_date}") + + Sentry.capture_exception(MissingStartPriceError.new("Could not determine start price for ticker")) do |scope| + scope.set_tags(security_id: security.id) + scope.set_context("security", { + id: security.id, + start_date: start_date + }) + end + + return 0 + end + + gapfilled_prices = effective_start_date.upto(end_date).map do |date| + db_price_value = db_prices[date]&.price + provider_price_value = provider_prices[date]&.price + provider_currency = provider_prices[date]&.currency + + chosen_price = if clear_cache + provider_price_value || db_price_value # overwrite when possible + else + db_price_value || provider_price_value # fill gaps + end + + # Gap-fill using LOCF (last observation carried forward) + chosen_price ||= prev_price_value + prev_price_value = chosen_price + + { + security_id: security.id, + date: date, + price: chosen_price, + currency: provider_currency || prev_price_currency || db_price_currency || "USD" + } + end + + upsert_rows(gapfilled_prices) + end + + private + attr_reader :security, :security_provider, :start_date, :end_date, :clear_cache + + def provider_prices + @provider_prices ||= begin + provider_fetch_start_date = effective_start_date - 5.days + + response = security_provider.fetch_security_prices( + symbol: security.ticker, + exchange_operating_mic: security.exchange_operating_mic, + start_date: provider_fetch_start_date, + end_date: end_date + ) + + if response.success? + response.data.index_by(&:date) + else + Rails.logger.warn("#{security_provider.class.name} could not fetch prices for #{security.ticker} between #{provider_fetch_start_date} and #{end_date}. Provider error: #{response.error.message}") + Sentry.capture_exception(MissingSecurityPriceError.new("Could not fetch prices for ticker"), level: :warning) do |scope| + scope.set_tags(security_id: security.id) + scope.set_context("security", { id: security.id, start_date: start_date, end_date: end_date }) + end + + {} + end + end + end + + def db_prices + @db_prices ||= Security::Price.where(security_id: security.id, date: start_date..end_date) + .order(:date) + .to_a + .index_by(&:date) + end + + def all_prices_exist? + db_prices.count == expected_count + end + + def expected_count + (start_date..end_date).count + end + + # Skip over ranges that already exist unless clearing cache + def effective_start_date + return start_date if clear_cache + + (start_date..end_date).detect { |d| !db_prices.key?(d) } || end_date + end + + def start_price_value + provider_price_value = provider_prices.select { |date, _| date <= start_date } + .max_by { |date, _| date } + &.last&.price + db_price_value = db_prices[start_date]&.price + provider_price_value || db_price_value + end + + def upsert_rows(rows) + batch_size = 200 + total_upsert_count = 0 + + rows.each_slice(batch_size) do |batch| + ids = Security::Price.upsert_all( + batch, + unique_by: %i[security_id date currency], + returning: [ "id" ] + ) + total_upsert_count += ids.count + end + + total_upsert_count + end + + def db_price_currency + db_prices.values.first&.currency + end + + def prev_price_currency + @prev_price_currency ||= provider_prices.values.first&.currency + end + + # Clamp to today (EST) so we never call our price API for a future date (our API is in EST/EDT timezone) + def normalize_end_date(requested_end_date) + today_est = Date.current.in_time_zone("America/New_York").to_date + [ requested_end_date, today_est ].min + end +end diff --git a/app/models/security/provided.rb b/app/models/security/provided.rb index b342c9e5..7927d6e6 100644 --- a/app/models/security/provided.rb +++ b/app/models/security/provided.rb @@ -1,6 +1,8 @@ module Security::Provided extend ActiveSupport::Concern + SecurityInfoMissingError = Class.new(StandardError) + class_methods do def provider registry = Provider::Registry.for_concept(:securities) @@ -28,44 +30,6 @@ module Security::Provided end end - def sync_provider_prices(start_date:, end_date: Date.current) - unless has_prices? - Rails.logger.warn("Security id=#{id} ticker=#{ticker} is not known by provider, skipping price sync") - return 0 - end - - unless provider.present? - Rails.logger.warn("No security provider configured, cannot sync prices for id=#{id} ticker=#{ticker}") - return 0 - end - - response = provider.fetch_security_prices(self, start_date: start_date, end_date: end_date) - - unless response.success? - Rails.logger.error("Provider error for sync_provider_prices with id=#{id} ticker=#{ticker}: #{response.error}") - return 0 - end - - fetched_prices = response.data.map do |price| - { - security_id: price.security.id, - date: price.date, - price: price.price, - currency: price.currency - } - end - - valid_prices = fetched_prices.reject do |price| - is_invalid = price[:date].nil? || price[:price].nil? || price[:currency].nil? - if is_invalid - Rails.logger.warn("Invalid price data for security_id=#{id}: Missing required fields in price record: #{price.inspect}") - end - is_invalid - end - - Security::Price.upsert_all(valid_prices, unique_by: %i[security_id date currency]) - end - def find_or_fetch_price(date: Date.current, cache: true) price = prices.find_by(date: date) @@ -87,6 +51,50 @@ module Security::Provided price end + def import_provider_details(clear_cache: false) + unless provider.present? + Rails.logger.warn("No provider configured for Security.import_provider_details") + return + end + + if self.name.present? && self.logo_url.present? && !clear_cache + return + end + + response = provider.fetch_security_info( + symbol: ticker, + exchange_operating_mic: exchange_operating_mic + ) + + if response.success? + update( + name: response.data.name, + logo_url: response.data.logo_url, + ) + else + Rails.logger.warn("Failed to fetch security info for #{ticker} from #{provider.class.name}: #{response.error.message}") + Sentry.capture_exception(SecurityInfoMissingError.new("Failed to get security info"), level: :warning) do |scope| + scope.set_tags(security_id: self.id) + scope.set_context("security", { id: self.id, provider_error: response.error.message }) + end + end + end + + def import_provider_prices(start_date:, end_date:, clear_cache: false) + unless provider.present? + Rails.logger.warn("No provider configured for Security.import_provider_prices") + return 0 + end + + Security::Price::Importer.new( + security: self, + security_provider: provider, + start_date: start_date, + end_date: end_date, + clear_cache: clear_cache + ).import_provider_prices + end + private def provider self.class.provider diff --git a/app/models/session.rb b/app/models/session.rb index ce26938a..66faa0f5 100644 --- a/app/models/session.rb +++ b/app/models/session.rb @@ -9,4 +9,14 @@ class Session < ApplicationRecord self.user_agent = Current.user_agent self.ip_address = Current.ip_address end + + def get_preferred_tab(tab_key) + data.dig("tab_preferences", tab_key) + end + + def set_preferred_tab(tab_key, tab_value) + data["tab_preferences"] ||= {} + data["tab_preferences"][tab_key] = tab_value + save! + end end diff --git a/app/models/stock_exchange.rb b/app/models/stock_exchange.rb deleted file mode 100644 index 93631426..00000000 --- a/app/models/stock_exchange.rb +++ /dev/null @@ -1,3 +0,0 @@ -class StockExchange < ApplicationRecord - scope :in_country, ->(country_code) { where(country_code: country_code) } -end diff --git a/app/models/subscription.rb b/app/models/subscription.rb index 5f96361e..90b5303d 100644 --- a/app/models/subscription.rb +++ b/app/models/subscription.rb @@ -17,6 +17,7 @@ class Subscription < ApplicationRecord validates :stripe_id, presence: true, if: :active? validates :trial_ends_at, presence: true, if: :trialing? + validates :family_id, uniqueness: true class << self def new_trial_ends_at diff --git a/app/models/sync.rb b/app/models/sync.rb index 4d923f12..be4bb8c3 100644 --- a/app/models/sync.rb +++ b/app/models/sync.rb @@ -1,4 +1,13 @@ class Sync < ApplicationRecord + # We run a cron that marks any syncs that have not been resolved in 24 hours as "stale" + # Syncs often become stale when new code is deployed and the worker restarts + STALE_AFTER = 24.hours + + # The max time that a sync will show in the UI (after 5 minutes) + VISIBLE_FOR = 5.minutes + + include AASM + Error = Class.new(StandardError) belongs_to :syncable, polymorphic: true @@ -6,12 +15,44 @@ class Sync < ApplicationRecord belongs_to :parent, class_name: "Sync", optional: true has_many :children, class_name: "Sync", foreign_key: :parent_id, dependent: :destroy - enum :status, { pending: "pending", syncing: "syncing", completed: "completed", failed: "failed" } - scope :ordered, -> { order(created_at: :desc) } + scope :incomplete, -> { where("syncs.status IN (?)", %w[pending syncing]) } + scope :visible, -> { incomplete.where("syncs.created_at > ?", VISIBLE_FOR.ago) } - def child? - parent_id.present? + validate :window_valid + + # Sync state machine + aasm column: :status, timestamps: true do + state :pending, initial: true + state :syncing + state :completed + state :failed + state :stale + + after_all_transitions :log_status_change + + event :start, after_commit: :report_warnings do + transitions from: :pending, to: :syncing + end + + event :complete do + transitions from: :syncing, to: :completed + end + + event :fail do + transitions from: :syncing, to: :failed + end + + # Marks a sync that never completed within the expected time window + event :mark_stale do + transitions from: %i[pending syncing], to: :stale + end + end + + class << self + def clean + incomplete.where("syncs.created_at < ?", STALE_AFTER.ago).find_each(&:mark_stale!) + end end def perform @@ -19,43 +60,106 @@ class Sync < ApplicationRecord start! begin - syncable.sync_data(self, start_date: start_date) - - complete! - Rails.logger.info("Sync completed, starting post-sync") - syncable.post_sync(self) - Rails.logger.info("Post-sync completed") - rescue StandardError => error - fail! error, report_error: true + syncable.perform_sync(self) + rescue => e + fail! + update(error: e.message) + report_error(e) + ensure + finalize_if_all_children_finalized end end end - private - def start! - Rails.logger.info("Starting sync") - update! status: :syncing - end + # Finalizes the current sync AND parent (if it exists) + def finalize_if_all_children_finalized + Sync.transaction do + lock! - def complete! - Rails.logger.info("Sync completed") - update! status: :completed, last_ran_at: Time.current - end + # If this is the "parent" and there are still children running, don't finalize. + return unless all_children_finalized? - def fail!(error, report_error: false) - Rails.logger.error("Sync failed: #{error.message}") - - if report_error - Sentry.capture_exception(error) do |scope| - scope.set_context("sync", { id: id, syncable_type: syncable_type, syncable_id: syncable_id }) - scope.set_tags(sync_id: id) + if syncing? + if has_failed_children? + fail! + else + complete! end end - update!( - status: :failed, - error: error.message, - last_ran_at: Time.current - ) + # If we make it here, the sync is finalized. Run post-sync, regardless of failure/success. + perform_post_sync + end + + # If this sync has a parent, try to finalize it so the child status propagates up the chain. + parent&.finalize_if_all_children_finalized + end + + # If a sync is pending, we can adjust the window if new syncs are created with a wider window. + def expand_window_if_needed(new_window_start_date, new_window_end_date) + return unless pending? + return if self.window_start_date.nil? && self.window_end_date.nil? # already as wide as possible + + earliest_start_date = if self.window_start_date && new_window_start_date + [ self.window_start_date, new_window_start_date ].min + else + nil + end + + latest_end_date = if self.window_end_date && new_window_end_date + [ self.window_end_date, new_window_end_date ].max + else + nil + end + + update( + window_start_date: earliest_start_date, + window_end_date: latest_end_date + ) + end + + private + def log_status_change + Rails.logger.info("changing from #{aasm.from_state} to #{aasm.to_state} (event: #{aasm.current_event})") + end + + def has_failed_children? + children.failed.any? + end + + def all_children_finalized? + children.incomplete.empty? + end + + def perform_post_sync + Rails.logger.info("Performing post-sync for #{syncable_type} (#{syncable.id})") + syncable.perform_post_sync + syncable.broadcast_sync_complete + rescue => e + Rails.logger.error("Error performing post-sync for #{syncable_type} (#{syncable.id}): #{e.message}") + report_error(e) + end + + def report_error(error) + Sentry.capture_exception(error) do |scope| + scope.set_tags(sync_id: id) + end + end + + def report_warnings + todays_sync_count = syncable.syncs.where(created_at: Date.current.all_day).count + + if todays_sync_count > 10 + Sentry.capture_exception( + Error.new("#{syncable_type} (#{syncable.id}) has exceeded 10 syncs today (count: #{todays_sync_count})"), + level: :warning + ) + end + end + + def window_valid + if window_start_date && window_end_date && window_start_date > window_end_date + errors.add(:window_end_date, "must be greater than window_start_date") + end end end diff --git a/app/models/trade_builder.rb b/app/models/trade_builder.rb index 5a2f9df1..cf9800e5 100644 --- a/app/models/trade_builder.rb +++ b/app/models/trade_builder.rb @@ -129,13 +129,9 @@ class TradeBuilder def security ticker_symbol, exchange_operating_mic = ticker.present? ? ticker.split("|") : [ manual_ticker, nil ] - security = Security.find_or_create_by!( + Security.find_or_create_by!( ticker: ticker_symbol, exchange_operating_mic: exchange_operating_mic ) - - FetchSecurityInfoJob.perform_later(security.id) - - security end end diff --git a/app/views/accounts/_account.html.erb b/app/views/accounts/_account.html.erb index 07ffd3d5..475f953e 100644 --- a/app/views/accounts/_account.html.erb +++ b/app/views/accounts/_account.html.erb @@ -30,9 +30,13 @@ <% end %>
"> - <%= format_money account.balance_money %> -
+ <% if account.syncing? %> + + <% else %> +"> + <%= format_money account.balance_money %> +
+ <% end %> <% unless account.scheduled_for_deletion? %> <%= styled_form_with model: account, data: { turbo_frame: "_top", controller: "auto-submit-form" } do |f| %> diff --git a/app/views/accounts/_account_sidebar_tabs.html.erb b/app/views/accounts/_account_sidebar_tabs.html.erb index 9ca2781c..056e7be1 100644 --- a/app/views/accounts/_account_sidebar_tabs.html.erb +++ b/app/views/accounts/_account_sidebar_tabs.html.erb @@ -1,6 +1,6 @@ -<%# locals: (family:, active_account_group_tab:) %> +<%# locals: (family:, active_tab:, mobile: false) %> -