From dfc51a48ee7b7747a44147708cc59cb9075c3975 Mon Sep 17 00:00:00 2001 From: Clayton Date: Wed, 20 May 2026 09:56:42 -0500 Subject: [PATCH 1/4] feat: add incremental balance sync windows after provider imports --- app/models/account/balance_sync_window.rb | 41 ++++++++ app/models/account/schedules_balance_syncs.rb | 93 +++++++++++++++++++ app/models/balance/materializer.rb | 6 +- app/models/balance/reverse_calculator.rb | 50 +++++++++- app/models/binance_item.rb | 29 ++---- app/models/brex_item.rb | 28 ++---- app/models/coinbase_item.rb | 29 ++---- app/models/coinstats_item.rb | 29 ++---- app/models/enable_banking_item.rb | 27 +----- app/models/ibkr_item.rb | 28 +++--- app/models/indexa_capital_item.rb | 26 +----- app/models/kraken_item.rb | 22 +---- app/models/lunchflow_item.rb | 31 ++----- app/models/mercury_item.rb | 29 ++---- app/models/plaid_item.rb | 13 +-- app/models/simplefin_item.rb | 12 +-- app/models/snaptrade_item.rb | 33 +++---- app/models/sophtron_item.rb | 36 +++---- .../account/balance_sync_window_test.rb | 74 +++++++++++++++ .../account/schedules_balance_syncs_test.rb | 28 ++++++ .../models/balance/reverse_calculator_test.rb | 41 ++++++++ 21 files changed, 423 insertions(+), 282 deletions(-) create mode 100644 app/models/account/balance_sync_window.rb create mode 100644 app/models/account/schedules_balance_syncs.rb create mode 100644 test/models/account/balance_sync_window_test.rb create mode 100644 test/models/account/schedules_balance_syncs_test.rb diff --git a/app/models/account/balance_sync_window.rb b/app/models/account/balance_sync_window.rb new file mode 100644 index 0000000000..b57c78c309 --- /dev/null +++ b/app/models/account/balance_sync_window.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Resolves the earliest date from which an account balance sync should recalculate +# when no explicit window was passed on the parent sync. +class Account::BalanceSyncWindow + LOOKBACK = 7.days + + class << self + # @param account [Account] + # @param parent_sync [Sync, nil] Used to detect entries created/updated during this sync run + # @param parent_window_start_date [Date, nil] Explicit window from caller or parent sync + # @param import_window_start_date [Date, nil] Transaction fetch window from provider import + # @param last_synced_at [Time, nil] Provider item freshness timestamp + # @return [Date, nil] nil means full balance recalculation + def for_account(account, parent_sync: nil, parent_window_start_date: nil, import_window_start_date: nil, last_synced_at: nil) + return parent_window_start_date.to_date if parent_window_start_date.present? + + candidates = [] + candidates << import_window_start_date.to_date if import_window_start_date.present? + candidates << entries_touched_since(parent_sync, account) if parent_sync + candidates << (last_synced_at.to_date - LOOKBACK) if last_synced_at.present? + + window = candidates.compact.min + return nil unless window + + floor = [ account.opening_anchor_date, account.start_date ].compact.max + [ window, floor ].max + end + + private + + def entries_touched_since(sync, account) + sync_started_at = sync.created_at + return nil unless sync_started_at + + account.entries + .where("entries.created_at >= :t OR entries.updated_at >= :t", t: sync_started_at) + .minimum(:date) + end + end +end diff --git a/app/models/account/schedules_balance_syncs.rb b/app/models/account/schedules_balance_syncs.rb new file mode 100644 index 0000000000..899d99b574 --- /dev/null +++ b/app/models/account/schedules_balance_syncs.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +# Shared logic for provider items scheduling per-account balance syncs after import. +module Account::SchedulesBalanceSyncs + extend ActiveSupport::Concern + + # @param accounts [Enumerable, nil] Defaults to {#balance_sync_accounts} + # @return [Array, nil] Result rows when +report_results+ is true; otherwise nil + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil, import_window_start_date: nil, accounts: nil, report_results: schedule_account_syncs_report_results?) + schedule_account_syncs_for( + accounts || balance_sync_accounts, + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date, + import_window_start_date: import_window_start_date, + report_results: report_results + ) + end + + def schedule_account_syncs_for(accounts, parent_sync: nil, window_start_date: nil, window_end_date: nil, import_window_start_date: nil, report_results: schedule_account_syncs_report_results?) + return [] if report_results && accounts.blank? + + end_date = window_end_date || parent_sync&.window_end_date + last_synced = balance_sync_last_synced_at + + iterator = report_results ? :schedule_with_results : :schedule_each + send(iterator, accounts, parent_sync:, window_start_date:, end_date:, import_window_start_date:, last_synced:) + end + + private + + def schedule_account_syncs_report_results? + false + end + + def balance_sync_accounts + accts = accounts + accts.respond_to?(:visible) ? accts.visible : accts + end + + def balance_sync_last_synced_at + last_synced_at if respond_to?(:last_synced_at) + end + + def schedule_each(accounts, parent_sync:, window_start_date:, end_date:, import_window_start_date:, last_synced:) + accounts.each do |account| + schedule_balance_sync_for_account( + account, + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: end_date, + import_window_start_date: import_window_start_date, + last_synced_at: last_synced + ) + end + nil + end + + def schedule_with_results(accounts, parent_sync:, window_start_date:, end_date:, import_window_start_date:, last_synced:) + results = [] + accounts.each do |account| + schedule_balance_sync_for_account( + account, + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: end_date, + import_window_start_date: import_window_start_date, + last_synced_at: last_synced + ) + results << { account_id: account.id, success: true } + rescue => e + Rails.logger.error "#{self.class.name} #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" + results << { account_id: account.id, success: false, error: e.message } + end + results + end + + def schedule_balance_sync_for_account(account, parent_sync:, window_start_date:, window_end_date:, import_window_start_date:, last_synced_at:) + effective_window = Account::BalanceSyncWindow.for_account( + account, + parent_sync: parent_sync, + parent_window_start_date: window_start_date || parent_sync&.window_start_date, + import_window_start_date: import_window_start_date, + last_synced_at: last_synced_at + ) + + account.sync_later( + parent_sync: parent_sync, + window_start_date: effective_window, + window_end_date: window_end_date + ) + end +end diff --git a/app/models/balance/materializer.rb b/app/models/balance/materializer.rb index 4a2066454c..2b56a354cc 100644 --- a/app/models/balance/materializer.rb +++ b/app/models/balance/materializer.rb @@ -80,7 +80,7 @@ def purge_stale_balances # In incremental forward-sync, even when no balances were calculated for the window # (e.g. window_start_date is beyond the last entry), purge stale tail records that # now fall beyond the prior-balance boundary so orphaned future rows are cleaned up. - if strategy == :forward && calculator.incremental? && account.opening_anchor_date <= @window_start_date - 1 + if @window_start_date.present? && calculator.respond_to?(:incremental?) && calculator.incremental? && account.opening_anchor_date <= @window_start_date - 1 deleted_count = account.balances.delete_by( "date < ? OR date > ?", account.opening_anchor_date, @@ -98,7 +98,7 @@ def purge_stale_balances # Use opening_anchor_date as the lower purge bound to preserve them. # We ask the calculator whether it actually ran incrementally — it may have # fallen back to a full recalculation, in which case we use the normal bound. - oldest_valid_date = if strategy == :forward && calculator.incremental? + oldest_valid_date = if calculator.respond_to?(:incremental?) && calculator.incremental? account.opening_anchor_date else sorted_balances.first.date @@ -110,7 +110,7 @@ def purge_stale_balances def calculator @calculator ||= if strategy == :reverse - Balance::ReverseCalculator.new(account) + Balance::ReverseCalculator.new(account, window_start_date: @window_start_date) else Balance::ForwardCalculator.new(account, window_start_date: @window_start_date) end diff --git a/app/models/balance/reverse_calculator.rb b/app/models/balance/reverse_calculator.rb index 39b5301784..f883d20c39 100644 --- a/app/models/balance/reverse_calculator.rb +++ b/app/models/balance/reverse_calculator.rb @@ -1,4 +1,16 @@ class Balance::ReverseCalculator < Balance::BaseCalculator + def initialize(account, window_start_date: nil) + super(account) + @window_start_date = window_start_date + @fell_back = nil + end + + # True when a window was provided and we successfully limited recalculation to that range. + def incremental? + raise "incremental? must not be called before calculate" if @window_start_date.present? && @fell_back.nil? + @window_start_date.present? && @fell_back == false + end + def calculate Rails.logger.tagged("Balance::ReverseCalculator") do # Since it's a reverse sync, we're starting with the "end of day" balance components and @@ -9,8 +21,10 @@ def calculate ) end_non_cash_balance = account.current_anchor_balance - end_cash_balance + calc_start_date = resolve_calc_start_date + # Calculates in reverse-chronological order (End of day -> Start of day) - account.current_anchor_date.downto(account.opening_anchor_date).map do |date| + account.current_anchor_date.downto(calc_start_date).map do |date| flows = flows_for_date(date) valuation = sync_cache.get_valuation(date) @@ -68,6 +82,40 @@ def calculate private + def resolve_calc_start_date + if @window_start_date.present? + if multi_currency_account? + Rails.logger.info("Account has multi-currency entries or is foreign, falling back to full reverse recalculation") + @fell_back = true + return account.opening_anchor_date + end + + prior = prior_balance + + if prior + Rails.logger.info("Incremental reverse sync from #{@window_start_date}, preserving balances before #{@window_start_date}") + @fell_back = false + return [ @window_start_date, account.opening_anchor_date ].max + else + Rails.logger.info("No persisted balance found for #{@window_start_date - 1}, falling back to full reverse recalculation") + @fell_back = true + end + end + + account.opening_anchor_date + end + + def multi_currency_account? + account.entries.where.not(currency: account.currency).exists? || + account.currency != account.family.currency + end + + def prior_balance + account.balances + .where(currency: account.currency) + .find_by(date: @window_start_date - 1) + end + # Negative entries amount on an "asset" account means, "account value has increased" # Negative entries amount on a "liability" account means, "account debt has decreased" # Positive entries amount on an "asset" account means, "account value has decreased" diff --git a/app/models/binance_item.rb b/app/models/binance_item.rb index b482cd5015..60aa36c8fb 100644 --- a/app/models/binance_item.rb +++ b/app/models/binance_item.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class BinanceItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -78,27 +78,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue StandardError => e - Rails.logger.error "BinanceItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - def upsert_binance_snapshot!(payload) update!(raw_payload: payload) end @@ -156,4 +135,10 @@ def set_binance_institution_defaults! institution_color: "#F0B90B" ) end + + private + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/brex_item.rb b/app/models/brex_item.rb index 865797e65f..482d1281e6 100644 --- a/app/models/brex_item.rb +++ b/app/models/brex_item.rb @@ -1,5 +1,5 @@ class BrexItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs BLANK_TOKEN_SENTINELS = [ "", " ", " ", " ", "\t", "\n", "\r" ].freeze @@ -75,27 +75,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "BrexItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - def upsert_brex_snapshot!(accounts_snapshot) update!(raw_payload: BrexAccount.sanitize_payload(accounts_snapshot)) end @@ -168,6 +147,11 @@ def effective_base_url end private + + def schedule_account_syncs_report_results? + true + end + def normalize_token self.token = token&.strip end diff --git a/app/models/coinbase_item.rb b/app/models/coinbase_item.rb index 0adf2fe118..6cc9bba230 100644 --- a/app/models/coinbase_item.rb +++ b/app/models/coinbase_item.rb @@ -1,5 +1,5 @@ class CoinbaseItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs include Encryptable enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -78,27 +78,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "CoinbaseItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - def upsert_coinbase_snapshot!(accounts_snapshot) assign_attributes( raw_payload: accounts_snapshot @@ -174,4 +153,10 @@ def set_coinbase_institution_defaults! institution_color: "#0052FF" ) end + + private + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/coinstats_item.rb b/app/models/coinstats_item.rb index 6a1bf82917..6b7af5f576 100644 --- a/app/models/coinstats_item.rb +++ b/app/models/coinstats_item.rb @@ -1,7 +1,7 @@ # Represents a CoinStats API connection for a family. # Stores credentials and manages associated wallet and exchange portfolio accounts. class CoinstatsItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -77,27 +77,6 @@ def process_accounts # @param window_start_date [Date, nil] Start of sync window # @param window_end_date [Date, nil] End of sync window # @return [Array] Results with success status per account - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "CoinstatsItem #{id} - Failed to schedule sync for wallet #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - # Persists raw API response for debugging and reprocessing. # @param accounts_snapshot [Hash] Raw API response data def upsert_coinstats_snapshot!(accounts_snapshot) @@ -153,4 +132,10 @@ def credentials_configured? def exchange_configured? exchange_portfolio_id.present? && exchange_connection_id.present? end + + private + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/enable_banking_item.rb b/app/models/enable_banking_item.rb index 7f3b6a5405..4c6fcf8549 100644 --- a/app/models/enable_banking_item.rb +++ b/app/models/enable_banking_item.rb @@ -1,5 +1,5 @@ class EnableBankingItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -166,27 +166,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "EnableBankingItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - def upsert_enable_banking_snapshot!(accounts_snapshot) assign_attributes( raw_payload: accounts_snapshot @@ -288,6 +267,10 @@ def revoke_session private + def schedule_account_syncs_report_results? + true + end + def parse_session_expiry(session_result) if session_result[:access].present? && session_result[:access][:valid_until].present? parsed = Time.zone.parse(session_result[:access][:valid_until]) diff --git a/app/models/ibkr_item.rb b/app/models/ibkr_item.rb index aca60d1b22..003ca52f6e 100644 --- a/app/models/ibkr_item.rb +++ b/app/models/ibkr_item.rb @@ -1,5 +1,5 @@ class IbkrItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -60,22 +60,6 @@ def process_accounts end end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - accounts.reject { |account| account.pending_deletion? || account.disabled? }.each_with_object([]) do |account, results| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error("IbkrItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}") - results << { account_id: account.id, success: false, error: e.message } - end - end - end - def upsert_ibkr_snapshot!(payload) update!(raw_payload: payload, status: :good) end @@ -121,4 +105,14 @@ def sync_status_summary def institution_display_name I18n.t("ibkr_items.defaults.name") end + + private + + def balance_sync_accounts + accounts.reject { |account| account.pending_deletion? || account.disabled? } + end + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/indexa_capital_item.rb b/app/models/indexa_capital_item.rb index 53bd562a80..4c72a175f9 100644 --- a/app/models/indexa_capital_item.rb +++ b/app/models/indexa_capital_item.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class IndexaCapitalItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -81,26 +81,6 @@ def process_accounts end # Schedule sync jobs for all linked accounts - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "IndexaCapitalItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end def upsert_indexa_capital_snapshot!(accounts_snapshot) assign_attributes( @@ -178,4 +158,8 @@ def credentials_present_on_create errors.add(:base, :credentials_required) end + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/kraken_item.rb b/app/models/kraken_item.rb index 2c47d5f8ba..367aa6f391 100644 --- a/app/models/kraken_item.rb +++ b/app/models/kraken_item.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class KrakenItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -61,22 +61,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - accounts.visible.map do |account| - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - { account_id: account.id, success: true } - rescue StandardError => e - Rails.logger.error "KrakenItem #{id} - Failed to schedule sync for account #{account.id}: #{e.full_message}" - { account_id: account.id, success: false, error: e.message } - end - end - def upsert_kraken_snapshot!(payload) update!(raw_payload: payload) end @@ -146,6 +130,10 @@ def set_kraken_institution_defaults! private + def schedule_account_syncs_report_results? + true + end + def strip_credentials self.api_key = api_key.to_s.strip if api_key_changed? && !api_key.nil? self.api_secret = api_secret.to_s.strip if api_secret_changed? && !api_secret.nil? diff --git a/app/models/lunchflow_item.rb b/app/models/lunchflow_item.rb index 94cf078e7a..8ba85defca 100644 --- a/app/models/lunchflow_item.rb +++ b/app/models/lunchflow_item.rb @@ -1,5 +1,5 @@ class LunchflowItem < ApplicationRecord - include Syncable, Provided, Unlinking, Encryptable + include Syncable, Provided, Unlinking, Encryptable, Account::SchedulesBalanceSyncs DEFAULT_BASE_URL = "https://lunchflow.app/api/v1".freeze @@ -63,29 +63,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - # Only schedule syncs for active accounts - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "LunchflowItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - # Continue scheduling other accounts even if one fails - end - end - - results - end - def upsert_lunchflow_snapshot!(accounts_snapshot) assign_attributes( raw_payload: accounts_snapshot @@ -169,4 +146,10 @@ def effective_base_url rescue URI::InvalidURIError DEFAULT_BASE_URL end + + private + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/mercury_item.rb b/app/models/mercury_item.rb index 5869156fb9..08a1a314e9 100644 --- a/app/models/mercury_item.rb +++ b/app/models/mercury_item.rb @@ -1,5 +1,5 @@ class MercuryItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -76,27 +76,6 @@ def process_accounts # TODO: Customize sync scheduling if needed # This method schedules sync jobs for all linked accounts. - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - return [] if accounts.empty? - - results = [] - accounts.visible.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "MercuryItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end - def upsert_mercury_snapshot!(accounts_snapshot) assign_attributes( raw_payload: accounts_snapshot @@ -174,4 +153,10 @@ def credentials_configured? def effective_base_url base_url.presence || "https://api.mercury.com/api/v1" end + + private + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 58e39bae24..517e97aa7d 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -1,5 +1,5 @@ class PlaidItem < ApplicationRecord - include Syncable, Provided, Encryptable + include Syncable, Provided, Encryptable, Account::SchedulesBalanceSyncs enum :plaid_region, { us: "us", eu: "eu" } enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -74,17 +74,6 @@ def process_accounts end end - # Once all the data is fetched, we can schedule account syncs to calculate historical balances - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - accounts.each do |account| - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - end - end - # Saves the raw data fetched from Plaid API for this item def upsert_plaid_snapshot!(item_snapshot) assign_attributes( diff --git a/app/models/simplefin_item.rb b/app/models/simplefin_item.rb index 4483393f4c..45f9ac0a0a 100644 --- a/app/models/simplefin_item.rb +++ b/app/models/simplefin_item.rb @@ -1,5 +1,5 @@ class SimplefinItem < ApplicationRecord - include Syncable, Provided, Encryptable + include Syncable, Provided, Encryptable, Account::SchedulesBalanceSyncs include SimplefinItem::Unlinking enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -230,16 +230,6 @@ def merge_transactions(old_txns, new_txns) by_id.values end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - accounts.each do |account| - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - end - end - def upsert_simplefin_snapshot!(accounts_snapshot) assign_attributes( raw_payload: accounts_snapshot, diff --git a/app/models/snaptrade_item.rb b/app/models/snaptrade_item.rb index 367836e42c..0e436a4ec6 100644 --- a/app/models/snaptrade_item.rb +++ b/app/models/snaptrade_item.rb @@ -1,5 +1,5 @@ class SnaptradeItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs enum :status, { good: "good", requires_update: "requires_update" }, default: :good @@ -87,27 +87,6 @@ def process_accounts results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil) - linked_accounts = accounts.reject { |a| a.pending_deletion? || a.disabled? } - return [] if linked_accounts.empty? - - results = [] - linked_accounts.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "SnaptradeItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - end - end - - results - end def upsert_snaptrade_snapshot!(accounts_snapshot) assign_attributes( @@ -218,4 +197,14 @@ def brokerage_summary I18n.t("snaptrade_item.brokerage_summary.count", count: brokerages.count) end end + + private + + def balance_sync_accounts + accounts.reject { |a| a.pending_deletion? || a.disabled? } + end + + def schedule_account_syncs_report_results? + true + end end diff --git a/app/models/sophtron_item.rb b/app/models/sophtron_item.rb index aa61946813..9b5f685d50 100644 --- a/app/models/sophtron_item.rb +++ b/app/models/sophtron_item.rb @@ -12,7 +12,7 @@ # @attr [Boolean] scheduled_for_deletion Whether the item is scheduled for deletion # @attr [DateTime] last_synced_at When the last successful sync occurred class SophtronItem < ApplicationRecord - include Syncable, Provided, Unlinking + include Syncable, Provided, Unlinking, Account::SchedulesBalanceSyncs INITIAL_LOAD_LOOKBACK_DAYS = 120 MAX_TRANSACTION_HISTORY_YEARS = 3 @@ -148,28 +148,16 @@ def process_accounts(sophtron_accounts_scope: linked_visible_sophtron_accounts) results end - def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil, sophtron_accounts_scope: linked_visible_sophtron_accounts) + def schedule_account_syncs(parent_sync: nil, window_start_date: nil, window_end_date: nil, import_window_start_date: nil, sophtron_accounts_scope: linked_visible_sophtron_accounts) linked_accounts = sophtron_accounts_scope.includes(:account_provider).filter_map(&:current_account) - return [] if linked_accounts.empty? - - results = [] - # Only schedule syncs for active accounts - linked_accounts.each do |account| - begin - account.sync_later( - parent_sync: parent_sync, - window_start_date: window_start_date, - window_end_date: window_end_date - ) - results << { account_id: account.id, success: true } - rescue => e - Rails.logger.error "SophtronItem #{id} - Failed to schedule sync for account #{account.id}: #{e.message}" - results << { account_id: account.id, success: false, error: e.message } - # Continue scheduling other accounts even if one fails - end - end - - results + schedule_account_syncs_for( + linked_accounts, + parent_sync: parent_sync, + window_start_date: window_start_date, + window_end_date: window_end_date, + import_window_start_date: import_window_start_date, + report_results: true + ) end def start_initial_load_later @@ -390,6 +378,10 @@ def generated_customer_name private + def schedule_account_syncs_report_results? + true + end + def find_matching_customer(customers) customers = Array(customers) diff --git a/test/models/account/balance_sync_window_test.rb b/test/models/account/balance_sync_window_test.rb new file mode 100644 index 0000000000..e2e783f751 --- /dev/null +++ b/test/models/account/balance_sync_window_test.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require "test_helper" + +class Account::BalanceSyncWindowTest < ActiveSupport::TestCase + setup do + @account = accounts(:depository) + @family = @account.family + end + + test "returns explicit parent window when provided" do + explicit = 10.days.ago.to_date + + window = Account::BalanceSyncWindow.for_account( + @account, + parent_window_start_date: explicit + ) + + assert_equal explicit, window + end + + test "derives window from last_synced_at lookback when no explicit window" do + last_synced = 2.days.ago + + window = Account::BalanceSyncWindow.for_account( + @account, + last_synced_at: last_synced + ) + + assert_equal (last_synced.to_date - Account::BalanceSyncWindow::LOOKBACK), window + end + + test "uses earliest entry touched since parent sync" do + parent_sync = @family.syncs.create! + touched_date = 4.days.ago.to_date + + @account.entries.create!( + name: "Recent import", + date: touched_date, + amount: -50, + currency: "USD", + entryable: Transaction.new, + created_at: parent_sync.created_at + 1.minute, + updated_at: parent_sync.created_at + 1.minute + ) + + window = Account::BalanceSyncWindow.for_account(@account, parent_sync: parent_sync) + + assert_equal touched_date, window + end + + test "floors derived window at opening_anchor_date" do + anchor_date = 3.days.ago.to_date + @account.entries.create!( + name: "Opening", + date: anchor_date, + amount: 1000, + currency: "USD", + entryable: Valuation.new(kind: "opening_anchor") + ) + + window = Account::BalanceSyncWindow.for_account( + @account, + last_synced_at: 1.day.ago, + import_window_start_date: 30.days.ago.to_date + ) + + assert_equal anchor_date, window + end + + test "returns nil when no window signals are available" do + assert_nil Account::BalanceSyncWindow.for_account(@account) + end +end diff --git a/test/models/account/schedules_balance_syncs_test.rb b/test/models/account/schedules_balance_syncs_test.rb new file mode 100644 index 0000000000..0e9b8397ba --- /dev/null +++ b/test/models/account/schedules_balance_syncs_test.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "test_helper" + +class Account::SchedulesBalanceSyncsTest < ActiveSupport::TestCase + setup do + @item = plaid_items(:one) + @account = accounts(:depository) + @parent_sync = @item.syncs.create! + end + + test "schedule_account_syncs_for passes derived incremental window to account sync" do + expected_window = 5.days.ago.to_date + + Account::BalanceSyncWindow.stubs(:for_account).returns(expected_window) + + @account.expects(:sync_later).with( + parent_sync: @parent_sync, + window_start_date: expected_window, + window_end_date: nil + ).once + + @item.schedule_account_syncs_for( + [ @account ], + parent_sync: @parent_sync + ) + end +end diff --git a/test/models/balance/reverse_calculator_test.rb b/test/models/balance/reverse_calculator_test.rb index cee497bc84..40c0cb9152 100644 --- a/test/models/balance/reverse_calculator_test.rb +++ b/test/models/balance/reverse_calculator_test.rb @@ -626,4 +626,45 @@ class Balance::ReverseCalculatorTest < ActiveSupport::TestCase ] ) end + + test "incremental reverse sync only recalculates from window_start_date when prior balance exists" do + window_start = 2.days.ago.to_date + + account = create_account_with_ledger( + account: { type: Depository, balance: 20000, cash_balance: 20000, currency: "USD" }, + entries: [ + { type: "current_anchor", date: Date.current, balance: 20000 }, + { type: "opening_anchor", date: 10.days.ago, balance: 15000 } + ] + ) + + create_balance( + account: account, + date: window_start - 1, + balance: 18000, + cash_balance: 18000 + ) + + calculator = Balance::ReverseCalculator.new(account, window_start_date: window_start) + calculated = calculator.calculate + + assert calculator.incremental? + assert_equal (Date.current.downto(window_start).to_a), calculated.map(&:date) + end + + test "incremental reverse sync falls back to full recalculation without prior balance" do + account = create_account_with_ledger( + account: { type: Depository, balance: 20000, cash_balance: 20000, currency: "USD" }, + entries: [ + { type: "current_anchor", date: Date.current, balance: 20000 }, + { type: "opening_anchor", date: 5.days.ago, balance: 15000 } + ] + ) + + calculator = Balance::ReverseCalculator.new(account, window_start_date: 2.days.ago.to_date) + calculated = calculator.calculate + + assert_not calculator.incremental? + assert_equal (Date.current.downto(account.opening_anchor_date).to_a), calculated.map(&:date) + end end From e3b6a19d74bd3ad3f647eb6f01edcbba3db653bc Mon Sep 17 00:00:00 2001 From: Clayton Date: Wed, 20 May 2026 10:04:48 -0500 Subject: [PATCH 2/4] fix: test --- test/models/balance/reverse_calculator_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/models/balance/reverse_calculator_test.rb b/test/models/balance/reverse_calculator_test.rb index 40c0cb9152..2781388435 100644 --- a/test/models/balance/reverse_calculator_test.rb +++ b/test/models/balance/reverse_calculator_test.rb @@ -2,6 +2,7 @@ class Balance::ReverseCalculatorTest < ActiveSupport::TestCase include LedgerTestingHelper + include BalanceTestHelper # When syncing backwards, we start with the account balance and generate everything from there. test "when missing anchor and no entries, falls back to cached account balance" do From fb05e91282b2b34bf623cc05e255f84c5c27b428 Mon Sep 17 00:00:00 2001 From: Clayton Date: Wed, 20 May 2026 10:15:11 -0500 Subject: [PATCH 3/4] fix: test --- test/models/account/balance_sync_window_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/models/account/balance_sync_window_test.rb b/test/models/account/balance_sync_window_test.rb index e2e783f751..872dac0c7a 100644 --- a/test/models/account/balance_sync_window_test.rb +++ b/test/models/account/balance_sync_window_test.rb @@ -21,13 +21,15 @@ class Account::BalanceSyncWindowTest < ActiveSupport::TestCase test "derives window from last_synced_at lookback when no explicit window" do last_synced = 2.days.ago + lookback_start = last_synced.to_date - Account::BalanceSyncWindow::LOOKBACK + floor = [ @account.opening_anchor_date, @account.start_date ].compact.max window = Account::BalanceSyncWindow.for_account( @account, last_synced_at: last_synced ) - assert_equal (last_synced.to_date - Account::BalanceSyncWindow::LOOKBACK), window + assert_equal [ lookback_start, floor ].compact.max, window end test "uses earliest entry touched since parent sync" do From ef25a165ab59b14df4644c757e2a22c4b207a9f0 Mon Sep 17 00:00:00 2001 From: Clayton Date: Wed, 20 May 2026 22:35:25 -0500 Subject: [PATCH 4/4] fix: test --- app/models/balance/materializer.rb | 1 + app/models/coinstats_item.rb | 5 ---- app/models/plaid_item.rb | 7 +++++ app/models/simplefin_item.rb | 7 +++++ .../account/schedules_balance_syncs_test.rb | 27 +++++++++++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/app/models/balance/materializer.rb b/app/models/balance/materializer.rb index 2b56a354cc..0f11843085 100644 --- a/app/models/balance/materializer.rb +++ b/app/models/balance/materializer.rb @@ -73,6 +73,7 @@ def persist_balances ) end + # Called only from #materialize_balances after #calculate_balances, so calculator.incremental? is safe. def purge_stale_balances sorted_balances = @balances.sort_by(&:date) diff --git a/app/models/coinstats_item.rb b/app/models/coinstats_item.rb index 6b7af5f576..aa7b5ef5a7 100644 --- a/app/models/coinstats_item.rb +++ b/app/models/coinstats_item.rb @@ -72,11 +72,6 @@ def process_accounts results end - # Queues balance sync jobs for all visible accounts. - # @param parent_sync [Sync, nil] Parent sync for tracking - # @param window_start_date [Date, nil] Start of sync window - # @param window_end_date [Date, nil] End of sync window - # @return [Array] Results with success status per account # Persists raw API response for debugging and reprocessing. # @param accounts_snapshot [Hash] Raw API response data def upsert_coinstats_snapshot!(accounts_snapshot) diff --git a/app/models/plaid_item.rb b/app/models/plaid_item.rb index 517e97aa7d..0905fb5837 100644 --- a/app/models/plaid_item.rb +++ b/app/models/plaid_item.rb @@ -102,6 +102,13 @@ def supports_product?(product) end private + + # Plaid #accounts returns a plain Array (not an Account relation), so schedule all + # linked accounts — including disabled/pending-deletion — matching pre-refactor behavior. + def balance_sync_accounts + accounts + end + def remove_plaid_item return unless plaid_provider.present? diff --git a/app/models/simplefin_item.rb b/app/models/simplefin_item.rb index 45f9ac0a0a..32e384d581 100644 --- a/app/models/simplefin_item.rb +++ b/app/models/simplefin_item.rb @@ -454,6 +454,13 @@ def stale_pending_status(days: 8) end private + + # SimpleFIN #accounts returns a plain Array (not an Account relation), so schedule all + # linked accounts — including disabled/pending-deletion — matching pre-refactor behavior. + def balance_sync_accounts + accounts + end + # Parse sync_stats, handling cases where it might be a raw JSON string # (e.g., from console testing or bypassed serialization) def parse_sync_stats(sync_stats) diff --git a/test/models/account/schedules_balance_syncs_test.rb b/test/models/account/schedules_balance_syncs_test.rb index 0e9b8397ba..bd68fb619e 100644 --- a/test/models/account/schedules_balance_syncs_test.rb +++ b/test/models/account/schedules_balance_syncs_test.rb @@ -25,4 +25,31 @@ class Account::SchedulesBalanceSyncsTest < ActiveSupport::TestCase parent_sync: @parent_sync ) end + + test "schedule_account_syncs_for continues scheduling remaining accounts when one fails" do + other_account = accounts(:credit_card) + expected_window = 5.days.ago.to_date + + Account::BalanceSyncWindow.stubs(:for_account).returns(expected_window) + + @account.expects(:sync_later).raises(StandardError, "sync failed") + other_account.expects(:sync_later).with( + parent_sync: @parent_sync, + window_start_date: expected_window, + window_end_date: nil + ).once + + results = @item.schedule_account_syncs_for( + [ @account, other_account ], + parent_sync: @parent_sync, + report_results: true + ) + + assert_equal 2, results.size + assert_equal @account.id, results[0][:account_id] + assert_equal false, results[0][:success] + assert_equal "sync failed", results[0][:error] + assert_equal other_account.id, results[1][:account_id] + assert_equal true, results[1][:success] + end end