From 9de1ac5ac5b8ceeb002a65cf771a041da463654b Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 10 May 2026 05:31:13 -0700 Subject: [PATCH 1/2] fix(imports): import raw balance records --- app/models/family/data_importer.rb | 50 ++++++- test/models/family/data_importer_test.rb | 168 +++++++++++++++++++++++ 2 files changed, 216 insertions(+), 2 deletions(-) diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index 171caf620..505088f99 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -1,7 +1,7 @@ require "set" class Family::DataImporter - SUPPORTED_TYPES = %w[Account Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze + SUPPORTED_TYPES = %w[Account Balance Category Tag Merchant RecurringTransaction Transaction Transfer RejectedTransfer Trade Holding Valuation Budget BudgetCategory Rule].freeze ACCOUNTABLE_TYPES = Accountable::TYPES.freeze def initialize(family, ndjson_content) @@ -30,6 +30,7 @@ def import! Import.transaction do # Import in dependency order import_accounts(records["Account"] || []) + import_balances(records["Balance"] || []) import_categories(records["Category"] || []) import_tags(records["Tag"] || []) import_merchants(records["Merchant"] || []) @@ -128,6 +129,51 @@ def importable_account_status(status) status.to_s.in?(%w[active disabled draft]) ? status.to_s : "active" end + def import_balances(records) + records.each do |record| + data = record["data"] || {} + new_account_id = @id_mappings[:accounts][data["account_id"]] + balance_date = parse_import_date(data["date"]) + next if new_account_id.blank? || balance_date.blank? || data["balance"].blank? + + account = @family.accounts.find(new_account_id) + currency = data["currency"].presence || account.currency + balance = account.balances.find_or_initialize_by(date: balance_date, currency: currency) + + balance.assign_attributes(imported_balance_attributes(data)) + balance.save! + end + end + + def imported_balance_attributes(data) + { + balance: data["balance"].to_d, + cash_balance: optional_decimal(data["cash_balance"]), + start_cash_balance: decimal_or_default(data["start_cash_balance"]), + start_non_cash_balance: decimal_or_default(data["start_non_cash_balance"]), + cash_inflows: decimal_or_default(data["cash_inflows"]), + cash_outflows: decimal_or_default(data["cash_outflows"]), + non_cash_inflows: decimal_or_default(data["non_cash_inflows"]), + non_cash_outflows: decimal_or_default(data["non_cash_outflows"]), + net_market_flows: decimal_or_default(data["net_market_flows"]), + cash_adjustments: decimal_or_default(data["cash_adjustments"]), + non_cash_adjustments: decimal_or_default(data["non_cash_adjustments"]), + flows_factor: balance_flows_factor_for(data["flows_factor"]) + } + end + + def optional_decimal(value) + value.presence&.to_d + end + + def decimal_or_default(value, default = 0) + value.present? ? value.to_d : default + end + + def balance_flows_factor_for(value) + value.to_i.in?([ -1, 1 ]) ? value.to_i : 1 + end + def import_categories(records) # First pass: create all categories without parent relationships parent_mappings = {} @@ -472,7 +518,7 @@ def oldest_import_entry_dates_by_account(records) # Account-level opening balances must precede every imported account # activity, including standalone valuation snapshots. - %w[Transaction Trade Holding Valuation].each do |type| + %w[Balance Transaction Trade Holding Valuation].each do |type| records[type].to_a.each do |record| data = record["data"] || {} account_id = data["account_id"] diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index b216acc9d..9e81f9ff6 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -77,6 +77,126 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal "active", account.status end + test "imports raw balance history records" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Balance History Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + start_cash_balance: "1000.00", + start_non_cash_balance: "0.00", + cash_inflows: "300.00", + cash_outflows: "200.00", + non_cash_inflows: "0.00", + non_cash_outflows: "0.00", + net_market_flows: "0.00", + cash_adjustments: "0.00", + non_cash_adjustments: "0.00", + flows_factor: 1 + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Balance History Checking") + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1200.0, balance.balance.to_f + assert_equal 1100.0, balance.cash_balance.to_f + assert_equal 1000.0, balance.start_cash_balance.to_f + assert_equal 300.0, balance.cash_inflows.to_f + assert_equal 200.0, balance.cash_outflows.to_f + assert_equal 1, balance.flows_factor + end + + test "imports duplicate raw balance records idempotently by account date and currency" do + balance_record = { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + flows_factor: 1 + } + } + + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Idempotent Balance Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + balance_record, + balance_record.deep_merge(data: { id: "balance-1-duplicate", balance: "1300.00", cash_balance: "1250.00" }) + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Idempotent Balance Checking") + assert_equal 1, account.balances.where(date: Date.parse("2024-01-31"), currency: "USD").count + + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + assert_equal 1300.0, balance.balance.to_f + assert_equal 1250.0, balance.cash_balance.to_f + end + + test "dates synthesized account opening balance before imported balance history" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Balance Anchored Checking", + balance: "500.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-02-01", + balance: "500.00", + currency: "USD" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Balance Anchored Checking") + opening_anchor = account.valuations.opening_anchor.first + + assert_not_nil opening_anchor + assert_equal Date.parse("2024-01-31"), opening_anchor.entry.date + end + test "dates synthesized account opening balance before oldest imported activity" do ndjson = build_ndjson([ { @@ -719,6 +839,54 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert imported_holding.security_locked end + test "round trips raw balance history through full export" do + source_family = Family.create!( + name: "Source Balance Family", + currency: "USD", + locale: "en", + date_format: "%Y-%m-%d" + ) + source_account = source_family.accounts.create!( + name: "Round Trip Balance Checking", + accountable: Depository.new, + balance: 1_500, + currency: "USD" + ) + source_account.balances.create!( + date: Date.parse("2024-01-31"), + balance: 1_500, + cash_balance: 1_450, + currency: "USD", + start_cash_balance: 1_000, + start_non_cash_balance: 0, + cash_inflows: 700, + cash_outflows: 250, + non_cash_inflows: 0, + non_cash_outflows: 0, + net_market_flows: 0, + cash_adjustments: 0, + non_cash_adjustments: 0, + flows_factor: 1 + ) + + zip_data = Family::DataExporter.new(source_family).generate_export + ndjson = nil + Zip::File.open_buffer(zip_data) do |zip| + ndjson = zip.read("all.ndjson") + end + + Family::DataImporter.new(@family, ndjson).import! + + imported_account = @family.accounts.find_by!(name: "Round Trip Balance Checking") + imported_balance = imported_account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1500.0, imported_balance.balance.to_f + assert_equal 1450.0, imported_balance.cash_balance.to_f + assert_equal 1000.0, imported_balance.start_cash_balance.to_f + assert_equal 700.0, imported_balance.cash_inflows.to_f + assert_equal 250.0, imported_balance.cash_outflows.to_f + end + test "imports holding snapshots with ticker fallback when exchange mic is missing" do existing_security = Security.create!( ticker: "VTI", From 44f09d86c01121c69f5ba7a7e9f3cc30809da948 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 11 May 2026 04:56:06 -0700 Subject: [PATCH 2/2] fix(imports): preserve partial balance components --- app/models/family/data_importer.rb | 30 +++++++------- test/models/family/data_importer_test.rb | 50 ++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/app/models/family/data_importer.rb b/app/models/family/data_importer.rb index 505088f99..7601516ce 100644 --- a/app/models/family/data_importer.rb +++ b/app/models/family/data_importer.rb @@ -146,30 +146,28 @@ def import_balances(records) end def imported_balance_attributes(data) - { + attributes = { balance: data["balance"].to_d, cash_balance: optional_decimal(data["cash_balance"]), - start_cash_balance: decimal_or_default(data["start_cash_balance"]), - start_non_cash_balance: decimal_or_default(data["start_non_cash_balance"]), - cash_inflows: decimal_or_default(data["cash_inflows"]), - cash_outflows: decimal_or_default(data["cash_outflows"]), - non_cash_inflows: decimal_or_default(data["non_cash_inflows"]), - non_cash_outflows: decimal_or_default(data["non_cash_outflows"]), - net_market_flows: decimal_or_default(data["net_market_flows"]), - cash_adjustments: decimal_or_default(data["cash_adjustments"]), - non_cash_adjustments: decimal_or_default(data["non_cash_adjustments"]), - flows_factor: balance_flows_factor_for(data["flows_factor"]) - } + start_cash_balance: optional_decimal(data["start_cash_balance"]), + start_non_cash_balance: optional_decimal(data["start_non_cash_balance"]), + cash_inflows: optional_decimal(data["cash_inflows"]), + cash_outflows: optional_decimal(data["cash_outflows"]), + non_cash_inflows: optional_decimal(data["non_cash_inflows"]), + non_cash_outflows: optional_decimal(data["non_cash_outflows"]), + net_market_flows: optional_decimal(data["net_market_flows"]), + cash_adjustments: optional_decimal(data["cash_adjustments"]), + non_cash_adjustments: optional_decimal(data["non_cash_adjustments"]) + }.compact + + attributes[:flows_factor] = balance_flows_factor_for(data["flows_factor"]) if data["flows_factor"].present? + attributes end def optional_decimal(value) value.presence&.to_d end - def decimal_or_default(value, default = 0) - value.present? ? value.to_d : default - end - def balance_flows_factor_for(value) value.to_i.in?([ -1, 1 ]) ? value.to_i : 1 end diff --git a/test/models/family/data_importer_test.rb b/test/models/family/data_importer_test.rb index 9e81f9ff6..b1a3c7454 100644 --- a/test/models/family/data_importer_test.rb +++ b/test/models/family/data_importer_test.rb @@ -164,6 +164,56 @@ class Family::DataImporterTest < ActiveSupport::TestCase assert_equal 1250.0, balance.cash_balance.to_f end + test "preserves omitted raw balance components on duplicate records" do + ndjson = build_ndjson([ + { + type: "Account", + data: { + id: "acct-1", + name: "Partial Balance Checking", + balance: "1200.00", + currency: "USD", + accountable_type: "Depository" + } + }, + { + type: "Balance", + data: { + id: "balance-1", + account_id: "acct-1", + date: "2024-01-31", + balance: "1200.00", + currency: "USD", + cash_balance: "1100.00", + cash_inflows: "300.00", + cash_outflows: "200.00", + flows_factor: -1 + } + }, + { + type: "Balance", + data: { + id: "balance-1-partial", + account_id: "acct-1", + date: "2024-01-31", + balance: "1300.00", + currency: "USD" + } + } + ]) + + Family::DataImporter.new(@family, ndjson).import! + + account = @family.accounts.find_by!(name: "Partial Balance Checking") + balance = account.balances.find_by!(date: Date.parse("2024-01-31"), currency: "USD") + + assert_equal 1300.0, balance.balance.to_f + assert_equal 1100.0, balance.cash_balance.to_f + assert_equal 300.0, balance.cash_inflows.to_f + assert_equal 200.0, balance.cash_outflows.to_f + assert_equal(-1, balance.flows_factor) + end + test "dates synthesized account opening balance before imported balance history" do ndjson = build_ndjson([ {